diff --git a/converters/salesforce/README.md b/converters/salesforce/README.md new file mode 100644 index 00000000..b1599de6 --- /dev/null +++ b/converters/salesforce/README.md @@ -0,0 +1,221 @@ +# OSI Salesforce Converter + +A two-way converter between [OSI semantic models](../../core-spec/spec.md) and [Salesforce Semantic Model](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html). + +This converter provides lossless, bidirectional conversion between OSI YAML format and Salesforce Semantic Model JSON format. + +## Requirements + +- **Java 17+** +- **Maven 3.6+** — required to build the jar + +## Building + +Build the executable jar from source: + +```bash +mvn clean package +``` + +This produces a self-contained executable jar at `target/osi-salesforce-converter-0.1.0-SNAPSHOT.jar` with all dependencies bundled. + +## Setup + +Both schemas must be obtained and placed under `src/main/resources/schemas/` before building, so they get bundled into the jar. + +### Salesforce Semantic Model Schema + +1. Visit the [Salesforce Semantic Model Schema documentation](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html) +2. Copy the JSON schema content from the page +3. Save it to `src/main/resources/schemas/salesforce-semantic-model-schema.json` + +### OSI Schema + +1. Visit the [OSI schema on GitHub](https://github.com/open-semantic-interchange/OSI/blob/main/core-spec/osi-schema.json) +2. Copy the raw JSON contents +3. Save it to `src/main/resources/schemas/osi-schema.json` + +## Usage + +### Command Line + +#### Import (Salesforce → OSI) + +Convert a Salesforce Semantic Model JSON file to OSI YAML format: + +```bash +java -jar target/osi-salesforce-converter-0.1.0-SNAPSHOT.jar toOSI input.json +# Output: Customer_Orders_Model.yaml (named after model's 'name' field) +# Created in the same directory as the input file +``` + +Example: +```bash +java -jar target/osi-salesforce-converter-0.1.0-SNAPSHOT.jar toOSI \ + src/test/resources/examples/salesforceToOsi.json +# Output: src/test/resources/examples/Customer_Orders_Model.yaml +``` + +#### Export (OSI → Salesforce) + +Convert an OSI YAML file to Salesforce Semantic Model JSON format: + +```bash +java -jar target/osi-salesforce-converter-0.1.0-SNAPSHOT.jar toSF input.yaml +# Output: Customer_Orders_Model.json (named after model's 'apiName' field) +# Created in the same directory as the input file +``` + +Example: +```bash +java -jar target/osi-salesforce-converter-0.1.0-SNAPSHOT.jar toSF \ + src/test/resources/examples/osiToSalesforce.yaml +# Output: src/test/resources/examples/Customer_Orders_Model.json +``` + +### Programmatic API + +#### String Conversion + +```java +import org.osi.converter.Converter; +import org.osi.converter.ConverterFactory; +import org.osi.converter.ConversionDirection; + +Converter sfToOsi = ConverterFactory.getConverter(ConversionDirection.SALESFORCE_TO_OSI); +List osiYamlList = sfToOsi.convert(salesforceJsonString); +String osiYaml = osiYamlList.get(0); + +Converter osiToSf = ConverterFactory.getConverter(ConversionDirection.OSI_TO_SALESFORCE); +List salesforceJsonList = osiToSf.convert(osiYamlString); +``` + +#### File Conversion + +```java +import org.osi.converter.Converter; +import org.osi.converter.ConverterFactory; +import org.osi.converter.ConversionDirection; + +import java.nio.file.Paths; + +Converter sfToOsi = ConverterFactory.getConverter(ConversionDirection.SALESFORCE_TO_OSI); +sfToOsi.convert(Paths.get("input/model.json"), Paths.get("output/")); + +Converter osiToSf = ConverterFactory.getConverter(ConversionDirection.OSI_TO_SALESFORCE); +osiToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/")); +``` + +### Features + +- **Schema-validated** - Input is validated against JSON Schema before processing +- **Lossless conversion** - Unmapped properties are preserved in `custom_extensions` +- **Bidirectional** - Full bi-directional conversion without data loss +- **Supports OSI Specification v0.1.1** + +## Mapping Reference + +### Import (Salesforce → OSI) + +| Salesforce | OSI | +|------------|-----| +| `apiName` | `name` | +| `semanticDataObjects[]` | `datasets[]` | +| `semanticDataObjects[].apiName` | `datasets[].name` | +| `semanticDataObjects[].dataObjectName` | `datasets[].source` | +| `semanticDimensions[]` + `semanticMeasurements[]` | `fields[]` | +| `dataObjectFieldName` | `expression.dialects[].expression` | +| `semanticRelationships[]` | `relationships[]` | +| `criteria[]` | `from_columns` + `to_columns` | +| `semanticCalculatedMeasurements[]` | `metrics[]` | +| `semanticCalculatedDimensions[]` | Converted to `fields[]` if single data object dependency, otherwise stored in `custom_extensions` | +| `businessPreferences` | `ai_context` | +| Unmapped properties | `custom_extensions` (vendor: `SALESFORCE`) | + +### Export (OSI → Salesforce) + +| OSI | Salesforce | +|-----|------------| +| `name` | `apiName` | +| `datasets[]` | `semanticDataObjects[]` | +| `datasets[].name` | `semanticDataObjects[].apiName` | +| `datasets[].source` | `semanticDataObjects[].dataObjectName` | +| `fields[]` | Split into `semanticDimensions[]` and `semanticMeasurements[]` based on `expression` analysis | +| `expression.dialects[].expression` | `dataObjectFieldName` | +| `relationships[]` | `semanticRelationships[]` | +| `from_columns` + `to_columns` | `criteria[]` | +| `metrics[]` | `semanticCalculatedMeasurements[]` | +| `ai_context` | `businessPreferences` | +| `custom_extensions` (vendor: `SALESFORCE`) | Restored properties | + +### Type Detection (Export) + +Fields are automatically classified as dimensions or measurements based on expression analysis: + +- **Measurements** — expressions containing SQL aggregation functions (`SUM`, `COUNT`, `AVG`, etc.) +- **Dimensions** — all other fields +- **Time dimensions** — Date/DateTime types set `dimension.is_time: true` + +### Relationship Handling + +**Unsupported relationships** (containing Formula or SemanticField types) are stored in `custom_extensions` at the model level rather than being converted to OSI relationships. + +## Architecture + +``` + ┌───────────────────────┐ + │ OsiSalesforceConverter│ + │ (CLI App) │ + └───────────┬───────────┘ + │ + ┌───────┴────────┐ + │ ConverterFactory│ + └───────┬────────┘ + │ + ┌─────────────┴─────────────┐ + │ ConverterImpl │ + │ (Pipeline-based) │ + │ │ + │ • Configurable pipeline │ + │ • Bidirectional mapping │ + └─────────────┬─────────────┘ + │ + ┌─────────────┴─────────────┐ + │ Pipeline Handlers │ + ├───────────────────────────┤ + │ • DatasetMappingHandler │ + │ • FieldMappingHandler │ + │ • RelationshipHandler │ + │ • MetricMappingHandler │ + │ • SemanticModelHandler │ + └─────────────┬─────────────┘ + │ + ┌─────────────┴─────────────┐ + │ Support Components │ + ├───────────────────────────┤ + │ • GenericMappingEngine │ + │ • CustomExtensionHandler │ + │ • SchemaValidator │ + └───────────────────────────┘ +``` + +**ConverterFactory** — Creates converter instances for specified direction + +**Pipeline Configuration** — Handlers and direction-specific settings defined in `osi-salesforce-converter-config.yaml` + +**GenericMappingEngine** — Path-based property mapping using `mappings.yaml` configuration + +**CustomExtensionHandler** — Preserves unmapped Salesforce properties in OSI's `custom_extensions` for lossless bi-directional conversion + +**SchemaValidator** — Validates input against JSON schemas before conversion + +## Examples + +See the test suite for sample models demonstrating various features: +- `src/test/resources/examples/osiToSalesforce.yaml` - OSI model example +- `src/test/java/org/osi/OsiToSalesforceConverterTest.java` - OSI to Salesforce conversion tests +- `src/test/java/org/osi/SalesforceToOsiConverterTest.java` - Salesforce to OSI conversion tests + +## License + +Apache License 2.0 — see [LICENSE](../../LICENSE). diff --git a/converters/salesforce/pom.xml b/converters/salesforce/pom.xml new file mode 100644 index 00000000..14419e8c --- /dev/null +++ b/converters/salesforce/pom.xml @@ -0,0 +1,107 @@ + + + 4.0.0 + + org.osi + osi-salesforce-converter + 0.1.0-SNAPSHOT + jar + + OSI Salesforce Converter + OSI Salesforce bidirectional semantic model converter + + + 17 + 17 + 17 + 17 + UTF-8 + + 2.18.6 + 5.11.4 + 2.0.16 + 1.5.25 + 1.5.5 + 3.6.0 + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${jackson.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + ch.qos.logback + logback-classic + ${logback.version} + runtime + + + + com.networknt + json-schema-validator + ${json-schema-validator.version} + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + false + + + org.osi.app.OsiSalesforceConverter + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + diff --git a/converters/salesforce/src/main/java/org/osi/app/OsiSalesforceConverter.java b/converters/salesforce/src/main/java/org/osi/app/OsiSalesforceConverter.java new file mode 100644 index 00000000..2b0fea23 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/app/OsiSalesforceConverter.java @@ -0,0 +1,71 @@ +package org.osi.app; + +import org.osi.converter.Converter; +import org.osi.converter.ConverterFactory; +import org.osi.converter.ConversionDirection; +import org.osi.exception.ConversionException; +import org.osi.exception.InvalidInputException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Main application class for the OSI-Salesforce Converter. + * + *

Converts between Salesforce Semantic Model and OSI formats. + * Output file is placed in the same directory as input with the appropriate extension.

+ * + */ +public class OsiSalesforceConverter { + + public static void main(String[] args) { + if (args.length < 2) { + System.exit(1); + } + + try { + OsiSalesforceConverter app = new OsiSalesforceConverter(); + String directionArg = args[0]; + Path inputPath = Paths.get(args[1]); + + ConversionDirection direction = parseDirection(directionArg); + app.convert(direction, inputPath); + } catch (InvalidInputException e) { + System.exit(2); + } catch (ConversionException e) { + System.exit(3); + } + } + + private static ConversionDirection parseDirection(String direction) { + return switch (direction.toLowerCase()) { + case "tosf" -> ConversionDirection.OSI_TO_SALESFORCE; + case "toosi" -> ConversionDirection.SALESFORCE_TO_OSI; + default -> throw new InvalidInputException( + "Invalid direction: " + direction + ". Expected: toSF or toOSI" + ); + }; + } + + /** + * Converts a file. Output files are written to the same directory as the input file, + * with filenames based on model apiNames. + * + * @param direction The conversion direction + * @param inputPath path to the input file + */ + public void convert(ConversionDirection direction, Path inputPath) { + if (!Files.exists(inputPath)) { + throw new InvalidInputException("Input file not found: " + inputPath); + } + + Path outputDir = inputPath.getParent(); + if (outputDir == null) { + outputDir = Path.of("."); + } + + Converter converter = ConverterFactory.getConverter(direction); + converter.convert(inputPath, outputDir); + } + +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/AbstractConverter.java b/converters/salesforce/src/main/java/org/osi/converter/AbstractConverter.java new file mode 100644 index 00000000..dd18967f --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/AbstractConverter.java @@ -0,0 +1,197 @@ +package org.osi.converter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import org.osi.exception.ConversionException; +import org.osi.exception.InvalidInputException; +import org.osi.mapper.FileBasedPropertyMapper; +import org.osi.mapper.PropertyMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Abstract base class for converters that provides common functionality + * for YAML and JSON conversion with property mapping support. + * + */ +public abstract class AbstractConverter implements Converter { + + private static final String MAPPING_RESOURCE = "/mappings.yaml"; + private static final Logger logger = LoggerFactory.getLogger(AbstractConverter.class); + + + protected final PropertyMapper mapper; + protected final ObjectMapper jsonMapper; + protected final ObjectMapper yamlMapper; + protected final CustomExtensionHandler customExtensionHandler; + + /** + * Constructs an AbstractConverter with the bundled mapping configuration. + */ + protected AbstractConverter() { + this(loadBundledMapper()); + } + + /** + * Loads the bundled mapping configuration from classpath. + */ + private static PropertyMapper loadBundledMapper() { + return FileBasedPropertyMapper.fromResource(MAPPING_RESOURCE); + } + + /** + * Constructs an AbstractConverter with the specified property mapper. + * + * @param mapper the property mapper to use (required, cannot be null) + */ + protected AbstractConverter(PropertyMapper mapper) { + this.mapper = mapper; + + this.jsonMapper = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); + + YAMLFactory yamlFactory = new YAMLFactory() + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE); + + this.yamlMapper = new ObjectMapper(yamlFactory) + .enable(SerializationFeature.INDENT_OUTPUT); + + this.customExtensionHandler = new CustomExtensionHandler(this.jsonMapper); + } + + @Override + public void convert(Path inputPath, Path outputDir) { + validateInputPath(inputPath); + + if (!Files.isDirectory(outputDir)) { + throw new InvalidInputException("Output path must be a directory: " + outputDir); + } + + try { + String inputContent = Files.readString(inputPath); + List results = convert(inputContent); + writeOutputFiles(outputDir, results); + } catch (IOException e) { + throw new ConversionException("Failed to convert file: " + inputPath, e); + } + } + + /** + * Writes conversion results to files using model names. + * + *

Each file is named after the model's apiName field. + * Example: models with apiName "Sales_Model" and "Marketing_Model" + * generate "Sales_Model.json" and "Marketing_Model.json" + * + * @param outputDir the directory where output files will be written + * @param results the list of conversion results to write + * @throws IOException if writing fails + * @throws ConversionException if model name cannot be extracted + */ + private void writeOutputFiles(Path outputDir, List results) throws IOException { + String extension = getFileExtension(); + + for (String result : results) { + String modelName = extractModelName(result); + Path outputFile = outputDir.resolve(modelName + extension); + Files.writeString(outputFile, result); + logger.info("Generated: {}", outputFile.toAbsolutePath()); + } + } + + /** + * Determines the file extension to use for output files. + * Must be overridden by subclasses to specify their output format. + */ + protected abstract String getFileExtension(); + + /** + * Extracts model name from converted result for use as filename. + * Must be overridden by subclasses based on their output format. + * + * @param result the converted result string + * @return the model name to use for the filename + * @throws ConversionException if model name cannot be extracted + */ + protected abstract String extractModelName(String result); + + /** + * Validates that the input path exists. + * + * @param inputPath the path to validate + */ + protected void validateInputPath(Path inputPath) { + if (!Files.exists(inputPath)) { + throw new InvalidInputException("Input file does not exist: " + inputPath); + } + } + + /** + * Parses JSON content to a Map. + * + * @param content the JSON content + * @return the parsed Map + */ + protected Map parseJson(String content) { + try { + return jsonMapper.readValue(content, new TypeReference>() {}); + } catch (JsonProcessingException e) { + throw new InvalidInputException("Invalid JSON content: " + e.getMessage(), e); + } + } + + /** + * Parses YAML content to a Map. + * + * @param content the YAML content + * @return the parsed Map + */ + protected Map parseYaml(String content) { + try { + return yamlMapper.readValue(content, new TypeReference>() {}); + } catch (JsonProcessingException e) { + throw new InvalidInputException("Invalid YAML content: " + e.getMessage(), e); + } + } + + /** + * Serializes a Map to JSON string. + * + * @param data the data to serialize + * @return the JSON string + */ + protected String toJson(Map data) { + try { + return jsonMapper.writeValueAsString(data); + } catch (JsonProcessingException e) { + throw new ConversionException("Failed to serialize to JSON", e); + } + } + + /** + * Serializes a Map to YAML string. + * + * @param data the data to serialize + * @return the YAML string + */ + protected String toYaml(Map data) { + try { + return yamlMapper.writeValueAsString(data); + } catch (JsonProcessingException e) { + throw new ConversionException("Failed to serialize to YAML", e); + } + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/ConversionDirection.java b/converters/salesforce/src/main/java/org/osi/converter/ConversionDirection.java new file mode 100644 index 00000000..e87a1b5a --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/ConversionDirection.java @@ -0,0 +1,30 @@ +package org.osi.converter; + +/** + * Enum representing the direction of conversion. + * + */ +public enum ConversionDirection { + /** + * Converting from OSI YAML format to Salesforce JSON format. + */ + OSI_TO_SALESFORCE, + + /** + * Converting from Salesforce JSON format to OSI YAML format. + */ + SALESFORCE_TO_OSI; + + /** + * Converts enum name to pipeline configuration key. + * Maps OSI_TO_SALESFORCE -> "osiToSalesforce" and SALESFORCE_TO_OSI -> "salesforceToOsi" + * + * @return The pipeline key used in YAML configuration + */ + public String toPipelineKey() { + return switch (this) { + case OSI_TO_SALESFORCE -> "osiToSalesforce"; + case SALESFORCE_TO_OSI -> "salesforceToOsi"; + }; + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/Converter.java b/converters/salesforce/src/main/java/org/osi/converter/Converter.java new file mode 100644 index 00000000..97d08481 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/Converter.java @@ -0,0 +1,36 @@ +package org.osi.converter; + +import java.nio.file.Path; +import java.util.List; + +/** + * Interface for converting between data formats (YAML and JSON). + * + *

This is the external API for conversion. Implementations handle the conversion + * of data from one format to another, with support for property mapping.

+ * + */ +public interface Converter { + + /** + * Converts the input file and writes results to the specified output directory. + * + *

Each semantic model is written to a separate file named after its apiName. + * Example: "Sales_Model.json", "Marketing_Model.json" + * + * @param inputPath the path to the input file + * @param outputDir the directory where output files will be written + */ + void convert(Path inputPath, Path outputDir); + + /** + * Converts string content from the source format to the target format. + * + *

For OSI to Salesforce: returns one Salesforce model per OSI semantic_model entry. + *

For Salesforce to OSI: returns one OSI document with one semantic_model entry. + * + * @param content the content to convert + * @return list of converted content strings (one per semantic model) + */ + List convert(String content); +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/ConverterConstants.java b/converters/salesforce/src/main/java/org/osi/converter/ConverterConstants.java new file mode 100644 index 00000000..6a2b6c6c --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/ConverterConstants.java @@ -0,0 +1,114 @@ +package org.osi.converter; + +/** + * Constants used across converters for property names and structure keys. + * + */ +public final class ConverterConstants { + + private ConverterConstants() {} + + /** + * Enum representing the level at which custom extensions can be stored or restored. + */ + public enum Level { + SEMANTIC_MODEL, + DATASETS, + RELATIONSHIPS, + METRICS + } + + // OSI root structure + public static final String VERSION = "version"; + public static final String OSI_VERSION = "0.1.1"; + public static final String SEMANTIC_MODEL = "semantic_model"; + + // OSI semantic model structure + public static final String CUSTOM_EXTENSIONS = "custom_extensions"; + public static final String DATASETS = "datasets"; + public static final String FIELDS = "fields"; + public static final String METRICS = "metrics"; + + // Salesforce semantic model structure + public static final String SEMANTIC_DATA_OBJECTS = "semanticDataObjects"; + public static final String SEMANTIC_RELATIONSHIPS = "semanticRelationships"; + public static final String SEMANTIC_CALCULATED_MEASUREMENTS = "semanticCalculatedMeasurements"; + public static final String SEMANTIC_DIMENSIONS = "semanticDimensions"; + public static final String SEMANTIC_MEASUREMENTS = "semanticMeasurements"; + public static final String SEMANTIC_CALCULATED_DIMENSIONS = "semanticCalculatedDimensions"; + + public static final String DEPENDENCIES = "dependencies"; + public static final String DEPENDENT_DEFINITION_API_NAME = "dependentDefinitionApiName"; + + // Common property names + public static final String NAME = "name"; + public static final String API_NAME = "apiName"; + public static final String LABEL = "label"; + public static final String DESCRIPTION = "description"; + public static final String DATA_TYPE = "dataType"; + public static final String AI_CONTEXT = "ai_context"; + public static final String BUSINESS_PREFERENCES = "businessPreferences"; + + // AI Context object properties + public static final String AI_INSTRUCTIONS = "instructions"; + public static final String AI_SYNONYMS = "synonyms"; + public static final String AI_EXAMPLES = "examples"; + + // Field properties + public static final String DIMENSION = "dimension"; + public static final String IS_TIME = "is_time"; + public static final String DATA_OBJECT_FIELD_NAME = "dataObjectFieldName"; + + // Custom extensions structure + public static final String VENDOR_NAME = "vendor_name"; + public static final String DATA = "data"; + public static final String VENDOR_NAME_VALUE = "SALESFORCE"; + + // Expression properties + public static final String EXPRESSION = "expression"; + public static final String DIALECTS = "dialects"; + public static final String DIALECT = "dialect"; + public static final String DIALECT_TABLEAU = "TABLEAU"; + + // Relationship properties + public static final String CRITERIA = "criteria"; + public static final String RELATIONSHIPS = "relationships"; + public static final String FROM = "from"; + public static final String TO = "to"; + public static final String FROM_COLUMNS = "from_columns"; + public static final String TO_COLUMNS = "to_columns"; + public static final String LEFT_SEMANTIC_FIELD_API_NAME = "leftSemanticFieldApiName"; + public static final String RIGHT_SEMANTIC_FIELD_API_NAME = "rightSemanticFieldApiName"; + public static final String LEFT_SEMANTIC_DEFINITION_API_NAME = "leftSemanticDefinitionApiName"; + public static final String RIGHT_SEMANTIC_DEFINITION_API_NAME = "rightSemanticDefinitionApiName"; + + // Relationship criteria field types + public static final String LEFT_FIELD_TYPE = "leftFieldType"; + public static final String RIGHT_FIELD_TYPE = "rightFieldType"; + public static final String FIELD_TYPE_TABLE_FIELD = "TableField"; + public static final String FIELD_TYPE_SEMANTIC_FIELD = "SemanticField"; + public static final String FIELD_TYPE_FORMULA = "Formula"; + + // Data type values + public static final String DATA_TYPE_DATE = "Date"; + public static final String DATA_TYPE_DATE_TIME = "DateTime"; + + // Default values settings + public static final String CARDINALITY = "cardinality"; + public static final String IS_ENABLED = "isEnabled"; + public static final String JOIN_TYPE = "joinType"; + public static final String TABLE_TYPE = "tableType"; + public static final String DISPLAY_CATEGORY = "displayCategory"; + public static final String DISPLAY_CATEGORY_CONTINUOUS = "Continuous"; + public static final String STANDARD_TABLE_TYPE = "Standard"; + public static final String DEFAULT_CARDINALITY = "ManyToMany"; + public static final String DEFAULT_JOIN_TYPE = "Auto"; + + // Format names + public static final String JSON = "json"; + public static final String YAML = "yaml"; + + // File extensions + public static final String JSON_EXTENSION = ".json"; + public static final String YAML_EXTENSION = ".yaml"; +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/ConverterFactory.java b/converters/salesforce/src/main/java/org/osi/converter/ConverterFactory.java new file mode 100644 index 00000000..9e78e9b0 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/ConverterFactory.java @@ -0,0 +1,18 @@ +package org.osi.converter; + +/** + * Factory for creating converters based on conversion direction. + * + */ +public class ConverterFactory { + + /** + * Creates a converter for the specified direction. + * + * @param direction The conversion direction + * @return A Converter instance configured for the specified direction + */ + public static Converter getConverter(ConversionDirection direction) { + return new ConverterImpl(direction); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/ConverterImpl.java b/converters/salesforce/src/main/java/org/osi/converter/ConverterImpl.java new file mode 100644 index 00000000..722dc2b1 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/ConverterImpl.java @@ -0,0 +1,154 @@ +package org.osi.converter; + +import static org.osi.converter.ConverterConstants.*; +import static org.osi.util.DataStructureUtils.*; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.osi.converter.pipeline.*; +import org.osi.converter.pipeline.*; +import org.osi.exception.ConversionException; +import org.osi.validator.SchemaValidator; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Unified converter that executes pipelines configured in osi-salesforce-converter-config.yaml. + * + */ +public class ConverterImpl extends AbstractConverter { + + private final ConversionDirection direction; + private final DirectionConfig directionConfig; + private final List steps; + private final SchemaValidator schemaValidator; + + public ConverterImpl(ConversionDirection direction) { + this(direction, PipelineConfigLoader.loadFromResource()); + } + + ConverterImpl(ConversionDirection direction, PipelineConfig config) { + super(); + this.direction = direction; + + // Get handler list for this direction + List handlerNames = config.getPipelines().get(direction.toPipelineKey()); + if (handlerNames == null || handlerNames.isEmpty()) { + throw new ConversionException("No pipeline defined for direction: " + direction); + } + + // Get direction-specific configuration + this.directionConfig = config.getDirectionConfigs().get(direction.toPipelineKey()); + if (this.directionConfig == null) { + throw new ConversionException("No configuration found for direction: " + direction); + } + + // Initialize schema validator + ObjectMapper schemaMapper = YAML.equals(directionConfig.getInputFormat()) + ? yamlMapper : jsonMapper; + this.schemaValidator = new SchemaValidator( + schemaMapper, + directionConfig.getSchemaPath() + ); + + // Initialize pipeline steps using factory + HandlerFactory factory = new HandlerFactory(customExtensionHandler); + this.steps = handlerNames.stream() + .map(handlerName -> factory.createHandler(handlerName, direction)) + .toList(); + } + + @Override + public List convert(String content) { + Map sourceData = YAML.equals(directionConfig.getInputFormat()) + ? parseYaml(content) + : parseJson(content); + + schemaValidator.validate(sourceData); + + if (direction == ConversionDirection.OSI_TO_SALESFORCE) { + return convertOsiToSalesforce(sourceData); + } else { + return convertSalesforceToOsi(sourceData); + } + } + + private List convertOsiToSalesforce(Map osiRoot) { + List semanticModels = getList(osiRoot, SEMANTIC_MODEL); + List results = new ArrayList<>(); + + for (Object modelObj : semanticModels) { + Map sourceData = asMap(modelObj); + String result = executePipeline(sourceData); + results.add(result); + } + return results; + } + + private List convertSalesforceToOsi(Map sourceData) { + String result = executePipeline(sourceData); + + // Wrap output in OSI root structure + try { + Map outputData = yamlMapper.readValue(result, new TypeReference<>() {}); + Map osiRoot = new LinkedHashMap<>(); + osiRoot.put(VERSION, OSI_VERSION); + osiRoot.put(SEMANTIC_MODEL, List.of(outputData)); + return List.of(toYaml(osiRoot)); + } catch (JsonProcessingException e) { + throw new ConversionException("Failed to wrap output in OSI root", e); + } + } + + private String executePipeline(Map sourceData) { + Map outputData = new LinkedHashMap<>(); + Map mappings = new LinkedHashMap<>(direction == ConversionDirection.OSI_TO_SALESFORCE + ? mapper.getOsiToSalesforceMappings() + : mapper.getSalesforceToOsiMappings()); + + for (PipelineStep step : steps) { + step.execute(sourceData, outputData, mappings); + } + + return serialize(outputData); + } + + private String serialize(Map data) { + return JSON.equals(directionConfig.getOutputFormat()) + ? toJson(data) + : toYaml(data); + } + + @Override + protected String getFileExtension() { + return directionConfig.getFileExtension(); + } + + @Override + protected String extractModelName(String result) { + try { + Map data = JSON.equals(directionConfig.getOutputFormat()) + ? jsonMapper.readValue(result, new TypeReference<>() {}) + : yamlMapper.readValue(result, new TypeReference<>() {}); + + String field = directionConfig.getExtractModelNameFrom(); + + // Handle OSI format (wrapped in semantic_model array) + if (direction == ConversionDirection.SALESFORCE_TO_OSI) { + List models = getList(data, SEMANTIC_MODEL); + if (models != null && !models.isEmpty()) { + Map firstModel = asMap(models.get(0)); + return firstModel.get(field).toString(); + } + } + + return data.get(field).toString(); + } catch (JsonProcessingException e) { + throw new ConversionException("Failed to extract model name", e); + } + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/CustomExtensionHandler.java b/converters/salesforce/src/main/java/org/osi/converter/CustomExtensionHandler.java new file mode 100644 index 00000000..aaf16d73 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/CustomExtensionHandler.java @@ -0,0 +1,331 @@ +package org.osi.converter; + +import static org.osi.converter.ConverterConstants.*; +import static org.osi.util.DataStructureUtils.*; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.osi.converter.ConverterConstants.Level; +import org.osi.util.PathUtils; +import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Handler for adding and restoring custom_extensions in the Semantic Model. + * + *

Handles identifying unmapped properties during Salesforce-to-OSI conversion + * and restoring them during OSI-to-Salesforce conversion.

+ * + */ +public class CustomExtensionHandler { + + private static final Logger logger = LoggerFactory.getLogger(CustomExtensionHandler.class); + + private final ObjectMapper jsonMapper; + + public CustomExtensionHandler(ObjectMapper jsonMapper) { + this.jsonMapper = jsonMapper; + } + + /** + * Unified method to restore custom_extensions at any level. + * + * @param outputData The output data containing semanticModel + * @param sourceData The source OSI data + * @param level The level to restore + */ + public void restoreCustomExtensionsAtLevel( + Map outputData, Map sourceData, Level level) { + + switch (level) { + case SEMANTIC_MODEL: + restoreSalesforceCustomExtension(outputData, sourceData); + break; + + case DATASETS: + restoreArrayLevelExtensions(outputData, sourceData, ConverterConstants.DATASETS, SEMANTIC_DATA_OBJECTS, NAME, API_NAME); + break; + + case RELATIONSHIPS: + restoreArrayLevelExtensions( + outputData, sourceData, ConverterConstants.RELATIONSHIPS, SEMANTIC_RELATIONSHIPS, NAME, API_NAME); + break; + + case METRICS: + restoreArrayLevelExtensions( + outputData, sourceData, ConverterConstants.METRICS, SEMANTIC_CALCULATED_MEASUREMENTS, NAME, API_NAME); + break; + } + } + + /** + * Restores custom_extensions for an array at a specific level. + * Generic method that works for datasets, relationships, etc. + * + * @param semanticModel The semanticModel object + * @param sourceData The source OSI data + * @param sourceArrayKey The key in source data (e.g., "datasets", "relationships") + * @param targetArrayKey The key in semanticModel (e.g., "semanticDataObjects", "semanticRelationships") + * @param sourceIdKey The identifying key in source items (e.g., "name") + * @param targetIdKey The identifying key in target items (e.g., "apiName") + */ + private void restoreArrayLevelExtensions( + Map semanticModel, + Map sourceData, + String sourceArrayKey, + String targetArrayKey, + String sourceIdKey, + String targetIdKey) { + + List sourceArray = getList(sourceData, sourceArrayKey); + if (sourceArray == null) { + return; + } + + List targetArray = getList(semanticModel, targetArrayKey); + if (targetArray == null) { + return; + } + + streamMaps(sourceArray).forEach(sourceItem -> { + // Check if source item has SALESFORCE custom_extensions + if (!hasSalesforceCustomExtension(sourceItem)) { + return; + } + + // Find matching target item by identifier + String itemId = getString(sourceItem, sourceIdKey); + if (itemId == null) return; + + Map targetItem = findItemById(targetArray, targetIdKey, itemId); + if (targetItem == null) return; + + // Restore custom_extensions + restoreSalesforceCustomExtension(targetItem, sourceItem); + }); + } + + /** + * Checks if an item has SALESFORCE vendor custom_extensions. + * + * @param item The item to check + * @return true if the item has SALESFORCE custom_extensions + */ + private boolean hasSalesforceCustomExtension(Map item) { + Object customExtensionsObj = item.get(CUSTOM_EXTENSIONS); + if (customExtensionsObj == null) { + return false; + } + + List customExtensions = asList(customExtensionsObj); + + return streamMaps(customExtensions) + .anyMatch(ext -> VENDOR_NAME_VALUE.equals(ext.get(VENDOR_NAME))); + } + + /** + * Unified method to store unmapped properties as custom_extensions at any level. + * This is the inverse of restoreCustomExtensionsAtLevel. + * + * @param outputData The output OSI data + * @param sourceData The source Salesforce data + * @param allHandledProps All properties that were handled (mapped + programmatic) + * @param level The level to store + */ + public void storeUnmappedProperties( + Map outputData, Map sourceData, Set allHandledProps, Level level) { + + switch (level) { + case SEMANTIC_MODEL: + storeUnmappedItemProperties(outputData, sourceData, allHandledProps); + break; + + case DATASETS: + storeArrayLevelUnmappedProperties( + outputData, sourceData, allHandledProps, ConverterConstants.DATASETS, SEMANTIC_DATA_OBJECTS, NAME, API_NAME); + break; + + case RELATIONSHIPS: + storeArrayLevelUnmappedProperties( + outputData, sourceData, allHandledProps, ConverterConstants.RELATIONSHIPS, SEMANTIC_RELATIONSHIPS, NAME, API_NAME); + break; + + case METRICS: + storeArrayLevelUnmappedProperties( + outputData, + sourceData, + allHandledProps, + ConverterConstants.METRICS, + SEMANTIC_CALCULATED_MEASUREMENTS, + NAME, + API_NAME); + break; + } + } + + /** + * Stores unmapped properties for array items (datasets, relationships, metrics). + * + * @param osiData The OSI output data + * @param sfData The Salesforce source data + * @param allHandledProps All properties that were handled (from mappings + programmatic) + * @param osiArrayKey The OSI array key (e.g., "datasets") + * @param sfArrayKey The SF array key (e.g., "semanticDataObjects") + * @param osiIdKey The OSI identifier key (e.g., "name") + * @param sfIdKey The SF identifier key (e.g., "apiName") + */ + private void storeArrayLevelUnmappedProperties( + Map osiData, + Map sfData, + Set allHandledProps, + String osiArrayKey, + String sfArrayKey, + String osiIdKey, + String sfIdKey) { + + List osiArray = getList(osiData, osiArrayKey); + if (osiArray == null) { + return; + } + + List sfArray = getList(sfData, sfArrayKey); + if (sfArray == null) { + return; + } + + // Process each OSI item and find its matching SF item + streamMaps(osiArray).forEach(osiItem -> { + // Find matching SF item by identifier + String itemId = getString(osiItem, osiIdKey); + if (itemId == null) return; + + Map sfItem = findItemById(sfArray, sfIdKey, itemId); + if (sfItem == null) return; + + // Find unmapped properties: SF properties NOT handled + Map unmappedProperties = new LinkedHashMap<>(); + for (Map.Entry entry : sfItem.entrySet()) { + // If property is not handled → it's unmapped + if (!allHandledProps.contains(entry.getKey())) { + unmappedProperties.put(entry.getKey(), PathUtils.deepCopyValue(entry.getValue())); + } + } + + // Store unmapped properties in custom_extensions + if (!unmappedProperties.isEmpty()) { + addCustomExtension(osiItem, unmappedProperties); + } + }); + } + + /** + * Stores unmapped properties from a source item into an OSI item's custom_extensions. + * Generic method that can be used for fields or any other individual items. + * + * @param osiItem The target OSI item to add custom_extensions to + * @param sfItem The source Salesforce item + * @param handledProps Set of property keys that were handled (mapped or programmatically processed) + */ + public void storeUnmappedItemProperties( + Map osiItem, Map sfItem, Set handledProps) { + // Find unmapped properties: everything in SF item that wasn't handled + Map unmappedProps = new LinkedHashMap<>(); + for (Map.Entry entry : sfItem.entrySet()) { + if (!handledProps.contains(entry.getKey())) { + unmappedProps.put(entry.getKey(), PathUtils.deepCopyValue(entry.getValue())); + } + } + + // Store in custom_extensions + if (!unmappedProps.isEmpty()) { + String itemName = getString(osiItem, NAME); + if (itemName == null) { + logger.warn("Item has no name, skipping custom_extensions storage for unmapped properties"); + return; + } + addCustomExtension(osiItem, unmappedProps); + } + } + + /** + * Adds a custom_extensions entry with SALESFORCE vendor data. + * If a SALESFORCE custom_extension already exists, merges the data. + */ + public void addCustomExtension(Map target, Map customData) { + try { + List customExtensions = + asList(target.computeIfAbsent(CUSTOM_EXTENSIONS, k -> new ArrayList<>())); + + if (!customExtensions.isEmpty()) { + // Merge into existing extension + Map salesforceExtension = asMap(customExtensions.get(0)); + String existingDataJson = (String) salesforceExtension.get(DATA); + Map existingData = jsonMapper.readValue( + existingDataJson, new TypeReference>() {}); + + existingData.putAll(customData); + salesforceExtension.put(DATA, jsonMapper.writeValueAsString(existingData)); + } else { + // Create new extension + Map extension = new LinkedHashMap<>(); + extension.put(VENDOR_NAME, VENDOR_NAME_VALUE); + extension.put(DATA, jsonMapper.writeValueAsString(customData)); + customExtensions.add(extension); + } + + } catch (Exception e) { + logger.warn("Failed to store custom_extensions: {}", e.getMessage()); + } + } + + /** + * Extracts SALESFORCE vendor custom_extension from OSI item and merges into SF item. + * This is a generic method that can be used for any level (dataset, field, relationship, metric). + * + * @param sfItem The Salesforce item to merge properties into + * @param osiItem The OSI item containing custom_extensions + */ + public void restoreSalesforceCustomExtension(Map sfItem, Map osiItem) { + Object customExtensionsObj = osiItem.get(CUSTOM_EXTENSIONS); + if (customExtensionsObj == null) { + return; + } + + List customExtensions = asList(customExtensionsObj); + + streamMaps(customExtensions).forEach(ext -> { + if (!VENDOR_NAME_VALUE.equals(ext.get(VENDOR_NAME))) { + return; + } + + Object dataObj = ext.get(DATA); + if (dataObj == null) { + return; + } + + try { + // Parse JSON string to Map + Map salesforceProperties = jsonMapper.readValue( + (String) dataObj, + new TypeReference>() {} + ); + + String itemName = getString(osiItem, NAME); + if (itemName == null) { + logger.warn("Item has no name, skipping custom_extensions restoration"); + return; + } + + for (Map.Entry entry : salesforceProperties.entrySet()) { + if (!sfItem.containsKey(entry.getKey())) { + sfItem.put(entry.getKey(), PathUtils.deepCopyValue(entry.getValue())); + } + } + + } catch (Exception e) { + logger.warn("Failed to restore custom_extensions: {}", e.getMessage()); + } + }); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/DatasetMappingHandler.java b/converters/salesforce/src/main/java/org/osi/converter/DatasetMappingHandler.java new file mode 100644 index 00000000..94517dac --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/DatasetMappingHandler.java @@ -0,0 +1,143 @@ +package org.osi.converter; + +import static org.osi.converter.ConverterConstants.*; +import static org.osi.util.DataStructureUtils.*; + +import org.osi.converter.ConverterConstants.Level; +import org.osi.converter.pipeline.PipelineStep; +import java.util.*; +import java.util.stream.Collectors; + +import org.osi.util.MappingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Bidirectional handler for mapping datasets between OSI and Salesforce formats. + * + *

Supports both conversion directions: + *

    + *
  • OSI → Salesforce: datasets → semanticDataObjects, apply SF defaults, restore custom_extensions
    • + *
  • Salesforce → OSI: semanticDataObjects → datasets, store unmapped properties in custom_extensions
    • + *
+ * + */ +public class DatasetMappingHandler implements PipelineStep { + + private static final Logger logger = LoggerFactory.getLogger(DatasetMappingHandler.class); + + private final ConversionDirection direction; + private final CustomExtensionHandler customExtensionHandler; + + public DatasetMappingHandler(ConversionDirection direction, CustomExtensionHandler customExtensionHandler) { + this.direction = direction; + this.customExtensionHandler = customExtensionHandler; + } + + @Override + public void execute(Map sourceData, Map outputData, Map mappings) { + logger.debug("Mapping datasets in {} direction", direction); + if (direction == ConversionDirection.OSI_TO_SALESFORCE) { + mapOsiToSalesforce(sourceData, outputData, mappings); + } else { + mapSalesforceToOsi(sourceData, outputData, mappings); + } + } + + /** + * Maps OSI datasets to Salesforce semanticDataObjects. + */ + private void mapOsiToSalesforce( + Map sourceData, Map outputData, Map mappings) { + + Map datasetMappings = MappingUtils.filterMappingsByPrefix(mappings, DATASETS); + + var mappedData = GenericMappingEngine.applyMappings(sourceData, datasetMappings); + datasetMappings.keySet().forEach(mappings::remove); + + outputData.putAll(mappedData); + + customExtensionHandler.restoreCustomExtensionsAtLevel(outputData, sourceData, Level.DATASETS); + + List sfDataObjects = getList(outputData, SEMANTIC_DATA_OBJECTS); + applyDefaults(sfDataObjects); + } + + /** + * Maps Salesforce semanticDataObjects to OSI datasets. + */ + private void mapSalesforceToOsi( + Map sourceData, Map outputData, Map mappings) { + + List sfDataObjects = getList(sourceData, SEMANTIC_DATA_OBJECTS); + if (sfDataObjects == null) { + return; + } + + Map> partitioned = sfDataObjects.stream() + .collect(Collectors.partitioningBy(obj -> { + Map dataObject = asMap(obj); + String tableType = getString(dataObject, TABLE_TYPE); + return tableType == null || STANDARD_TABLE_TYPE.equals(tableType); + })); + + List standardEntities = partitioned.get(true); + List sharedEntities = partitioned.get(false); + + // Create filtered source data with only standard semantic data objects + Map filteredSourceData = new LinkedHashMap<>(sourceData); + filteredSourceData.put(SEMANTIC_DATA_OBJECTS, standardEntities); + + Map datasetMappings = MappingUtils.filterMappingsByPrefix(mappings, SEMANTIC_DATA_OBJECTS); + + Set allHandledProps = datasetMappings.isEmpty()? new HashSet<>() : MappingUtils.extractHandledProperties(datasetMappings); + + // Add child array keys as handled since FieldMappingHandler processes them + allHandledProps.add(SEMANTIC_DIMENSIONS); + allHandledProps.add(SEMANTIC_MEASUREMENTS); + + var mappedData = GenericMappingEngine.applyMappings(filteredSourceData, datasetMappings); + // Remove consumed mappings from original map + datasetMappings.keySet().forEach(mappings::remove); + + outputData.putAll(mappedData); + + // Store unmapped SF properties in custom_extensions + customExtensionHandler.storeUnmappedProperties(outputData, filteredSourceData, allHandledProps, Level.DATASETS); + + if (!sharedEntities.isEmpty()) { + storeSharedEntitiesInCustomExtensions(outputData, sharedEntities); + } + } + + /** + * Applies default values for required Salesforce data object properties. + * Used when converting OSI → Salesforce. + */ + private void applyDefaults(List dataObjects) { + for (Object dataObjectObj : dataObjects) { + Map dataObject = asMap(dataObjectObj); + + if (!dataObject.containsKey(LABEL) && dataObject.containsKey(API_NAME)) { + String apiName = getString(dataObject, API_NAME); + dataObject.put(LABEL, apiName); + } + dataObject.putIfAbsent(TABLE_TYPE, STANDARD_TABLE_TYPE); + } + } + + /** + * Stores shared entities in semanticDataObjects in top-level custom_extensions. + * + * @param outputData The output OSI data + * @param sharedEntities List of non-standard semanticDataObjects + */ + private void storeSharedEntitiesInCustomExtensions( + Map outputData, List sharedEntities) { + + Map customData = new LinkedHashMap<>(); + customData.put(SEMANTIC_DATA_OBJECTS, sharedEntities); + + customExtensionHandler.addCustomExtension(outputData, customData); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/FieldMappingHandler.java b/converters/salesforce/src/main/java/org/osi/converter/FieldMappingHandler.java new file mode 100644 index 00000000..01ccfc96 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/FieldMappingHandler.java @@ -0,0 +1,691 @@ +package org.osi.converter; + +import org.osi.converter.pipeline.PipelineStep; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.regex.Pattern; + +import static org.osi.converter.ConverterConstants.*; +import static org.osi.util.DataStructureUtils.*; + +/** + * Bidirectional handler for mapping fields between OSI and Salesforce formats. + * + *

OSI → SF: Maps dataset fields to SemanticDimensions and SemanticMeasurements + *

SF → OSI: Maps SemanticDimensions and SemanticMeasurements to dataset fields + * + */ +public class FieldMappingHandler implements PipelineStep { + + private static final Logger logger = LoggerFactory.getLogger(FieldMappingHandler.class); + + // Properties handled when converting SF dimensions/measurements to OSI fields + private static final Set SF_FIELD_HANDLED_PROPS = + Set.of(API_NAME, LABEL, DESCRIPTION, DATA_OBJECT_FIELD_NAME); + + // Compiled regex pattern for SQL keywords that indicate calculated expressions + private static final Pattern CALCULATED_KEYWORDS_PATTERN = Pattern.compile( + "\\b(CASE|WHEN|THEN|ELSE|END|CAST|CONVERT|EXTRACT|SUBSTRING|SUBSTR|" + + "COALESCE|NULLIF|IFNULL|CONCAT|UPPER|LOWER|TRIM|LENGTH|" + + "AND|OR|NOT|IN|BETWEEN|LIKE|IS\\s+NULL|IS\\s+NOT\\s+NULL|DISTINCT|" + + "COUNT|SUM|AVG|MIN|MAX|DATE|YEAR|MONTH|DAY)\\b" + ); + + private final ConversionDirection direction; + private final CustomExtensionHandler customExtensionHandler; + + /** + * Enum representing the four possible field types in Salesforce Semantic Model. + */ + private enum FieldType { + DIMENSION, // Direct dimension: !isCalculated + hasDimension + MEASUREMENT, // Direct measurement: !isCalculated + !hasDimension + CALCULATED_DIMENSION, // Calculated dimension: isCalculated + hasDimension + CALCULATED_MEASUREMENT // Calculated measurement: isCalculated + !hasDimension + } + + public FieldMappingHandler(ConversionDirection direction, CustomExtensionHandler customExtensionHandler) { + this.direction = direction; + this.customExtensionHandler = customExtensionHandler; + } + + /** + * Executes field mapping based on conversion direction. + */ + @Override + public void execute(Map sourceData, Map outputData, Map mappings) { + logger.debug("Mapping fields in {} direction", direction); + if (direction == ConversionDirection.OSI_TO_SALESFORCE) { + mapOsiToSalesforce(sourceData, outputData); + } else { + mapSalesforceToOsi(sourceData, outputData); + } + } + + /** + * Maps OSI dataset fields to Salesforce SemanticDimensions and SemanticMeasurements. + * + * @param outputData The output map containing semanticModel + * @param sourceData The source OSI data + */ + private void mapOsiToSalesforce( + Map sourceData, Map outputData) { + + List osiDatasets = getList(sourceData, DATASETS); + + List sfDataObjects = getList(outputData, SEMANTIC_DATA_OBJECTS); + + for (Object osiDatasetObj : osiDatasets) { + Map osiDataset = asMap(osiDatasetObj); + + String datasetName = getString(osiDataset, NAME); + if (datasetName == null) continue; + + // Find matching SemanticDataObject + Map sfDataObject = findItemById(sfDataObjects, API_NAME, datasetName); + if (sfDataObject == null) continue; + + processFieldsForDataset(osiDataset, sfDataObject, outputData); + } + } + + /** + * Maps Salesforce SemanticDimensions and SemanticMeasurements to OSI dataset fields. + */ + private void mapSalesforceToOsi( + Map sourceData, Map outputData) { + + List sfDataObjects = getList(sourceData, SEMANTIC_DATA_OBJECTS); + if (sfDataObjects == null) { + return; + } + + List osiDatasets = getList(outputData, DATASETS); + + for (Object sfDataObjectObj : sfDataObjects) { + Map sfDataObject = asMap(sfDataObjectObj); + + String apiName = getString(sfDataObject, API_NAME); + if (apiName == null) continue; + + // Find matching OSI dataset by name (mapped from apiName) + Map osiDataset = findItemById(osiDatasets, NAME, apiName); + if (osiDataset == null) continue; + + // Convert SF dimensions and measurements to OSI fields + convertSalesforceFieldsToOsi(sfDataObject, osiDataset); + } + + // Process model-level calculated dimensions + processModelLevelCalculatedDimensions(sourceData, outputData); + + // Cleanup: remove processed structural key + sourceData.remove(SEMANTIC_DATA_OBJECTS); + } + + /** + * Converts Salesforce dimensions and measurements to OSI fields for a dataset. + */ + private void convertSalesforceFieldsToOsi( + Map sfDataObject, Map osiDataset) { + List osiFields = getOrCreateList(osiDataset, FIELDS); + + // Process semanticDimensions → OSI fields + List sfDimensions = getList(sfDataObject, SEMANTIC_DIMENSIONS); + if (sfDimensions != null) { + for (Object sfDimObj : sfDimensions) { + Map sfDim = asMap(sfDimObj); + Map osiField = convertDimensionToOsiField(sfDim); + osiFields.add(osiField); + } + } + + // Process semanticMeasurements → OSI fields + List sfMeasurements = getList(sfDataObject, SEMANTIC_MEASUREMENTS); + if (sfMeasurements != null) { + for (Object sfMeasObj : sfMeasurements) { + Map sfMeas = asMap(sfMeasObj); + Map osiField = convertMeasurementToOsiField(sfMeas); + osiFields.add(osiField); + } + } + } + + /** + * Converts a Salesforce dimension to an OSI field with dimension property. + */ + private Map convertDimensionToOsiField(Map sfDimension) { + Map osiField = new LinkedHashMap<>(); + + mapCommonFieldProperties(sfDimension, osiField); + + // Add dimension property with is_time based on dataType + Map dimensionProp = new LinkedHashMap<>(); + String dataType = getString(sfDimension, DATA_TYPE); + if (DATA_TYPE_DATE.equals(dataType) || DATA_TYPE_DATE_TIME.equals(dataType)) { + dimensionProp.put(IS_TIME, true); + } else { + dimensionProp.put(IS_TIME, false); + } + osiField.put(DIMENSION, dimensionProp); + + // Wrap dataObjectFieldName in expression structure + String dataObjectFieldName = getString(sfDimension, DATA_OBJECT_FIELD_NAME); + if (dataObjectFieldName != null) { + osiField.put(EXPRESSION, wrapExpression(dataObjectFieldName)); + } + + // Store unmapped properties in custom_extensions + customExtensionHandler.storeUnmappedItemProperties(osiField, sfDimension, SF_FIELD_HANDLED_PROPS); + + return osiField; + } + + /** + * Converts a Salesforce measurement to an OSI field without dimension property. + */ + private Map convertMeasurementToOsiField(Map sfMeasurement) { + Map osiField = new LinkedHashMap<>(); + + mapCommonFieldProperties(sfMeasurement, osiField); + + String dataObjectFieldName = getString(sfMeasurement, DATA_OBJECT_FIELD_NAME); + if (dataObjectFieldName != null) { + osiField.put(EXPRESSION, wrapExpression(dataObjectFieldName)); + } + + // Store unmapped properties in custom_extensions + customExtensionHandler.storeUnmappedItemProperties(osiField, sfMeasurement, SF_FIELD_HANDLED_PROPS); + + return osiField; + } + + /** + * Maps common field properties from Salesforce to OSI format. + * Common properties: name (from apiName), label, description. + */ + private void mapCommonFieldProperties(Map sfField, Map osiField) { + String apiName = getString(sfField, API_NAME); + osiField.put(NAME, apiName); + + String label = getString(sfField, LABEL); + if (label != null) { + osiField.put(LABEL, label); + } + + String description = getString(sfField, DESCRIPTION); + if (description != null) { + osiField.put(DESCRIPTION, description); + } + } + + /** + * Wraps a simple expression string in OSI's expression.dialects structure. + * Tags expressions with TABLEAU dialect as they come from Salesforce (Tableau CRM). + */ + private Map wrapExpression(String expressionValue) { + Map dialect = new LinkedHashMap<>(); + dialect.put(DIALECT, DIALECT_TABLEAU); + dialect.put(EXPRESSION, expressionValue); + + List dialects = new ArrayList<>(); + dialects.add(dialect); + + Map expression = new LinkedHashMap<>(); + expression.put(DIALECTS, dialects); + + return expression; + } + + /** + * Processes all fields for a single dataset. + * + *

Routing Logic: + * + * + * + * + * + *
Expression TypeHas dimension?Routes To
DirectYesdataObject.semanticDimensions
DirectNodataObject.semanticMeasurements
CalculatedN/AMODEL.semanticCalculatedDimensions
+ * + * @param osiDataset The OSI dataset + * @param sfDataObject The Salesforce data object to add direct fields to + * @param outputData The Salesforce model for adding calculated dimensions + */ + private void processFieldsForDataset( + Map osiDataset, Map sfDataObject, Map outputData) { + List osiFields = getList(osiDataset, FIELDS); + if (osiFields == null) { + return; + } + + List sfDimensions = getList(sfDataObject, SEMANTIC_DIMENSIONS); + List sfMeasurements = getList(sfDataObject, SEMANTIC_MEASUREMENTS); + + for (Object osiFieldObj : osiFields) { + Map osiField = asMap(osiFieldObj); + + // Determine field type based on OSI structure + boolean hasDimension = osiField.containsKey(DIMENSION); + ExpressionInfo expressionInfo = unwrapExpression(osiField); + + String expression = expressionInfo.expression(); + String dialect = expressionInfo.dialect(); + + // Skip calculated fields for non-Tableau dialects till we agree on a common dialect. + if (!DIALECT_TABLEAU.equals(dialect) && isCalculatedExpression(expression)) { + continue; + } + + // Check if this is a calculated field (Tableau dialect with calculated expression) + boolean isCalculated = DIALECT_TABLEAU.equals(dialect) && isCalculatedExpression(expression); + + if (isCalculated) { + // Create a semantic calculated dimension + Map calcDim = createSemanticCalculatedDimension(osiField, expression); + + customExtensionHandler.restoreSalesforceCustomExtension(calcDim, osiField); + + applyFieldDefaults(calcDim); + + // Add to semantic calculated dimensions array + List calcDimensions = getOrCreateList(outputData, SEMANTIC_CALCULATED_DIMENSIONS); + calcDimensions.add(calcDim); + } else { + // Non calculated field - add to data object + FieldType fieldType = hasDimension? FieldType.DIMENSION : FieldType.MEASUREMENT; + + Map sfField = mapFieldProperties(osiField, expression); + + customExtensionHandler.restoreSalesforceCustomExtension(sfField, osiField); + + applyFieldDefaults(sfField); + + RoutingResult result = + routeFieldToArray(sfField, fieldType, sfDataObject, sfDimensions, sfMeasurements); + sfDimensions = result.dataObjectDimensions(); + sfMeasurements = result.dataObjectMeasurements(); + } + } + } + + + /** + * Maps field properties based on whether the field is calculated. + * Includes common properties plus type-specific properties. + * + * @param osiField The OSI field + * @param expression The extracted expression string + * @return A map with Salesforce field properties + */ + private Map mapFieldProperties( + Map osiField, String expression) { + + Map sfField = new LinkedHashMap<>(); + + String name = getString(osiField, NAME); + sfField.put(API_NAME, name); + + String description = getString(osiField, DESCRIPTION); + if (description != null) { + sfField.put(DESCRIPTION, description); + } + + String label = getString(osiField, LABEL); + if (label != null) { + sfField.put(LABEL, label); + } + + sfField.put(DATA_OBJECT_FIELD_NAME, expression); + return sfField; + } + + /** + * Creates a Salesforce semanticCalculatedDimension from an OSI field with a calculated expression. + * Per schema: required properties are apiName and expression. + * + * @param osiField The OSI field + * @param expression The calculated expression + * @return A map representing a semanticCalculatedDimension + */ + private Map createSemanticCalculatedDimension( + Map osiField, String expression) { + + Map calcDim = new LinkedHashMap<>(); + + // Required properties + String name = getString(osiField, NAME); + calcDim.put(API_NAME, name); + calcDim.put(EXPRESSION, expression); + + // Optional properties + String description = getString(osiField, DESCRIPTION); + if (description != null) { + calcDim.put(DESCRIPTION, description); + } + + String label = getString(osiField, LABEL); + if (label != null) { + calcDim.put(LABEL, label); + } + + // Set syntax for Tableau expressions + calcDim.put("syntax", DIALECT_TABLEAU); + + return calcDim; + } + + /** + * Routes a field to the appropriate array based on its type. + * Initializes arrays lazily using computeIfAbsent. + * + * @param sfField The Salesforce field to route + * @param fieldType The field type + * @param sfDataObject The data object (for data object-level arrays) + * @return Updated arrays for all levels + */ + private RoutingResult routeFieldToArray( + Map sfField, + FieldType fieldType, + Map sfDataObject, + List currentDataObjectDimensions, + List currentDataObjectMeasurements) { + + List dataObjectDimensions = currentDataObjectDimensions; + List dataObjectMeasurements = currentDataObjectMeasurements; + + switch (fieldType) { + case DIMENSION: + dataObjectDimensions = getOrCreateList(sfDataObject, SEMANTIC_DIMENSIONS); + dataObjectDimensions.add(sfField); + break; + + case MEASUREMENT: + dataObjectMeasurements = getOrCreateList(sfDataObject, SEMANTIC_MEASUREMENTS); + dataObjectMeasurements.add(sfField); + break; + } + + return new RoutingResult(dataObjectDimensions, dataObjectMeasurements); + } + + /** + * Helper record to return updated data object arrays. + */ + private record RoutingResult(List dataObjectDimensions, List dataObjectMeasurements) {} + + /** + * Helper record to return expression value along with its dialect type. + */ + private record ExpressionInfo(String expression, String dialect) {} + + /** + * Extracts the expression value and dialect from OSI field's expression.dialects[0].expression. + * This unwraps the nested structure to get the simple column reference and its dialect. + * + * @param osiField The OSI field containing expression structure + * @return ExpressionInfo containing the expression string and dialect type, or null if not found + */ + private ExpressionInfo unwrapExpression(Map osiField) { + Object expressionObj = osiField.get(EXPRESSION); + + Map expression = asMap(expressionObj); + Object dialectsObj = expression.get(DIALECTS); + + List dialects = asList(dialectsObj); + + Object selectedDialectObj = null; + for (Object dialectObj : dialects) { + Map dialect = asMap(dialectObj); + String dialectType = getString(dialect, DIALECT); + if (DIALECT_TABLEAU.equals(dialectType)) { + selectedDialectObj = dialectObj; + break; + } + } + + if (selectedDialectObj == null) { + selectedDialectObj = dialects.get(0); + } + + Map selectedDialect = asMap(selectedDialectObj); + Object expressionValue = selectedDialect.get(EXPRESSION); + String dialectType = getString(selectedDialect, DIALECT); + + return new ExpressionInfo((String) expressionValue, dialectType); + } + + /** + * Determines if an expression is calculated or a direct column reference. + * + *

A calculated expression contains: + *

    + *
  • SQL functions: CONCAT(), SUM(), CAST(), etc.
    • + *
  • Operators: +, -, *, /, %, ||
    • + *
  • SQL keywords: CASE, WHEN, AND, OR, etc.
    • + *
  • Comparisons: {@literal >, <, =, !=, <>}
    • + *
+ * + *

A direct reference is a simple column name (possibly table-qualified): + *

    + *
  • customer_name
    • + *
  • customers.customer_name
    • + *
  • schema.table.column
    • + *
+ * + * @param expression The SQL expression to evaluate + * @return true if calculated, false if direct reference + */ + private boolean isCalculatedExpression(String expression) { + if (expression == null || expression.isEmpty()) { + return false; + } + + String normalized = expression.trim().toUpperCase(); + + // Check for function calls (presence of parentheses) + if (normalized.contains("(") || normalized.contains("[")) { + return true; + } + + // Check for operators (arithmetic, comparison, string concatenation) + if (normalized.contains("*") || normalized.contains("/") || normalized.contains("%") || + normalized.contains("||") || normalized.contains("::") || + normalized.contains(">") || normalized.contains("<") || + normalized.contains("!=") || normalized.contains("<>")) { + return true; + } + + // Check for arithmetic/comparison operators with spaces (avoid false positives like "customer-id") + if (normalized.contains(" + ") || normalized.contains(" - ") || + normalized.contains(" * ") || normalized.contains(" / ") || + normalized.contains(" = ")) { + return true; + } + + // Check for SQL keywords using compiled pattern + return CALCULATED_KEYWORDS_PATTERN.matcher(normalized).find(); + } + + /** + * Applies default values for required Salesforce field properties. + * Only sets defaults if the property is not already present. + * Defaults are applied AFTER custom extensions and mappings. + * + * @param sfField The Salesforce field to apply defaults to + */ + private void applyFieldDefaults(Map sfField) { + sfField.putIfAbsent(DISPLAY_CATEGORY, DISPLAY_CATEGORY_CONTINUOUS); + } + + /** + * Processes model-level semanticCalculatedDimensions and converts them to dataset fields + * if all their dependencies point to the same data object. + * + *

Logic: + *

    + *
  • If all dependencies have the same dependentDefinitionApiName: + * convert to field and add to that dataset
    • + *
  • Otherwise: leave in sourceData for custom extension handling
    • + *
+ * + * @param sourceData The source Salesforce data (will be modified to remove converted dimensions) + * @param outputData The output OSI data containing datasets + */ + private void processModelLevelCalculatedDimensions( + Map sourceData, Map outputData) { + + List calcDims = getList(sourceData, SEMANTIC_CALCULATED_DIMENSIONS); + if (calcDims == null) { + return; + } + + logger.debug("Processing {} semanticCalculatedDimensions", calcDims.size()); + + List osiDatasets = getList(outputData, DATASETS); + List remainingCalcDims = new ArrayList<>(); + + for (Object calcDimObj : calcDims) { + Map calcDim = asMap(calcDimObj); + + List dependencies = getList(calcDim, DEPENDENCIES); + String targetDataObject = getSingleDataObjectFromDependencies(dependencies); + + if (targetDataObject != null) { + Map osiDataset = findItemById(osiDatasets, NAME, targetDataObject); + if (osiDataset != null) { + Map osiField = convertCalculatedDimensionToField(calcDim); + List fields = getOrCreateList(osiDataset, FIELDS); + fields.add(osiField); + + // Update relationships for this converted field + String calcFieldName = getString(calcDim, API_NAME); + updateRelationshipsForConvertedField(sourceData, calcFieldName); + + logger.debug("Converted calculated dimension '{}' to field in dataset '{}'", + getString(calcDim, API_NAME), targetDataObject); + continue; + } + } + remainingCalcDims.add(calcDim); + } + + // Update sourceData with only the remaining calculated dimensions + // The ones that couldn't be converted to dataset fields + if (remainingCalcDims.isEmpty()) { + sourceData.remove(SEMANTIC_CALCULATED_DIMENSIONS); + logger.debug("All calculated dimensions converted to dataset fields"); + } else { + sourceData.put(SEMANTIC_CALCULATED_DIMENSIONS, remainingCalcDims); + logger.debug("{} calculated dimensions kept for custom extension handling", remainingCalcDims.size()); + } + } + + /** + * Checks if all dependencies point to the same data object. + * + * @param dependencies List of dependency objects + * @return The common data object API name if all dependencies reference the same object, + * null if dependencies are empty, mixed, or missing dependentDefinitionApiName + */ + private String getSingleDataObjectFromDependencies(List dependencies) { + if (dependencies == null || dependencies.isEmpty()) { + return null; + } + + String commonDataObject = null; + for (Object depObj : dependencies) { + Map dep = asMap(depObj); + String defApiName = getString(dep, DEPENDENT_DEFINITION_API_NAME); + + if (defApiName == null) { + continue; + } + + if (commonDataObject == null) { + commonDataObject = defApiName; + } else if (!commonDataObject.equals(defApiName)) { + return null; + } + } + + return commonDataObject; + } + + /** + * Converts a Salesforce semanticCalculatedDimension to an OSI field with dimension property. + * Similar to convertDimensionToOsiField but handles expression (not dataObjectFieldName). + * + * @param calcDim The Salesforce calculated dimension + * @return An OSI field map with dimension property and wrapped expression + */ + private Map convertCalculatedDimensionToField(Map calcDim) { + Map osiField = new LinkedHashMap<>(); + + // Common properties: name, label, description + mapCommonFieldProperties(calcDim, osiField); + + // Add dimension property with is_time based on dataType + Map dimensionProp = new LinkedHashMap<>(); + String dataType = getString(calcDim, DATA_TYPE); + if (DATA_TYPE_DATE.equals(dataType) || DATA_TYPE_DATE_TIME.equals(dataType)) { + dimensionProp.put(IS_TIME, true); + } else { + dimensionProp.put(IS_TIME, false); + } + osiField.put(DIMENSION, dimensionProp); + + // Get expression (calculated dimensions have expression, not dataObjectFieldName) + String expressionValue = getString(calcDim, EXPRESSION); + if (expressionValue != null) { + osiField.put(EXPRESSION, wrapExpression(expressionValue)); + } + + Set handledProps = Set.of(API_NAME, LABEL, DESCRIPTION, EXPRESSION, DEPENDENCIES); + customExtensionHandler.storeUnmappedItemProperties(osiField, calcDim, handledProps); + return osiField; + } + + /** + * Updates relationships that reference a converted calculated field. + * Changes fieldType from "SemanticField" to "TableField" for the specific field. + * + * @param sourceData The source Salesforce data containing relationships + * @param calcFieldName The API name of the calculated field that was converted + */ + private void updateRelationshipsForConvertedField(Map sourceData, String calcFieldName) { + List relationships = getList(sourceData, SEMANTIC_RELATIONSHIPS); + if (relationships == null) { + return; + } + + for (Object relObj : relationships) { + Map rel = asMap(relObj); + List criteria = getList(rel, CRITERIA); + if (criteria == null) { + continue; + } + + for (Object critObj : criteria) { + Map criterion = asMap(critObj); + + if (FIELD_TYPE_SEMANTIC_FIELD.equals(getString(criterion, LEFT_FIELD_TYPE))) { + if (calcFieldName.equals(getString(criterion, LEFT_SEMANTIC_FIELD_API_NAME))) { + criterion.put(LEFT_FIELD_TYPE, FIELD_TYPE_TABLE_FIELD); + logger.debug("Updated left field '{}' type from SemanticField to TableField", calcFieldName); + } + } + + if (FIELD_TYPE_SEMANTIC_FIELD.equals(getString(criterion, RIGHT_FIELD_TYPE))) { + if (calcFieldName.equals(getString(criterion, RIGHT_SEMANTIC_FIELD_API_NAME))) { + criterion.put(RIGHT_FIELD_TYPE, FIELD_TYPE_TABLE_FIELD); + logger.debug("Updated right field '{}' type from SemanticField to TableField", calcFieldName); + } + } + } + } + } + +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/GenericMappingEngine.java b/converters/salesforce/src/main/java/org/osi/converter/GenericMappingEngine.java new file mode 100644 index 00000000..86abc7df --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/GenericMappingEngine.java @@ -0,0 +1,128 @@ +package org.osi.converter; + +import static org.osi.util.DataStructureUtils.*; + +import org.osi.util.MappingUtils; +import org.osi.util.PathUtils; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Simplified mapping engine that applies straightforward property mappings from mappings.yaml. + * + *

Supports: + * - Simple property mappings (apiName -> name) + * - Array mappings (semanticDataObjects -> datasets) + * - Nested property mappings (semanticDataObjects.apiName -> datasets.name) + *

+ * + *

Complex transformations (fields, relationships, metrics) are handled by dedicated handlers. + * + */ +public class GenericMappingEngine { + + private static final Logger logger = LoggerFactory.getLogger(GenericMappingEngine.class); + + /** + * Applies mappings from source to target format. + * Assumes mappings are pre-filtered to the same top-level key. + */ + public static Map applyMappings(Map sourceData, Map mappings) { + Map outputData = new LinkedHashMap<>(); + + if (mappings.isEmpty()) { + return outputData; + } + + // All mappings are pre-filtered to the same top-level key + String topLevelKey = MappingUtils.getFirstPathSegment(mappings.keySet().iterator().next()); + + Object sourceValue = sourceData.get(topLevelKey); + + // Check if this is an array mapping + boolean isArray = sourceValue instanceof List; + + if (isArray) { + logger.debug("Processing array mapping for key: {}", topLevelKey); + processArrayMapping(sourceData, outputData, topLevelKey, mappings); + } else { + logger.debug("Processing simple mapping for key: {}", topLevelKey); + processSimpleMapping(sourceData, outputData, topLevelKey, mappings); + } + + return outputData; + } + + /** + * Processes simple (non-array) property mappings. + */ + private static void processSimpleMapping( + Map sourceData, + Map outputData, + String topLevelKey, + Map mappings) { + + for (Map.Entry entry : mappings.entrySet()) { + String sourcePath = entry.getKey(); + String targetPath = entry.getValue(); + + Object value = sourcePath.equals(topLevelKey) + ? sourceData.get(topLevelKey) + : PathUtils.getValueAtPath(sourceData, sourcePath); + + if (value != null) { + PathUtils.setValueAtPath(outputData, targetPath, value); + } + } + } + + /** + * Processes array mappings (e.g., datasets -> semanticDataObjects). + */ + private static void processArrayMapping( + Map sourceData, + Map outputData, + String sourceArrayKey, + Map mappings) { + + List sourceArray = getList(sourceData, sourceArrayKey); + + // Get the target array path from mappings + String targetArrayPath = mappings.get(sourceArrayKey); + + // Process each item in the source array + List targetArray = streamMaps(sourceArray) + .map(sourceItem -> { + Map targetItem = new LinkedHashMap<>(); + + // Map nested properties for this array item + for (Map.Entry entry : mappings.entrySet()) { + String sourcePath = entry.getKey(); + String targetPath = entry.getValue(); + + if (sourcePath.equals(sourceArrayKey)) continue; // Skip the array-level mapping + + if (!sourcePath.startsWith(sourceArrayKey + ".")) continue; // Not a nested property of this array + + // Extract the nested path (remove array prefix) + String nestedSourcePath = sourcePath.substring(sourceArrayKey.length() + 1); + String nestedTargetPath = + targetPath.contains(".") ? targetPath.substring(targetPath.lastIndexOf(".") + 1) : targetPath; + + Object value = PathUtils.getValueAtPath(sourceItem, nestedSourcePath); + if (value != null) { + targetItem.put(nestedTargetPath, value); + } + } + + return targetItem; + }) + .collect(java.util.stream.Collectors.toList()); + + PathUtils.setValueAtPath(outputData, targetArrayPath, targetArray); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/osi/converter/MetricMappingHandler.java new file mode 100644 index 00000000..4d1a9da8 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/MetricMappingHandler.java @@ -0,0 +1,137 @@ +package org.osi.converter; + +import static org.osi.converter.ConverterConstants.*; +import static org.osi.util.DataStructureUtils.*; + +import org.osi.converter.ConverterConstants.Level; +import org.osi.converter.pipeline.PipelineStep; +import java.util.*; + +import org.osi.util.MappingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Bidirectional handler for mapping metrics between OSI and Salesforce formats. + * + *

Supports both conversion directions: + *

    + *
  • OSI → Salesforce: unwrap expression from dialects structure
    • + *
  • Salesforce → OSI: wrap expression in dialects structure
    • + *
+ * + */ +public class MetricMappingHandler implements PipelineStep { + + private static final Logger logger = LoggerFactory.getLogger(MetricMappingHandler.class); + + private static final String METRICS = "metrics"; + private static final String SEMANTIC_CALCULATED_MEASUREMENTS = "semanticCalculatedMeasurements"; + + private final ConversionDirection direction; + private final CustomExtensionHandler customExtensionHandler; + + public MetricMappingHandler(ConversionDirection direction, CustomExtensionHandler customExtensionHandler) { + this.direction = direction; + this.customExtensionHandler = customExtensionHandler; + } + + @Override + public void execute(Map sourceData, Map outputData, Map mappings) { + logger.debug("Mapping metrics in {} direction", direction); + if (direction == ConversionDirection.OSI_TO_SALESFORCE) { + mapOsiToSalesforce(sourceData, outputData, mappings); + } else { + mapSalesforceToOsi(sourceData, outputData, mappings); + } + } + + /** + * Maps OSI metrics to Salesforce semanticCalculatedMeasurements. + */ + private void mapOsiToSalesforce( + Map sourceData, Map outputData, Map mappings) { + + List osiMetrics = getList(sourceData, METRICS); + if (osiMetrics == null) { + return; + } + + // Filter mappings to get only metric-related entries + Map metricMappings = MappingUtils.filterMappingsByPrefix(mappings, METRICS); + metricMappings.keySet().forEach(mappings::remove); + + logger.debug("Metrics are not mapped in OSI to Salesforce direction"); + } + + /** + * Maps Salesforce semanticCalculatedMeasurements to OSI metrics. + */ + private void mapSalesforceToOsi( + Map sourceData, Map outputData, Map mappings) { + + List sfMetrics = getList(sourceData, SEMANTIC_CALCULATED_MEASUREMENTS); + if (sfMetrics == null) { + return; + } + + Map metricMappings = + MappingUtils.filterMappingsByPrefix(mappings, SEMANTIC_CALCULATED_MEASUREMENTS); + + Set allHandledProps = metricMappings.isEmpty()? new HashSet<>() : MappingUtils.extractHandledProperties(metricMappings); + allHandledProps.add(EXPRESSION); + + Map mappedData = GenericMappingEngine.applyMappings(sourceData, metricMappings); + metricMappings.keySet().forEach(mappings::remove); + + outputData.putAll(mappedData); + + List osiMetrics = getList(outputData, METRICS); + if (osiMetrics != null) { + wrapExpressions(sfMetrics, osiMetrics); + } + + // Store unmapped SF properties in custom_extensions + customExtensionHandler.storeUnmappedProperties(outputData, sourceData, allHandledProps, Level.METRICS); + + // Cleanup: remove processed structural key + sourceData.remove(SEMANTIC_CALCULATED_MEASUREMENTS); + } + + + /** + * Wraps expressions for SF→OSI conversion. + */ + private void wrapExpressions(List sfMetrics, List osiMetrics) { + for (int i = 0; i < sfMetrics.size() && i < osiMetrics.size(); i++) { + Map sfMetric = asMap(sfMetrics.get(i)); + Map osiMetric = asMap(osiMetrics.get(i)); + + // Get expression from SF metric + String expressionValue = getString(sfMetric, EXPRESSION); + if (expressionValue != null) { + // Wrap in OSI dialect structure + osiMetric.put(EXPRESSION, wrapExpression(expressionValue)); + } + } + } + + /** + * Wraps a simple expression string in OSI's expression.dialects structure. + * Tags expressions with TABLEAU dialect as they come from Salesforce (Tableau CRM). + */ + private Map wrapExpression(String expressionValue) { + Map dialect = new LinkedHashMap<>(); + dialect.put(DIALECT, DIALECT_TABLEAU); + dialect.put(EXPRESSION, expressionValue); + + List dialects = new ArrayList<>(); + dialects.add(dialect); + + Map expression = new LinkedHashMap<>(); + expression.put(DIALECTS, dialects); + + return expression; + } + +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/RelationshipMappingHandler.java b/converters/salesforce/src/main/java/org/osi/converter/RelationshipMappingHandler.java new file mode 100644 index 00000000..cd669a65 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/RelationshipMappingHandler.java @@ -0,0 +1,406 @@ +package org.osi.converter; + +import static org.osi.converter.ConverterConstants.*; +import static org.osi.util.DataStructureUtils.*; + +import org.osi.converter.ConverterConstants.Level; +import org.osi.converter.pipeline.PipelineStep; +import java.util.*; + +import org.osi.util.MappingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Bidirectional handler for mapping relationships between OSI and Salesforce formats. + * + *

Supports both conversion directions: + *

    + *
  • OSI → Salesforce: from/to/from_columns/to_columns → criteria array
    • + *
  • Salesforce → OSI: criteria array → from/to/from_columns/to_columns
    • + *
+ * + */ +public class RelationshipMappingHandler implements PipelineStep { + + private static final Logger logger = LoggerFactory.getLogger(RelationshipMappingHandler.class); + + private final ConversionDirection direction; + private final CustomExtensionHandler customExtensionHandler; + + public RelationshipMappingHandler(ConversionDirection direction, CustomExtensionHandler customExtensionHandler) { + this.direction = direction; + this.customExtensionHandler = customExtensionHandler; + } + + @Override + public void execute(Map sourceData, Map outputData, Map mappings) { + logger.debug("Mapping relationships in {} direction", direction); + if (direction == ConversionDirection.OSI_TO_SALESFORCE) { + mapOsiToSalesforce(sourceData, outputData, mappings); + } else { + mapSalesforceToOsi(sourceData, outputData, mappings); + } + } + + /** + * Maps OSI relationships to Salesforce semanticRelationships. + */ + private void mapOsiToSalesforce( + Map sourceData, Map outputData, Map mappings) { + + List osiRelationships = getList(sourceData, RELATIONSHIPS); + if (osiRelationships == null) { + return; + } + + // Validate and filter relationships - remove those with non-existent fields + List validRelationships = validateAndFilterRelationships(osiRelationships, outputData); + if (validRelationships.isEmpty()) { + return; + } + + // Update sourceData with filtered relationships + sourceData.put(RELATIONSHIPS, validRelationships); + + Map relationshipMappings = MappingUtils.filterMappingsByPrefix(mappings, RELATIONSHIPS); + + Map mappedData = GenericMappingEngine.applyMappings(sourceData, relationshipMappings); + relationshipMappings.keySet().forEach(mappings::remove); + + outputData.putAll(mappedData); + + // Apply manual mappings (criteria) + List sfRelationships = getList(outputData, SEMANTIC_RELATIONSHIPS); + if (sfRelationships != null) { + reconstructCriteria(osiRelationships, sfRelationships); + } + + customExtensionHandler.restoreCustomExtensionsAtLevel(outputData, sourceData, Level.RELATIONSHIPS); + + if (sfRelationships != null) { + applyDefaults(sfRelationships); + } + } + + /** + * Maps Salesforce semanticRelationships to OSI relationships. + */ + private void mapSalesforceToOsi( + Map sourceData, Map outputData, Map mappings) { + + List sfRelationships = getList(sourceData, SEMANTIC_RELATIONSHIPS); + if (sfRelationships == null) { + return; + } + + // Filter out unsupported relationships (Formula/SemanticField) and store at model level + List unsupportedRelationships = new ArrayList<>(); + List supportedRelationships = new ArrayList<>(); + + for (Object relObj : sfRelationships) { + Map sfRel = asMap(relObj); + if (hasUnsupportedFieldTypes(sfRel)) { + unsupportedRelationships.add(sfRel); + } else { + supportedRelationships.add(sfRel); + } + } + + // Store unsupported relationships in custom_extensions at SEMANTIC_MODEL level + if (!unsupportedRelationships.isEmpty()) { + storeUnsupportedRelationshipsAtModelLevel(outputData, unsupportedRelationships); + } + + // Only process supported relationships + if (supportedRelationships.isEmpty()) { + return; + } + + // Update sourceData to only contain supported relationships + sourceData.put(SEMANTIC_RELATIONSHIPS, supportedRelationships); + + Map relationshipMappings = + MappingUtils.filterMappingsByPrefix(mappings, SEMANTIC_RELATIONSHIPS); + + Set allHandledProps = relationshipMappings.isEmpty()? new HashSet<>() : MappingUtils.extractHandledProperties(relationshipMappings); + + allHandledProps.add(LEFT_SEMANTIC_DEFINITION_API_NAME); + allHandledProps.add(RIGHT_SEMANTIC_DEFINITION_API_NAME); + allHandledProps.add(CRITERIA); + + Map mappedData = GenericMappingEngine.applyMappings(sourceData, relationshipMappings); + relationshipMappings.keySet().forEach(mappings::remove); + + outputData.putAll(mappedData); + + List osiRelationships = getList(outputData, RELATIONSHIPS); + if (osiRelationships != null) { + deconstructCriteria(supportedRelationships, osiRelationships); + } + + // Store unmapped SF properties in custom_extensions + customExtensionHandler.storeUnmappedProperties(outputData, sourceData, allHandledProps, Level.RELATIONSHIPS); + + // Cleanup: remove processed structural key + sourceData.remove(SEMANTIC_RELATIONSHIPS); + } + + /** + * Reconstructs criteria for OSI→SF conversion. + */ + private void reconstructCriteria(List osiRelationships, List sfRelationships) { + for (int i = 0; i < osiRelationships.size() && i < sfRelationships.size(); i++) { + Map osiRel = asMap(osiRelationships.get(i)); + Map sfRel = asMap(sfRelationships.get(i)); + + String fromEntity = getString(osiRel, FROM); + String toEntity = getString(osiRel, TO); + + if (fromEntity != null) { + sfRel.put(LEFT_SEMANTIC_DEFINITION_API_NAME, fromEntity); + } + if (toEntity != null) { + sfRel.put(RIGHT_SEMANTIC_DEFINITION_API_NAME, toEntity); + } + + Object fromColumnsObj = osiRel.get(FROM_COLUMNS); + Object toColumnsObj = osiRel.get(TO_COLUMNS); + + if (fromColumnsObj == null || toColumnsObj == null) { + return; + } + + List fromColumns = asList(fromColumnsObj); + List toColumns = asList(toColumnsObj); + + if (fromColumns.isEmpty() || toColumns.isEmpty()) { + return; + } + + if (fromColumns.size() != toColumns.size()) { + return; + } + + List> criteriaArray = new ArrayList<>(); + for (int j = 0; j < fromColumns.size(); j++) { + String fromCol = (String) fromColumns.get(j); + String toCol = (String) toColumns.get(j); + + Map criterion = new LinkedHashMap<>(); + criterion.put(LEFT_SEMANTIC_FIELD_API_NAME, fromCol); + criterion.put(RIGHT_SEMANTIC_FIELD_API_NAME, toCol); + criteriaArray.add(criterion); + } + + if (!criteriaArray.isEmpty()) { + sfRel.put(CRITERIA, criteriaArray); + } + } + } + + /** + * Deconstructs criteria for SF→OSI conversion. + * Note: This method only processes supported relationships (TableField types). + * Unsupported relationships are filtered out earlier and stored in custom_extensions. + */ + private void deconstructCriteria(List sfRelationships, List osiRelationships) { + for (int i = 0; i < sfRelationships.size() && i < osiRelationships.size(); i++) { + Map sfRel = asMap(sfRelationships.get(i)); + Map osiRel = asMap(osiRelationships.get(i)); + + // Extract leftSemanticDefinitionApiName → from + String leftDef = getString(sfRel, LEFT_SEMANTIC_DEFINITION_API_NAME); + if (leftDef != null) { + osiRel.put(FROM, leftDef); + } + + // Extract rightSemanticDefinitionApiName → to + String rightDef = getString(sfRel, RIGHT_SEMANTIC_DEFINITION_API_NAME); + if (rightDef != null) { + osiRel.put(TO, rightDef); + } + + Object criteriaObj = sfRel.get(CRITERIA); + if (criteriaObj != null) { + List criteria = asList(criteriaObj); + + List fromColumns = new ArrayList<>(); + List toColumns = new ArrayList<>(); + + for (Object criterionObj : criteria) { + Map criterion = asMap(criterionObj); + + String leftField = getString(criterion, LEFT_SEMANTIC_FIELD_API_NAME); + String rightField = getString(criterion, RIGHT_SEMANTIC_FIELD_API_NAME); + + if (leftField != null && rightField != null) { + fromColumns.add(leftField); + toColumns.add(rightField); + } + } + + if (!fromColumns.isEmpty()) { + osiRel.put(FROM_COLUMNS, fromColumns); + } + if (!toColumns.isEmpty()) { + osiRel.put(TO_COLUMNS, toColumns); + } + } + } + } + + /** + * Applies default values for required Salesforce relationship fields. + * Only sets defaults if the property is not already present. + */ + private void applyDefaults(List sfRelationships) { + for (Object relObj : sfRelationships) { + Map sfRel = asMap(relObj); + + sfRel.putIfAbsent(CARDINALITY, DEFAULT_CARDINALITY); + sfRel.putIfAbsent(IS_ENABLED, true); + sfRel.putIfAbsent(JOIN_TYPE, DEFAULT_JOIN_TYPE); + } + } + + /** + * Validates and filters relationships, removing those that reference non-existent fields. (Calculated fields that are not supported) + * + * @param osiRelationships List of OSI relationships to validate + * @param outputData The output data containing semanticDataObjects with their fields + * @return Filtered list of valid relationships + */ + private List validateAndFilterRelationships(List osiRelationships, Map outputData) { + List validRelationships = new ArrayList<>(); + List sfDataObjects = getList(outputData, SEMANTIC_DATA_OBJECTS); + + for (Object relObj : osiRelationships) { + Map osiRel = asMap(relObj); + String relName = getString(osiRel, NAME); + String fromEntity = getString(osiRel, FROM); + String toEntity = getString(osiRel, TO); + + Map fromDataObject = findDataObjectByName(sfDataObjects, fromEntity); + Map toDataObject = findDataObjectByName(sfDataObjects, toEntity); + + if (fromDataObject == null || toDataObject == null) { + logger.debug("Removing relationship '{}' - entity not found", relName); + continue; + } + + List fromColumns = getList(osiRel, FROM_COLUMNS); + List toColumns = getList(osiRel, TO_COLUMNS); + + if (!validateColumns(fromColumns, fromDataObject, fromEntity, relName) || + !validateColumns(toColumns, toDataObject, toEntity, relName)) { + continue; + } + validRelationships.add(osiRel); + } + return validRelationships; + } + + /** + * Validates that all columns exist in the given data object. + * + * @param columns List of column names to validate + * @param dataObject The data object containing the fields + * @param entityName The entity name (for logging) + * @param relName The relationship name (for logging) + * @return true if all columns exist, false otherwise + */ + private boolean validateColumns(List columns, Map dataObject, + String entityName, String relName) { + if (columns == null || columns.isEmpty()) { + return true; + } + + for (Object colObj : columns) { + String columnName = (String) colObj; + if (!fieldExistsInDataObject(dataObject, columnName)) { + logger.debug("Removing relationship '{}' - column '{}' not found in entity '{}'", + relName, columnName, entityName); + return false; + } + } + + return true; + } + + /** + * Finds a data object by its apiName. + */ + private Map findDataObjectByName(List dataObjects, String name) { + for (Object obj : dataObjects) { + Map dataObject = asMap(obj); + String apiName = getString(dataObject, API_NAME); + if (name.equals(apiName)) { + return dataObject; + } + } + return null; + } + + /** + * Checks if a field exists in a data object's semanticDimensions or semanticMeasurements. + */ + private boolean fieldExistsInDataObject(Map dataObject, String fieldName) { + // Check both semanticDimensions and semanticMeasurements + for (String fieldListKey : List.of(SEMANTIC_DIMENSIONS, SEMANTIC_MEASUREMENTS)) { + List fields = getList(dataObject, fieldListKey); + if (fields != null) { + for (Object fieldObj : fields) { + Map field = asMap(fieldObj); + String apiName = getString(field, API_NAME); + if (fieldName.equals(apiName)) { + return true; + } + } + } + } + return false; + } + + /** + * Checks if a relationship has unsupported field types (Formula or SemanticField). + * + * @param relationship The relationship to check + * @return true if the relationship contains Formula or SemanticField types + */ + private boolean hasUnsupportedFieldTypes(Map relationship) { + Object criteriaObj = relationship.get(CRITERIA); + if (criteriaObj == null) { + return false; + } + + List criteria = asList(criteriaObj); + for (Object criterionObj : criteria) { + Map criterion = asMap(criterionObj); + String leftFieldType = getString(criterion, LEFT_FIELD_TYPE); + String rightFieldType = getString(criterion, RIGHT_FIELD_TYPE); + + if (FIELD_TYPE_FORMULA.equals(leftFieldType) || FIELD_TYPE_FORMULA.equals(rightFieldType) || + FIELD_TYPE_SEMANTIC_FIELD.equals(leftFieldType) || FIELD_TYPE_SEMANTIC_FIELD.equals(rightFieldType)) { + logger.debug("Relationship '{}' contains unsupported field types (Formula/SemanticField)", + getString(relationship, API_NAME)); + return true; + } + } + return false; + } + + /** + * Stores unsupported relationships in custom_extensions at the SEMANTIC_MODEL level. + * + * @param outputData The output data structure + * @param unsupportedRelationships List of relationships that cannot be converted + */ + private void storeUnsupportedRelationshipsAtModelLevel(Map outputData, + List unsupportedRelationships) { + Map customData = new LinkedHashMap<>(); + customData.put(SEMANTIC_RELATIONSHIPS, unsupportedRelationships); + + customExtensionHandler.addCustomExtension(outputData, customData); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/SemanticModelMappingHandler.java b/converters/salesforce/src/main/java/org/osi/converter/SemanticModelMappingHandler.java new file mode 100644 index 00000000..5bbfbbd9 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/SemanticModelMappingHandler.java @@ -0,0 +1,140 @@ +package org.osi.converter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.osi.converter.ConverterConstants.Level; +import org.osi.converter.pipeline.PipelineStep; +import org.osi.exception.ConversionException; +import org.osi.util.MappingUtils; + +import java.util.Map; +import java.util.Set; + +import static org.osi.converter.ConverterConstants.AI_CONTEXT; +import static org.osi.converter.ConverterConstants.API_NAME; +import static org.osi.converter.ConverterConstants.BUSINESS_PREFERENCES; +import static org.osi.converter.ConverterConstants.LABEL; + +/** + * Bidirectional handler for mapping top-level semantic model properties. + * + *

Supports both conversion directions: + *

    + *
  • OSI → Salesforce: name → apiName, apply SF defaults, restore custom_extensions
    • + *
  • Salesforce → OSI: apiName → name, strip SF defaults, store custom_extensions
    • + *
+ * + *

Handles only top-level scalar properties (not arrays). + * + */ +public class SemanticModelMappingHandler implements PipelineStep { + + private final ConversionDirection direction; + private final CustomExtensionHandler customExtensionHandler; + private final ObjectMapper jsonMapper; + + public SemanticModelMappingHandler(ConversionDirection direction, CustomExtensionHandler customExtensionHandler) { + this.direction = direction; + this.customExtensionHandler = customExtensionHandler; + this.jsonMapper = new ObjectMapper(); + } + + @Override + public void execute(Map sourceData, Map outputData, Map mappings) { + if (direction == ConversionDirection.OSI_TO_SALESFORCE) { + mapOsiToSalesforce(sourceData, outputData, mappings); + } else { + mapSalesforceToOsi(sourceData, outputData, mappings); + } + } + + /** + * Maps OSI top-level properties to Salesforce format. + * Steps: generic mappings → manual conversions → restore custom extensions + */ + private void mapOsiToSalesforce( + Map sourceData, Map outputData, Map mappings) { + + var mappedData = GenericMappingEngine.applyMappings(sourceData, mappings); + outputData.putAll(mappedData); + + convertAiContextToBusinessPreferences(sourceData, outputData); + + customExtensionHandler.restoreCustomExtensionsAtLevel(outputData, sourceData, Level.SEMANTIC_MODEL); + + applyDefaults(outputData); + } + + /** + * Maps Salesforce top-level properties to OSI format. + * Steps: generic mappings → manual conversions → store custom extensions + */ + private void mapSalesforceToOsi( + Map sourceData, Map outputData, Map mappings) { + + var mappedData = GenericMappingEngine.applyMappings(sourceData, mappings); + outputData.putAll(mappedData); + + convertBusinessPreferencesToAiContext(sourceData, outputData); + + Set allHandledProps = MappingUtils.extractTopLevelKeys(mappings); + allHandledProps.add(BUSINESS_PREFERENCES); + + customExtensionHandler.storeUnmappedProperties(outputData, sourceData, allHandledProps, Level.SEMANTIC_MODEL); + } + + /** + * Converts Salesforce businessPreferences to OSI ai_context. + * + * @param sourceData Salesforce data containing businessPreferences + * @param outputData OSI data to populate with ai_context + */ + private void convertBusinessPreferencesToAiContext(Map sourceData, Map outputData) { + Object businessPreferences = sourceData.get(BUSINESS_PREFERENCES); + if (businessPreferences != null) { + outputData.put(AI_CONTEXT, businessPreferences); + } + } + + /** + * Converts OSI ai_context (string or object) to Salesforce businessPreferences (string). + * + *

ai_context can be: + *

    + *
  • A simple string - copied as-is
    • + *
  • An object - serialized to JSON string
    • + *
+ * + * @param sourceData OSI data containing ai_context + * @param outputData Salesforce data to populate with businessPreferences + */ + private void convertAiContextToBusinessPreferences(Map sourceData, Map outputData) { + Object aiContextObj = sourceData.get(AI_CONTEXT); + if (aiContextObj == null) { + return; + } + + String businessPreferences; + if (aiContextObj instanceof String) { + businessPreferences = aiContextObj.toString(); + } else { + try { + businessPreferences = jsonMapper.writeValueAsString(aiContextObj); + } catch (JsonProcessingException e) { + throw new ConversionException("Failed to serialize ai_context to JSON: " + e.getMessage(), e); + } + } + outputData.put(BUSINESS_PREFERENCES, businessPreferences); + } + + /** + * Applies default values for required Salesforce semantic model properties. + * Used when converting OSI → Salesforce. + */ + private void applyDefaults(Map outputData) { + if (!outputData.containsKey(LABEL)) { + String apiName = (String) outputData.get(API_NAME); + outputData.put(LABEL, apiName); + } + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/DirectionConfig.java b/converters/salesforce/src/main/java/org/osi/converter/pipeline/DirectionConfig.java new file mode 100644 index 00000000..b7df1dbe --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/pipeline/DirectionConfig.java @@ -0,0 +1,56 @@ +package org.osi.converter.pipeline; + +import static org.osi.converter.ConverterConstants.*; + +/** + * Configuration for a specific conversion direction. + * + */ +public class DirectionConfig { + private String inputFormat; + private String outputFormat; + private String schemaPath; + private String extractModelNameFrom; + + public String getInputFormat() { + return inputFormat; + } + + public void setInputFormat(String inputFormat) { + this.inputFormat = inputFormat; + } + + public String getOutputFormat() { + return outputFormat; + } + + public void setOutputFormat(String outputFormat) { + this.outputFormat = outputFormat; + } + + public String getSchemaPath() { + return schemaPath; + } + + public void setSchemaPath(String schemaPath) { + this.schemaPath = schemaPath; + } + + public String getExtractModelNameFrom() { + return extractModelNameFrom; + } + + public void setExtractModelNameFrom(String extractModelNameFrom) { + this.extractModelNameFrom = extractModelNameFrom; + } + + /** + * Get file extension based on output format. + * @return JSON_EXTENSION for json format, YAML_EXTENSION for yaml format + */ + public String getFileExtension() { + return JSON.equals(outputFormat) + ? JSON_EXTENSION + : YAML_EXTENSION; + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/HandlerFactory.java b/converters/salesforce/src/main/java/org/osi/converter/pipeline/HandlerFactory.java new file mode 100644 index 00000000..4b98824a --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/pipeline/HandlerFactory.java @@ -0,0 +1,42 @@ +package org.osi.converter.pipeline; + +import org.osi.converter.*; +import org.osi.exception.ConversionException; + +/** + * Factory for creating handler instances using a hardcoded registry. + * pipeline configuration specifies which handlers to run and in what order. + * + */ +public class HandlerFactory { + private final CustomExtensionHandler customExtensionHandler; + + public HandlerFactory(CustomExtensionHandler customExtensionHandler) { + this.customExtensionHandler = customExtensionHandler; + } + + /** + * Creates a handler instance from the registered handler names. + * + * @param handlerName The handler name from pipeline config + * @param direction The conversion direction + * @return A PipelineStep instance + * @throws ConversionException if handler name is unknown + */ + public PipelineStep createHandler(String handlerName, ConversionDirection direction) { + return switch(handlerName) { + case "DatasetMappingHandler" -> + new DatasetMappingHandler(direction, customExtensionHandler); + case "FieldMappingHandler" -> + new FieldMappingHandler(direction, customExtensionHandler); + case "RelationshipMappingHandler" -> + new RelationshipMappingHandler(direction, customExtensionHandler); + case "MetricMappingHandler" -> + new MetricMappingHandler(direction, customExtensionHandler); + case "SemanticModelMappingHandler" -> + new SemanticModelMappingHandler(direction, customExtensionHandler); + default -> + throw new ConversionException("Unknown handler: " + handlerName); + }; + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfig.java b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfig.java new file mode 100644 index 00000000..ec9f1423 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfig.java @@ -0,0 +1,29 @@ +package org.osi.converter.pipeline; + +import java.util.List; +import java.util.Map; + +/** + * Root configuration model matching pipeline-config.yaml structure. + * + */ +public class PipelineConfig { + private Map> pipelines; // Direction -> handler names + private Map directionConfigs; // Direction -> config + + public Map> getPipelines() { + return pipelines; + } + + public void setPipelines(Map> pipelines) { + this.pipelines = pipelines; + } + + public Map getDirectionConfigs() { + return directionConfigs; + } + + public void setDirectionConfigs(Map directionConfigs) { + this.directionConfigs = directionConfigs; + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfigLoader.java b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfigLoader.java new file mode 100644 index 00000000..6805a543 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfigLoader.java @@ -0,0 +1,60 @@ +package org.osi.converter.pipeline; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.osi.exception.ConversionException; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Loads pipeline configuration from YAML. + * Uses Jackson to deserialize osi-salesforce-converter-config.yaml into PipelineConfig model. + * + */ +public class PipelineConfigLoader { + private static final String CONFIG_RESOURCE = "/osi-salesforce-converter-config.yaml"; + private final ObjectMapper yamlMapper; + + public PipelineConfigLoader() { + YAMLFactory yamlFactory = new YAMLFactory(); + this.yamlMapper = new ObjectMapper(yamlFactory); + } + + public static PipelineConfig loadFromResource() { + PipelineConfigLoader loader = new PipelineConfigLoader(); + try (InputStream stream = PipelineConfigLoader.class.getResourceAsStream(CONFIG_RESOURCE)) { + if (stream == null) { + throw new ConversionException("Pipeline config not found: " + CONFIG_RESOURCE); + } + + // Read full YAML into map + Map rawConfig = loader.yamlMapper.readValue(stream, new TypeReference<>() {}); + + // Parse into structured config + PipelineConfig config = new PipelineConfig(); + config.setPipelines((Map>) rawConfig.get("pipelines")); + + // Parse direction configs (osiToSalesforce, salesforceToOsi sections) + Map directionConfigs = new HashMap<>(); + for (String direction : config.getPipelines().keySet()) { + if (rawConfig.containsKey(direction)) { + DirectionConfig dirConfig = loader.yamlMapper.convertValue( + rawConfig.get(direction), + DirectionConfig.class + ); + directionConfigs.put(direction, dirConfig); + } + } + config.setDirectionConfigs(directionConfigs); + + return config; + } catch (IOException e) { + throw new ConversionException("Failed to load pipeline config", e); + } + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineStep.java b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineStep.java new file mode 100644 index 00000000..bec154d2 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineStep.java @@ -0,0 +1,19 @@ +package org.osi.converter.pipeline; + +import java.util.Map; + +/** + * Base interface for pipeline steps. + * Both mapping handlers and special wrapper steps implement this interface. + * + */ +public interface PipelineStep { + /** + * Execute this pipeline step. + * + * @param sourceData The source data (may be modified by handler) + * @param outputData The output data being built + * @param mappings Property mappings + */ + void execute(Map sourceData, Map outputData, Map mappings); +} diff --git a/converters/salesforce/src/main/java/org/osi/exception/ConversionException.java b/converters/salesforce/src/main/java/org/osi/exception/ConversionException.java new file mode 100644 index 00000000..f9d8fdda --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/exception/ConversionException.java @@ -0,0 +1,28 @@ +package org.osi.exception; + +/** + * Exception thrown when a conversion operation fails. + * This can happen during YAML to JSON or JSON to YAML conversion. + * + */ +public class ConversionException extends RuntimeException { + + /** + * Constructs a new ConversionException with the specified detail message. + * + * @param message the detail message + */ + public ConversionException(String message) { + super(message); + } + + /** + * Constructs a new ConversionException with the specified detail message and cause. + * + * @param message the detail message + * @param cause the cause of the exception + */ + public ConversionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/exception/InvalidInputException.java b/converters/salesforce/src/main/java/org/osi/exception/InvalidInputException.java new file mode 100644 index 00000000..079dc63f --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/exception/InvalidInputException.java @@ -0,0 +1,29 @@ +package org.osi.exception; + +/** + * Exception thrown when the input is invalid. + * This includes cases where the file does not exist, is not readable, + * or if the file/String contains invalid YAML/JSON content. + * + */ +public class InvalidInputException extends RuntimeException { + + /** + * Constructs a new InvalidInputException with the specified detail message. + * + * @param message the detail message + */ + public InvalidInputException(String message) { + super(message); + } + + /** + * Constructs a new InvalidInputException with the specified detail message and cause. + * + * @param message the detail message + * @param cause the cause of the exception + */ + public InvalidInputException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/exception/ValidationException.java b/converters/salesforce/src/main/java/org/osi/exception/ValidationException.java new file mode 100644 index 00000000..68d6268b --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/exception/ValidationException.java @@ -0,0 +1,16 @@ +package org.osi.exception; + +/** + * Exception thrown when schema validation fails. + * + */ +public class ValidationException extends RuntimeException { + + public ValidationException(String message) { + super(message); + } + + public ValidationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/mapper/FileBasedPropertyMapper.java b/converters/salesforce/src/main/java/org/osi/mapper/FileBasedPropertyMapper.java new file mode 100644 index 00000000..40d4783f --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/mapper/FileBasedPropertyMapper.java @@ -0,0 +1,93 @@ +package org.osi.mapper; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.osi.exception.InvalidInputException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Property mapper that loads mappings from a YAML configuration file. + * + *

This mapper reads the bundled mappings.yaml which defines OSI-Salesforce + * property mappings. The transformation is performed by the converter classes.

+ * + */ +public class FileBasedPropertyMapper implements PropertyMapper { + + private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory()); + + private final Map mappings; + private final Map reverseMappings; + + /** + * Constructs a FileBasedPropertyMapper with the specified mappings. + * + * @param mappings the OSI-Salesforce mappings + */ + public FileBasedPropertyMapper(Map mappings) { + this.mappings = mappings != null ? new LinkedHashMap<>(mappings) : new LinkedHashMap<>(); + this.reverseMappings = generateReverseMappings(); + } + + /** + * Generates reverse mappings (Salesforce to OSI) from the OSI to Salesforce mappings. + * + * @return the Salesforce-to-OSI mappings + */ + private Map generateReverseMappings() { + return mappings.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + } + + /** + * Creates a FileBasedPropertyMapper from a classpath resource. + * + * @param resourcePath the path to the YAML resource (e.g., "/mappings.yaml") + * @return a new FileBasedPropertyMapper + * @throws InvalidInputException if the resource cannot be read or parsed + */ + public static FileBasedPropertyMapper fromResource(String resourcePath) { + try (InputStream is = FileBasedPropertyMapper.class.getResourceAsStream(resourcePath)) { + if (is == null) { + throw new InvalidInputException("Mapping configuration resource not found: " + resourcePath); + } + + String content = new String(is.readAllBytes()); + return parseYaml(content); + } catch (IOException e) { + throw new InvalidInputException("Failed to read mapping configuration resource: " + resourcePath, e); + } + } + + private static FileBasedPropertyMapper parseYaml(String content) { + try { + Map mappings = YAML_MAPPER.readValue( + content, + new TypeReference>() {} + ); + return new FileBasedPropertyMapper(mappings); + } catch (Exception e) { + throw new InvalidInputException("Failed to parse YAML mapping configuration: " + e.getMessage(), e); + } + } + + @Override + public Map getOsiToSalesforceMappings() { + return Collections.unmodifiableMap(mappings); + } + + @Override + public Map getSalesforceToOsiMappings() { + return Collections.unmodifiableMap(reverseMappings); + } + + @Override + public String toString() { + return "FileBasedPropertyMapper{" + "mappingCount=" + mappings.size() + '}'; + } +} diff --git a/converters/salesforce/src/main/java/org/osi/mapper/PropertyMapper.java b/converters/salesforce/src/main/java/org/osi/mapper/PropertyMapper.java new file mode 100644 index 00000000..af8789bc --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/mapper/PropertyMapper.java @@ -0,0 +1,30 @@ +package org.osi.mapper; + +import java.util.Map; + +/** + * Interface for mapping properties between OSI and Salesforce formats during conversion. + * + *

Implementations of this interface define bidirectional mappings between OSI YAML format + * and Salesforce JSON format. This supports nested paths using dot notation + * (e.g., "datasets.name" ↔ "semanticDataObjects.apiName").

+ * + * + */ +public interface PropertyMapper { + + /** + * Returns the mapping from OSI property paths to Salesforce property paths. + * + * @return a map where keys are OSI property paths and values are Salesforce property paths + */ + Map getOsiToSalesforceMappings(); + + /** + * Returns the mapping from Salesforce property paths to OSI property paths. + *

This is the reverse mapping of {@link #getOsiToSalesforceMappings()}. + * + * @return a map where keys are Salesforce property paths and values are OSI property paths + */ + Map getSalesforceToOsiMappings(); +} diff --git a/converters/salesforce/src/main/java/org/osi/util/DataStructureUtils.java b/converters/salesforce/src/main/java/org/osi/util/DataStructureUtils.java new file mode 100644 index 00000000..87f3a02f --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/util/DataStructureUtils.java @@ -0,0 +1,123 @@ +package org.osi.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * Utility class for working with generic Map/List data structures. + * Provides type-safe accessors to reduce casting boilerplate. + * + */ +public final class DataStructureUtils { + + private DataStructureUtils() {} + + /** + * Gets a String value from a map. + * + * @param map The map to get the value from + * @param key The key to look up + * @return The String value, or null if not found + */ + @SuppressWarnings("unchecked") + public static String getString(Map map, String key) { + return (String) map.get(key); + } + + /** + * Gets a Map value from a map. + * + * @param map The map to get the value from + * @param key The key to look up + * @return The Map value, or null if not found + */ + @SuppressWarnings("unchecked") + public static Map getMap(Map map, String key) { + return (Map) map.get(key); + } + + /** + * Gets a List value from a map. + * + * @param map The map to get the value from + * @param key The key to look up + * @return The List value, or null if not found + */ + @SuppressWarnings("unchecked") + public static List getList(Map map, String key) { + return (List) map.get(key); + } + + /** + * Safely casts an Object to Map. + * + * @param obj The object to cast + * @return The Map value + */ + @SuppressWarnings("unchecked") + public static Map asMap(Object obj) { + return (Map) obj; + } + + /** + * Safely casts an Object to List<Object>. + * Caller should check instanceof List before calling this method. + * + * @param obj the object to cast + * @return the object cast to List<Object> + */ + @SuppressWarnings("unchecked") + public static List asList(Object obj) { + return (List) obj; + } + + /** + * Gets an existing list from a map, or creates a new empty ArrayList if not present. + * Encapsulates the unchecked cast from computeIfAbsent. + * + * @param map The map to get or create the list in + * @param key The key for the list + * @return The existing or newly created list + */ + @SuppressWarnings("unchecked") + public static List getOrCreateList(Map map, String key) { + return (List) map.computeIfAbsent(key, k -> new ArrayList<>()); + } + + /** + * Filters a list to only Map items and returns a stream of safely cast Maps. + * This reduces boilerplate when iterating over heterogeneous lists. + * + *

Example usage: + * 

{@code
+     * streamMaps(array).forEach(item -> {
+     *     // process item...
+     * });
+     * }
+ * + * @param list The list to filter and stream + * @return A stream of Map items from the list + */ + public static Stream> streamMaps(List list) { + return list.stream() + .filter(obj -> obj instanceof Map) + .map(DataStructureUtils::asMap); + } + + /** + * Finds an item in an array by matching an identifier field. + * + * @param array The array to search + * @param idKey The key to match (e.g., "apiName", "name") + * @param idValue The value to match + * @return The matching item or null + */ + public static Map findItemById(List array, String idKey, String idValue) { + return streamMaps(array) + .filter(item -> idValue.equals(item.get(idKey))) + .findFirst() + .orElse(null); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/util/MappingUtils.java b/converters/salesforce/src/main/java/org/osi/util/MappingUtils.java new file mode 100644 index 00000000..cb17380c --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/util/MappingUtils.java @@ -0,0 +1,101 @@ +package org.osi.util; + +import java.util.*; + +/** + * Utility class for filtering and processing property mappings. + * + */ +public class MappingUtils { + + /** + * Extracts the first segment of a dot-separated path. + * + * Example: + * - "name" → "name" + * - "datasets.name" → "datasets" + * - "datasets.fields.name" → "datasets" + * + * @param path The dot-separated path + * @return The first segment before the first dot + */ + public static String getFirstPathSegment(String path) { + return path.split("\\.")[0]; + } + + /** + * Filters mappings to get all entries with a specific prefix. + * Returns map with only the matching entries. + * + * Example: filterMappingsByPrefix(mappings, "datasets") returns: + * - Input: {"datasets": "semanticDataObjects", "datasets.name": "semanticDataObjects.apiName", "name": "apiName"} + * - Output: {"datasets": "semanticDataObjects", "datasets.name": "semanticDataObjects.apiName", "datasets.source": "..."} + * + * @param allMappings All property mappings + * @param prefix The prefix to filter by (e.g., "datasets", "relationships", "metrics") + * @return Filtered map containing only entries that match the prefix + */ + public static Map filterMappingsByPrefix(Map allMappings, String prefix) { + Map filtered = new LinkedHashMap<>(); + String prefixWithDot = prefix + "."; + + for (Map.Entry entry : allMappings.entrySet()) { + String key = entry.getKey(); + if (key.equals(prefix) || key.startsWith(prefixWithDot)) { + filtered.put(key, entry.getValue()); + } + } + + return filtered; + } + + /** + * Extracts all unique top-level keys from mappings. + * + * Example: + * - "name" → "name" + * - "description" → "description" + * - "datasets.name" → "datasets" + * - "datasets.fields.name" → "datasets" + * + * @param mappings The mappings to extract from + * @return Set of unique top-level keys + */ + public static Set extractTopLevelKeys(Map mappings) { + Set topLevelKeys = new HashSet<>(); + for (String key : mappings.keySet()) { + topLevelKeys.add(getFirstPathSegment(key)); + } + return topLevelKeys; + } + + /** + * Extracts handled properties from filtered mappings. + * + *

Returns: prefix + all first-level nested properties

+ * + *

The prefix is inferred from the filtered mappings by extracting + * the first segment from any key.

+ * + * Example: prefix "semanticDataObjects" (inferred) + * Input mappings: {"semanticDataObjects" → "datasets", "semanticDataObjects.apiName" → "datasets.name"} + * Output: {"semanticDataObjects", "apiName", "description"} + * + * @param filteredMappings Filtered mappings + * @return Set including prefix + first-level properties + */ + public static Set extractHandledProperties(Map filteredMappings) { + + Set handledProps = new HashSet<>(); + String prefix = getFirstPathSegment(filteredMappings.keySet().iterator().next()); + handledProps.add(prefix); + + // Extract first-level properties from keys + for (String sourceKey : filteredMappings.keySet()) { + if (sourceKey.startsWith(prefix + ".")) { + handledProps.add(getFirstPathSegment(sourceKey.substring(prefix.length() + 1))); + } + } + return handledProps; + } +} diff --git a/converters/salesforce/src/main/java/org/osi/util/PathUtils.java b/converters/salesforce/src/main/java/org/osi/util/PathUtils.java new file mode 100644 index 00000000..4ed48611 --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/util/PathUtils.java @@ -0,0 +1,123 @@ +package org.osi.util; + +import static org.osi.util.DataStructureUtils.asMap; +import static org.osi.util.DataStructureUtils.asList; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Utility class for working with nested property paths. + * + *

Provides methods for getting and setting values in nested maps using + * dot-notation paths (e.g., "a.b.c").

+ * + */ +public final class PathUtils { + + private PathUtils() { + // Utility class, no instantiation + } + + /** + * Gets a value from a nested map structure using a dot-notation path. + * + * @param data the root map + * @param path the dot-notation path (e.g., "a.b.c") + * @return the value at the path, or null if not found + */ + public static Object getValueAtPath(Map data, String path) { + if (path == null || path.isEmpty()) { + return null; + } + + String[] parts = path.split("\\."); + Object current = data; + + for (String part : parts) { + if (current instanceof Map) { + current = asMap(current).get(part); + } else { + return null; + } + } + + return current; + } + + /** + * Sets a value in a nested map structure using a dot-notation path. + * Creates intermediate maps as needed. + * + * @param data the root map + * @param path the dot-notation path (e.g., "a.b.c") + * @param value the value to set + */ + public static void setValueAtPath(Map data, String path, Object value) { + if (path == null || path.isEmpty()) { + return; + } + + String[] parts = path.split("\\."); + Map current = data; + + for (int i = 0; i < parts.length - 1; i++) { + String part = parts[i]; + Object next = current.get(part); + + if (next instanceof Map) { + current = asMap(next); + } else { + Map newMap = new LinkedHashMap<>(); + current.put(part, newMap); + current = newMap; + } + } + + current.put(parts[parts.length - 1], value); + } + + /** + * Creates a deep copy of a map. + */ + public static Map deepCopy(Map data) { + if (data == null) { + return null; + } + + Map copy = new LinkedHashMap<>(); + for (Map.Entry entry : data.entrySet()) { + copy.put(entry.getKey(), deepCopyValue(entry.getValue())); + } + return copy; + } + + /** + * Creates a deep copy of a list. + */ + public static List deepCopyList(List list) { + List copy = new ArrayList<>(); + for (Object item : list) { + copy.add(deepCopyValue(item)); + } + return copy; + } + + /** + * Deep copies a value (Map, List, or primitive). + * + * @param value the value to copy + * @return the deep copy + */ + public static Object deepCopyValue(Object value) { + if (value instanceof Map) { + return deepCopy(asMap(value)); + } else if (value instanceof List) { + return deepCopyList(asList(value)); + } else { + return value; + } + } +} diff --git a/converters/salesforce/src/main/java/org/osi/validator/SchemaValidator.java b/converters/salesforce/src/main/java/org/osi/validator/SchemaValidator.java new file mode 100644 index 00000000..9cd0adaf --- /dev/null +++ b/converters/salesforce/src/main/java/org/osi/validator/SchemaValidator.java @@ -0,0 +1,104 @@ +package org.osi.validator; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import java.io.InputStream; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.osi.exception.ValidationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Validates OSI semantic model data against JSON Schema. + * + *

Uses networknt/json-schema-validator for schema validation. + * Collects all validation errors for better developer experience. + * + */ +public class SchemaValidator { + + private static final Logger logger = LoggerFactory.getLogger(SchemaValidator.class); + public static final String OSI_SCHEMA_PATH = "/schemas/osi-schema.json"; + public static final String SALESFORCE_SCHEMA_PATH = "/schemas/salesforce-semantic-model-schema.json"; + + private final JsonSchema schema; + private final ObjectMapper objectMapper; + private final String schemaPath; + + /** + * Creates a validator with a custom schema. + * + * @param objectMapper Jackson ObjectMapper for converting data to JsonNode + * @param schemaPath Path to the JSON schema file in classpath + */ + public SchemaValidator(ObjectMapper objectMapper, String schemaPath) { + this.objectMapper = objectMapper; + this.schemaPath = schemaPath; + this.schema = loadSchema(); + } + + /** + * Loads the JSON Schema from classpath. + */ + private JsonSchema loadSchema() { + try (InputStream schemaStream = getClass().getResourceAsStream(schemaPath)) { + if (schemaStream == null) { + throw new ValidationException("Schema file not found: " + schemaPath); + } + + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7); + JsonNode schemaNode = objectMapper.readTree(schemaStream); + + logger.info("Loaded schema from {}", schemaPath); + return factory.getSchema(schemaNode); + + } catch (Exception e) { + throw new ValidationException("Failed to load schema: " + e.getMessage(), e); + } + } + + /** + * Validates OSI semantic model data against the schema. + * + * @param data The semantic model data to validate + * @throws ValidationException if validation fails with details of all errors + */ + public void validate(Map data) { + try { + // Convert Map to JsonNode + JsonNode jsonNode = objectMapper.valueToTree(data); + + // Validate against schema + Set errors = schema.validate(jsonNode); + + if (!errors.isEmpty()) { + String errorMessage = formatValidationErrors(errors); + logger.error("Schema validation failed:\n{}", errorMessage); + throw new ValidationException("Schema validation failed:\n" + errorMessage); + } + + logger.debug("Schema validation passed"); + + } catch (ValidationException e) { + throw e; + } catch (Exception e) { + throw new ValidationException("Validation error: " + e.getMessage(), e); + } + } + + /** + * Formats validation errors into a readable message. + */ + private String formatValidationErrors(Set errors) { + return errors.stream() + .map(ValidationMessage::toString) + .collect(Collectors.joining("\n - ", " - ", "")); + } +} diff --git a/converters/salesforce/src/main/resources/mappings.yaml b/converters/salesforce/src/main/resources/mappings.yaml new file mode 100644 index 00000000..fddc085f --- /dev/null +++ b/converters/salesforce/src/main/resources/mappings.yaml @@ -0,0 +1,30 @@ +# ============================================================================= +# OSI-Salesforce Converter - Mapping Configuration +# ============================================================================= +# +# This file defines ONLY straightforward property mappings between OSI YAML +# format and Salesforce Semantic Model JSON format. +# +# Complex mappings (arrays, routing, transformations) are handled programmatically +# by special handlers and are NOT listed here. +# +# Format: +# osiPath: salesforcePath +# +# ============================================================================= + + + name: apiName + description: description + + datasets: semanticDataObjects + datasets.name: semanticDataObjects.apiName + datasets.source: semanticDataObjects.dataObjectName + datasets.description: semanticDataObjects.description + + relationships: semanticRelationships + relationships.name: semanticRelationships.apiName + + metrics: semanticCalculatedMeasurements + metrics.name: semanticCalculatedMeasurements.apiName + metrics.description: semanticCalculatedMeasurements.description diff --git a/converters/salesforce/src/main/resources/osi-salesforce-converter-config.yaml b/converters/salesforce/src/main/resources/osi-salesforce-converter-config.yaml new file mode 100644 index 00000000..fea5fea0 --- /dev/null +++ b/converters/salesforce/src/main/resources/osi-salesforce-converter-config.yaml @@ -0,0 +1,29 @@ +# Pipeline definitions - each direction lists handlers to execute in order + +pipelines: + osiToSalesforce: + - DatasetMappingHandler + - FieldMappingHandler + - RelationshipMappingHandler + - MetricMappingHandler + - SemanticModelMappingHandler + + salesforceToOsi: + - DatasetMappingHandler + - FieldMappingHandler + - RelationshipMappingHandler + - MetricMappingHandler + - SemanticModelMappingHandler + +# Direction-specific configuration +osiToSalesforce: + inputFormat: yaml + outputFormat: json + schemaPath: /schemas/osi-schema.json + extractModelNameFrom: apiName + +salesforceToOsi: + inputFormat: json + outputFormat: yaml + schemaPath: /schemas/salesforce-semantic-model-schema.json + extractModelNameFrom: name diff --git a/converters/salesforce/src/test/java/org/osi/OsiToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/osi/OsiToSalesforceConverterTest.java new file mode 100644 index 00000000..defdd367 --- /dev/null +++ b/converters/salesforce/src/test/java/org/osi/OsiToSalesforceConverterTest.java @@ -0,0 +1,363 @@ +package org.osi; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.osi.converter.Converter; +import org.osi.converter.ConverterFactory; +import org.osi.converter.ConversionDirection; +import org.osi.converter.CustomExtensionHandler; +import org.osi.validator.SchemaValidator; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.osi.converter.pipeline.DirectionConfig; +import org.osi.converter.pipeline.HandlerFactory; +import org.osi.converter.pipeline.PipelineConfig; +import org.osi.converter.pipeline.PipelineConfigLoader; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Comprehensive integration test for OSI → Salesforce conversion. + * Uses real example file: osiToSalesforce.yaml + */ +class OsiToSalesforceConverterTest { + + private static boolean salesforceSchemaExists; + private static boolean osiSchemaExists; + private static boolean warningPrinted = false; + + private Converter converter; + private ObjectMapper jsonMapper; + private String osiYaml; + private String osiYamlAnsiSql; + + @BeforeAll + static void checkSchemaAvailability() { + salesforceSchemaExists = OsiToSalesforceConverterTest.class + .getResourceAsStream(SchemaValidator.SALESFORCE_SCHEMA_PATH) != null; + osiSchemaExists = OsiToSalesforceConverterTest.class + .getResourceAsStream(SchemaValidator.OSI_SCHEMA_PATH) != null; + + if (!warningPrinted) { + if (!salesforceSchemaExists) { + System.err.println("\n WARNING: Salesforce schema not found at " + SchemaValidator.SALESFORCE_SCHEMA_PATH); + System.err.println(" Some tests in OsiToSalesforceConverterTest will be skipped."); + System.err.println(" To run all tests, download the schema from:"); + System.err.println(" https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html"); + System.err.println(" and save it to: src/main/resources/schemas/salesforce-semantic-model-schema.json\n"); + } + if (!osiSchemaExists) { + System.err.println("\n WARNING: OSI schema not found at " + SchemaValidator.OSI_SCHEMA_PATH); + System.err.println(" Skipping OsiToSalesforceConverterTest tests."); + System.err.println(" To run these tests, download the schema from:"); + System.err.println(" https://github.com/open-semantic-interchange/OSI/blob/main/core-spec/osi-schema.json"); + System.err.println(" and save it to: src/main/resources/schemas/osi-schema.json\n"); + } + warningPrinted = true; + } + } + + @BeforeEach + void setUp() throws IOException { + assumeTrue(osiSchemaExists, "OSI schema file is required but not found. See README for setup instructions."); + + converter = ConverterFactory.getConverter(ConversionDirection.OSI_TO_SALESFORCE); + jsonMapper = new ObjectMapper(); + osiYamlAnsiSql = Files.readString(Paths.get("src/test/resources/examples/osiToSalesforce.yaml")); + osiYaml = osiYamlAnsiSql; + } + + @Test + void testCompleteConversion() throws Exception { + List results = converter.convert(osiYaml); + + assertNotNull(results); + assertEquals(1, results.size()); + + String salesforceJson = results.get(0); + assertNotNull(salesforceJson); + assertTrue(salesforceJson.contains("\"apiName\" : \"Customer_Orders_Model\"")); + assertTrue(salesforceJson.contains("\"semanticDataObjects\"")); + + Map sfModel = jsonMapper.readValue(salesforceJson, new TypeReference>() {}); + assertNotNull(sfModel); + assertEquals("Customer_Orders_Model", sfModel.get("apiName")); + assertNotNull(sfModel.get("description")); + } + + @Test + void testDatasetMapping() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + List> dataObjects = (List>) sfModel.get("semanticDataObjects"); + assertNotNull(dataObjects); + assertEquals(3, dataObjects.size()); + + assertEquals("Customers", dataObjects.get(0).get("apiName")); + assertEquals("Orders", dataObjects.get(1).get("apiName")); + assertEquals("Products", dataObjects.get(2).get("apiName")); + + assertEquals("Customers__dll", dataObjects.get(0).get("dataObjectName")); + assertEquals("Orders__dll", dataObjects.get(1).get("dataObjectName")); + assertEquals("Products__dll", dataObjects.get(2).get("dataObjectName")); + + assertEquals("Customer master data", dataObjects.get(0).get("description")); + assertEquals("Order transaction data", dataObjects.get(1).get("description")); + assertEquals("Product catalog data", dataObjects.get(2).get("description")); + } + + @Test + void testFieldSplittingIntoDimensionsAndMeasurements() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + List> dataObjects = (List>) sfModel.get("semanticDataObjects"); + Map customersDataset = dataObjects.get(0); + + List> customerDimensions = (List>) customersDataset.get("semanticDimensions"); + List> customerMeasurements = (List>) customersDataset.get("semanticMeasurements"); + + assertNotNull(customerDimensions); + assertNotNull(customerMeasurements); + + boolean hasCustomerId = customerDimensions.stream() + .anyMatch(d -> "customer_id".equals(d.get("apiName"))); + assertTrue(hasCustomerId, "customer_id should be a dimension"); + + boolean hasEmail = customerDimensions.stream() + .anyMatch(d -> "email".equals(d.get("apiName"))); + assertTrue(hasEmail, "email should be a dimension"); + + boolean hasTotalPurchases = customerMeasurements.stream() + .anyMatch(m -> "total_purchases".equals(m.get("apiName"))); + assertTrue(hasTotalPurchases, "total_purchases should be a measurement"); + + Map ordersDataset = dataObjects.get(1); + List> orderMeasurements = (List>) ordersDataset.get("semanticMeasurements"); + + boolean hasAmount = orderMeasurements.stream() + .anyMatch(m -> "amount".equals(m.get("apiName"))); + assertTrue(hasAmount, "amount should be a measurement"); + } + + @Test + void testExpressionUnwrappingFromDialects() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + List> dataObjects = (List>) sfModel.get("semanticDataObjects"); + Map customersDataset = dataObjects.get(0); + List> customerDimensions = (List>) customersDataset.get("semanticDimensions"); + + Map customerIdDim = customerDimensions.stream() + .filter(d -> "customer_id".equals(d.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(customerIdDim); + assertEquals("customer_id__c", customerIdDim.get("dataObjectFieldName")); + + Map emailDim = customerDimensions.stream() + .filter(d -> "email".equals(d.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(emailDim); + assertEquals("email__c", emailDim.get("dataObjectFieldName")); + } + + @Test + void testRelationshipConversion() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + List> relationships = (List>) sfModel.get("semanticRelationships"); + assertNotNull(relationships); + assertTrue(relationships.size() >= 2, "Should have at least 2 valid relationships"); + + Map customersOrdersRel = relationships.stream() + .filter(r -> "Customers_Orders".equals(r.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(customersOrdersRel); + assertEquals("Customers", customersOrdersRel.get("leftSemanticDefinitionApiName")); + assertEquals("Orders", customersOrdersRel.get("rightSemanticDefinitionApiName")); + assertEquals("Customers to Orders", customersOrdersRel.get("label")); + assertEquals("OneToMany", customersOrdersRel.get("cardinality")); + + List> criteria = (List>) customersOrdersRel.get("criteria"); + assertNotNull(criteria); + assertEquals(1, criteria.size()); + assertEquals("customer_id", criteria.get(0).get("leftSemanticFieldApiName")); + assertEquals("customer_id", criteria.get(0).get("rightSemanticFieldApiName")); + } + + @Test + void testCalculatedFieldDetection() throws Exception { + List ansiResults = converter.convert(osiYamlAnsiSql); + Map ansiModel = jsonMapper.readValue(ansiResults.get(0), new TypeReference>() {}); + + List> ansiCalcDimensions = (List>) ansiModel.get("semanticCalculatedDimensions"); + assertNull(ansiCalcDimensions, "ANSI_SQL dialect: no semanticCalculatedDimensions"); + } + + @Test + void testInvalidRelationshipsFiltered() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + List> relationships = (List>) sfModel.get("semanticRelationships"); + assertNotNull(relationships); + assertEquals(2, relationships.size(), "Only 2 relationships should be present"); + + boolean hasValidCustomersOrders = relationships.stream() + .anyMatch(r -> "Customers_Orders".equals(r.get("apiName"))); + assertTrue(hasValidCustomersOrders, "Customers_Orders should be included"); + + boolean hasValidOrdersProducts = relationships.stream() + .anyMatch(r -> "Orders_Products".equals(r.get("apiName"))); + assertTrue(hasValidOrdersProducts, "Orders_Products should be included"); + } + + @Test + void testCustomExtensionsRestoration() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + assertEquals("Customer_Orders_Model", sfModel.get("label")); + assertEquals("default", sfModel.get("dataspace")); + + List> dataObjects = (List>) sfModel.get("semanticDataObjects"); + Map customersDataset = dataObjects.get(0); + assertEquals("Customers", customersDataset.get("label")); + assertEquals("Dlo", customersDataset.get("dataObjectType")); + + List> customerDimensions = (List>) customersDataset.get("semanticDimensions"); + Map customerIdDim = customerDimensions.stream() + .filter(d -> "customer_id".equals(d.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(customerIdDim); + assertEquals("Text", customerIdDim.get("dataType")); + assertEquals("Discrete", customerIdDim.get("displayCategory")); + } + + @Test + void testMetricsNotConvertedInOsiToSalesforce() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + List> calcMeasurements = (List>) sfModel.get("semanticCalculatedMeasurements"); + assertNull(calcMeasurements, "Metrics from OSI are not converted to semanticCalculatedMeasurements in OSI->SF direction"); + } + + @Test + void testTimeDimensionConversion() throws Exception { + List results = converter.convert(osiYaml); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + + List> dataObjects = (List>) sfModel.get("semanticDataObjects"); + Map ordersDataset = dataObjects.get(1); + List> orderDimensions = (List>) ordersDataset.get("semanticDimensions"); + + Map orderDateDim = orderDimensions.stream() + .filter(d -> "order_date".equals(d.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(orderDateDim); + assertEquals("Date", orderDateDim.get("dataType")); + assertEquals("Discrete", orderDateDim.get("displayCategory")); + } + + @Test + void testOutputCompilesWithSalesforceSchema() throws Exception { + assumeTrue(salesforceSchemaExists, "Salesforce schema file is required but not found. See README for setup instructions."); + + List results = converter.convert(osiYaml); + String salesforceJson = results.get(0); + + Map sfModel = jsonMapper.readValue(salesforceJson, new TypeReference>() {}); + + SchemaValidator validator = new SchemaValidator(jsonMapper, SchemaValidator.SALESFORCE_SCHEMA_PATH); + assertDoesNotThrow(() -> validator.validate(sfModel), "Output should comply with Salesforce schema"); + } + + @Test + void testPipelineConfigLoadsSuccessfully() { + // Verify pipeline configuration is loaded correctly + PipelineConfig config = + PipelineConfigLoader.loadFromResource(); + + assertNotNull(config); + assertNotNull(config.getPipelines()); + assertTrue(config.getPipelines().containsKey("osiToSalesforce")); + assertTrue(config.getPipelines().containsKey("salesforceToOsi")); + + // Verify handler list for osiToSalesforce + List handlers = config.getPipelines().get("osiToSalesforce"); + assertNotNull(handlers); + assertEquals(5, handlers.size()); + assertTrue(handlers.contains("DatasetMappingHandler")); + assertTrue(handlers.contains("FieldMappingHandler")); + assertTrue(handlers.contains("RelationshipMappingHandler")); + assertTrue(handlers.contains("MetricMappingHandler")); + assertTrue(handlers.contains("SemanticModelMappingHandler")); + + // Verify direction config + assertNotNull(config.getDirectionConfigs()); + DirectionConfig dirConfig = + config.getDirectionConfigs().get("osiToSalesforce"); + assertNotNull(dirConfig); + assertEquals("yaml", dirConfig.getInputFormat()); + assertEquals("json", dirConfig.getOutputFormat()); + assertEquals("/schemas/osi-schema.json", dirConfig.getSchemaPath()); + assertEquals("apiName", dirConfig.getExtractModelNameFrom()); + } + + @Test + void testHandlerFactoryCreatesAllHandlers() { + // Verify HandlerFactory can create all configured handlers + CustomExtensionHandler customExtensionHandler = + new CustomExtensionHandler(jsonMapper); + HandlerFactory factory = + new HandlerFactory(customExtensionHandler); + + ConversionDirection direction = ConversionDirection.OSI_TO_SALESFORCE; + + assertDoesNotThrow(() -> factory.createHandler("DatasetMappingHandler", direction)); + assertDoesNotThrow(() -> factory.createHandler("FieldMappingHandler", direction)); + assertDoesNotThrow(() -> factory.createHandler("RelationshipMappingHandler", direction)); + assertDoesNotThrow(() -> factory.createHandler("MetricMappingHandler", direction)); + assertDoesNotThrow(() -> factory.createHandler("SemanticModelMappingHandler", direction)); + } + + @Test + void testDirectionConfigFileExtension() { + // Test that getFileExtension() correctly derives from outputFormat + DirectionConfig jsonConfig = + new DirectionConfig(); + jsonConfig.setOutputFormat("json"); + assertEquals(".json", jsonConfig.getFileExtension()); + + DirectionConfig yamlConfig = + new DirectionConfig(); + yamlConfig.setOutputFormat("yaml"); + assertEquals(".yaml", yamlConfig.getFileExtension()); + } + + @Test + void testPipelineConfigLoaderConstructor() { + // Test that PipelineConfigLoader can be instantiated + PipelineConfigLoader loader = + new PipelineConfigLoader(); + assertNotNull(loader); + } + +} diff --git a/converters/salesforce/src/test/java/org/osi/SalesforceToOsiConverterTest.java b/converters/salesforce/src/test/java/org/osi/SalesforceToOsiConverterTest.java new file mode 100644 index 00000000..dd7c3361 --- /dev/null +++ b/converters/salesforce/src/test/java/org/osi/SalesforceToOsiConverterTest.java @@ -0,0 +1,385 @@ +package org.osi; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.osi.converter.Converter; +import org.osi.converter.ConverterFactory; +import org.osi.converter.ConversionDirection; +import org.osi.converter.CustomExtensionHandler; +import org.osi.validator.SchemaValidator; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.osi.converter.pipeline.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Comprehensive integration test for Salesforce → OSI conversion. + * Uses real example file: salesforceToOsi.json + */ +class SalesforceToOsiConverterTest { + + private static boolean schemaExists; + private static boolean osiSchemaExists; + private static boolean warningPrinted = false; + + private Converter converter; + private ObjectMapper yamlMapper; + private String salesforceJson; + + @BeforeAll + static void checkSchemaAvailability() { + schemaExists = SalesforceToOsiConverterTest.class + .getResourceAsStream(SchemaValidator.SALESFORCE_SCHEMA_PATH) != null; + osiSchemaExists = SalesforceToOsiConverterTest.class + .getResourceAsStream(SchemaValidator.OSI_SCHEMA_PATH) != null; + + if (!warningPrinted) { + if (!schemaExists) { + System.err.println("\n WARNING: Salesforce schema not found at " + SchemaValidator.SALESFORCE_SCHEMA_PATH); + System.err.println(" Skipping SalesforceToOsiConverterTest tests."); + System.err.println(" To run these tests, download the schema from:"); + System.err.println(" https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html"); + System.err.println(" and save it to: src/main/resources/schemas/salesforce-semantic-model-schema.json\n"); + } + if (!osiSchemaExists) { + System.err.println("\n WARNING: OSI schema not found at " + SchemaValidator.OSI_SCHEMA_PATH); + System.err.println(" Skipping SalesforceToOsiConverterTest tests."); + System.err.println(" To run these tests, download the schema from:"); + System.err.println(" https://github.com/open-semantic-interchange/OSI/blob/main/core-spec/osi-schema.json"); + System.err.println(" and save it to: src/main/resources/schemas/osi-schema.json\n"); + } + warningPrinted = true; + } + } + + @BeforeEach + void setUp() throws IOException { + assumeTrue(schemaExists, "Salesforce schema file is required but not found. See README for setup instructions."); + assumeTrue(osiSchemaExists, "OSI schema file is required but not found. See README for setup instructions."); + + converter = ConverterFactory.getConverter(ConversionDirection.SALESFORCE_TO_OSI); + yamlMapper = new ObjectMapper(new YAMLFactory()); + salesforceJson = Files.readString(Paths.get("src/test/resources/examples/salesforceToOsi.json")); + } + + @Test + void testCompleteConversion() throws Exception { + List results = converter.convert(salesforceJson); + + assertNotNull(results); + assertEquals(1, results.size()); + + String osiYaml = results.get(0); + assertNotNull(osiYaml); + assertTrue(osiYaml.contains("version: 0.1.1")); + assertTrue(osiYaml.contains("semantic_model:")); + + Map osiRoot = yamlMapper.readValue(osiYaml, Map.class); + assertNotNull(osiRoot); + assertEquals("0.1.1", osiRoot.get("version")); + + List> semanticModels = (List>) osiRoot.get("semantic_model"); + assertNotNull(semanticModels); + assertEquals(1, semanticModels.size()); + + Map model = semanticModels.get(0); + assertEquals("Customer_Orders_Model", model.get("name")); + assertNotNull(model.get("description")); + } + + @Test + void testDatasetMapping() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + + List> datasets = (List>) model.get("datasets"); + assertNotNull(datasets); + assertEquals(3, datasets.size()); + + // Verify dataset names + assertEquals("Customers", datasets.get(0).get("name")); + assertEquals("Orders", datasets.get(1).get("name")); + assertEquals("Products", datasets.get(2).get("name")); + + // Verify source mapping + assertEquals("Customers__dll", datasets.get(0).get("source")); + assertEquals("Orders__dll", datasets.get(1).get("source")); + assertEquals("Products__dll", datasets.get(2).get("source")); + } + + @Test + void testFieldMapping() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + List> datasets = (List>) model.get("datasets"); + + Map customersDataset = datasets.get(0); + List> fields = (List>) customersDataset.get("fields"); + assertNotNull(fields); + assertTrue(fields.size() >= 4); // customer_id, email, total_purchases, lifetime_value + + // Check field structure + Map customerIdField = fields.stream() + .filter(f -> "customer_id".equals(f.get("name"))) + .findFirst() + .orElse(null); + assertNotNull(customerIdField); + assertEquals("Customer ID", customerIdField.get("label")); + assertNotNull(customerIdField.get("dimension")); + assertNotNull(customerIdField.get("expression")); + + // Verify expression dialect structure + Map expression = (Map) customerIdField.get("expression"); + List> dialects = (List>) expression.get("dialects"); + assertNotNull(dialects); + assertEquals(1, dialects.size()); + assertEquals("TABLEAU", dialects.get(0).get("dialect")); + assertEquals("customer_id__c", dialects.get(0).get("expression")); + } + + @Test + void testCalculatedDimensionConversion() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + List> datasets = (List>) model.get("datasets"); + + // customer_email_domain should be converted to a field in Customers dataset (single dependency) + Map customersDataset = datasets.get(0); + List> fields = (List>) customersDataset.get("fields"); + boolean hasEmailDomain = fields.stream() + .anyMatch(f -> "customer_email_domain".equals(f.get("name"))); + assertTrue(hasEmailDomain, "customer_email_domain should be converted to a field"); + + // order_year should be converted to a field in Orders dataset (single dependency) + Map ordersDataset = datasets.get(1); + List> orderFields = (List>) ordersDataset.get("fields"); + boolean hasOrderYear = orderFields.stream() + .anyMatch(f -> "order_year".equals(f.get("name"))); + assertTrue(hasOrderYear, "order_year should be converted to a field"); + + // customer_order_key should remain in custom_extensions (multiple dependencies) + List> customExtensions = (List>) model.get("custom_extensions"); + assertNotNull(customExtensions); + Map sfExtension = customExtensions.stream() + .filter(ext -> "SALESFORCE".equals(ext.get("vendor_name"))) + .findFirst() + .orElse(null); + assertNotNull(sfExtension); + String data = (String) sfExtension.get("data"); + assertTrue(data.contains("customer_order_key"), "customer_order_key should be in custom_extensions"); + } + + @Test + void testRelationshipMapping() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + + List> relationships = (List>) model.get("relationships"); + assertNotNull(relationships); + assertEquals(2, relationships.size()); + + // Verify first relationship + Map rel1 = relationships.get(0); + assertEquals("Customers_Orders_TableField", rel1.get("name")); + assertEquals("Customers", rel1.get("from")); + assertEquals("Orders", rel1.get("to")); + + List fromColumns = (List) rel1.get("from_columns"); + List toColumns = (List) rel1.get("to_columns"); + assertNotNull(fromColumns); + assertNotNull(toColumns); + assertEquals(1, fromColumns.size()); + assertEquals(1, toColumns.size()); + assertEquals("customer_id", fromColumns.get(0)); + assertEquals("order_id", toColumns.get(0)); + } + + @Test + void testUnsupportedRelationshipsInCustomExtensions() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + + // Unsupported relationships (Formula/SemanticField) should be in custom_extensions + List> customExtensions = (List>) model.get("custom_extensions"); + assertNotNull(customExtensions); + + Map sfExtension = customExtensions.stream() + .filter(ext -> "SALESFORCE".equals(ext.get("vendor_name"))) + .findFirst() + .orElse(null); + assertNotNull(sfExtension); + + String data = (String) sfExtension.get("data"); + assertTrue(data.contains("semanticRelationships")); + assertTrue(data.contains("Customers_Products_Formula")); + } + + @Test + void testMetricMapping() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + + List> metrics = (List>) model.get("metrics"); + assertNotNull(metrics); + assertEquals(2, metrics.size()); + + // Verify first metric + Map metric1 = metrics.get(0); + assertEquals("total_revenue", metric1.get("name")); + assertNotNull(metric1.get("expression")); + + Map expression = (Map) metric1.get("expression"); + List> dialects = (List>) expression.get("dialects"); + assertNotNull(dialects); + assertEquals("TABLEAU", dialects.get(0).get("dialect")); + assertEquals("SUM([Orders].[amount])", dialects.get(0).get("expression")); + } + + @Test + void testCustomExtensionsPreservation() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + + // Check model-level custom_extensions + List> customExtensions = (List>) model.get("custom_extensions"); + assertNotNull(customExtensions); + assertTrue(customExtensions.size() > 0); + + Map sfExtension = customExtensions.stream() + .filter(ext -> "SALESFORCE".equals(ext.get("vendor_name"))) + .findFirst() + .orElse(null); + assertNotNull(sfExtension); + assertNotNull(sfExtension.get("data")); + + // Verify Salesforce-specific properties are preserved + String data = (String) sfExtension.get("data"); + assertTrue(data.contains("label")); + assertTrue(data.contains("dataspace")); + } + + @Test + void testTimeDimensionMapping() throws Exception { + List results = converter.convert(salesforceJson); + Map osiRoot = yamlMapper.readValue(results.get(0), Map.class); + Map model = ((List>) osiRoot.get("semantic_model")).get(0); + List> datasets = (List>) model.get("datasets"); + + Map ordersDataset = datasets.get(1); + List> fields = (List>) ordersDataset.get("fields"); + + Map orderDateField = fields.stream() + .filter(f -> "order_date".equals(f.get("name"))) + .findFirst() + .orElse(null); + assertNotNull(orderDateField); + + Map dimension = (Map) orderDateField.get("dimension"); + assertNotNull(dimension); + assertTrue((Boolean) dimension.get("is_time")); + } + + @Test + void testOutputCompilesWithOsiSchema() throws Exception { + List results = converter.convert(salesforceJson); + String osiYaml = results.get(0); + + Map osiRoot = yamlMapper.readValue(osiYaml, Map.class); + + SchemaValidator validator = new SchemaValidator(new ObjectMapper(), SchemaValidator.OSI_SCHEMA_PATH); + assertDoesNotThrow(() -> validator.validate(osiRoot), "Output should comply with OSI schema"); + } + + @Test + void testPipelineConfigForSalesforceToOsi() { + // Verify pipeline configuration for salesforceToOsi direction + PipelineConfig config = + PipelineConfigLoader.loadFromResource(); + + // Verify handler list for salesforceToOsi + List handlers = config.getPipelines().get("salesforceToOsi"); + assertNotNull(handlers); + assertEquals(5, handlers.size()); + assertTrue(handlers.contains("DatasetMappingHandler")); + assertTrue(handlers.contains("FieldMappingHandler")); + assertTrue(handlers.contains("RelationshipMappingHandler")); + assertTrue(handlers.contains("MetricMappingHandler")); + assertTrue(handlers.contains("SemanticModelMappingHandler")); + + // Verify direction config for salesforceToOsi + DirectionConfig dirConfig = + config.getDirectionConfigs().get("salesforceToOsi"); + assertNotNull(dirConfig); + assertEquals("json", dirConfig.getInputFormat()); + assertEquals("yaml", dirConfig.getOutputFormat()); + assertEquals("/schemas/salesforce-semantic-model-schema.json", dirConfig.getSchemaPath()); + assertEquals("name", dirConfig.getExtractModelNameFrom()); + assertEquals(".yaml", dirConfig.getFileExtension()); + } + + @Test + void testHandlerFactoryForSalesforceToOsi() { + // Verify HandlerFactory creates handlers for salesforceToOsi direction + ObjectMapper testJsonMapper = new ObjectMapper(); + CustomExtensionHandler customExtensionHandler = + new CustomExtensionHandler(testJsonMapper); + HandlerFactory factory = + new HandlerFactory(customExtensionHandler); + + ConversionDirection direction = ConversionDirection.SALESFORCE_TO_OSI; + + // Verify all handlers can be instantiated + PipelineStep datasetHandler = + factory.createHandler("DatasetMappingHandler", direction); + assertNotNull(datasetHandler); + + PipelineStep fieldHandler = + factory.createHandler("FieldMappingHandler", direction); + assertNotNull(fieldHandler); + + PipelineStep relationshipHandler = + factory.createHandler("RelationshipMappingHandler", direction); + assertNotNull(relationshipHandler); + + PipelineStep metricHandler = + factory.createHandler("MetricMappingHandler", direction); + assertNotNull(metricHandler); + + PipelineStep semanticModelHandler = + factory.createHandler("SemanticModelMappingHandler", direction); + assertNotNull(semanticModelHandler); + } + + @Test + void testConverterImplExtractModelNameFromOsiFormat() throws Exception { + // Test extractModelName specifically handles OSI wrapped format + List results = converter.convert(salesforceJson); + String osiYaml = results.get(0); + + // The result is wrapped OSI format - extractModelName should handle this + Map osiRoot = yamlMapper.readValue(osiYaml, Map.class); + assertTrue(osiRoot.containsKey("semantic_model")); + + // Verify it's properly wrapped + List> models = (List>) osiRoot.get("semantic_model"); + assertNotNull(models); + assertEquals(1, models.size()); + assertEquals("Customer_Orders_Model", models.get(0).get("name")); + } +} diff --git a/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml b/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml new file mode 100644 index 00000000..ff541475 --- /dev/null +++ b/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml @@ -0,0 +1,368 @@ +version: 0.1.1 +semantic_model: + - name: Customer_Orders_Model + description: Example model demonstrating calculated dimensions with single and multiple + data object dependencies + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataspace" : "default" + } + datasets: + - description: Customer master data + name: Customers + source: Customers__dll + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Customers", + "dataObjectType" : "Dlo" + } + fields: + - name: customer_id + label: Customer ID + description: Unique customer identifier + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete" + } + - name: email + label: Email + description: Customer email address + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: email__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Email" + } + - name: total_purchases + label: Total Purchases + description: Lifetime purchase count + expression: + dialects: + - dialect: ANSI_SQL + expression: total_purchases__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "aggregationType" : "Sum", + "displayCategory" : "Continuous" + } + - name: lifetime_value + label: Lifetime Value + description: Total customer value + expression: + dialects: + - dialect: ANSI_SQL + expression: lifetime_value__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Currency", + "aggregationType" : "Sum" + } + - name: customer_email_domain + label: Customer Email Domain + description: Customer Email Domain + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: SUBSTRING([Customers].[email], POSITION('@' IN [Customers].[email]) + + 1, LENGTH([Customers].[email])) + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text" + } + - description: Order transaction data + name: Orders + source: Orders__dll + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Orders", + "dataObjectType" : "Dlo" + } + fields: + - name: order_id + label: Order ID + description: Unique order identifier + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text" + } + - name: customer_id + label: Customer ID + description: Foreign key to Customers + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: product_id + label: Product ID + description: Foreign key to Products + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: product_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: order_date + label: Order Date + description: Date when order was placed + dimension: + is_time: true + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Date", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Descending" + } + - name: amount + label: Amount + description: Order amount in currency + expression: + dialects: + - dialect: ANSI_SQL + expression: amount__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Currency", + "aggregationType" : "Sum" + } + - name: quantity + label: Quantity + description: Number of items ordered + expression: + dialects: + - dialect: ANSI_SQL + expression: quantity__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "aggregationType" : "Sum" + } + - name: order_year + label: Order Year + description: Order Year + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: YEAR([Orders].[order_date]) + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "displayCategory" : "Discrete" + } + - description: Product catalog data + name: Products + source: Products__dll + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Products", + "dataObjectType" : "Dlo" + } + fields: + - name: product_id + label: Product ID + description: Unique product identifier + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: product_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: product_name + label: Product Name + description: Product display name + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: product_name__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: unit_price + label: Unit Price + description: Price per unit + expression: + dialects: + - dialect: ANSI_SQL + expression: unit_price__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Currency" + } + - name: stock_level + label: Stock Level + description: Current inventory count + expression: + dialects: + - dialect: ANSI_SQL + expression: stock_level__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "aggregationType" : "Sum", + "displayCategory" : "Continuous", + "isVisible" : true, + "isAggregatable" : true, + "shouldTreatNullsAsZeros" : false, + "decimalPlace" : 0 + } + relationships: + - name: Customers_Orders + from: Customers + to: Orders + from_columns: + - customer_id + to_columns: + - customer_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Customers to Orders", + "cardinality" : "OneToMany", + "joinType" : "Auto", + "isEnabled" : true + } + - name: Orders_Products + from: Orders + to: Products + from_columns: + - product_id + to_columns: + - product_id + - name: Customers_ByDomain + from: Customers + to: Orders + from_columns: + - customer_email_domain + to_columns: + - order_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Invalid Relationship - Uses Calculated Field" + } + - name: Orders_ByYear + from: Orders + to: Products + from_columns: + - order_year + to_columns: + - product_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Invalid Relationship - Uses Calculated Field", + "cardinality" : "ManyToMany", + "joinType" : "Auto", + "isEnabled" : true + } + metrics: + - description: Sum of all order amounts + name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM([Orders].[amount]) + - description: Average amount per order + name: avg_order_value + expression: + dialects: + - dialect: ANSI_SQL + expression: AVG([Orders].[amount]) diff --git a/converters/salesforce/src/test/resources/examples/salesforceToOsi.json b/converters/salesforce/src/test/resources/examples/salesforceToOsi.json new file mode 100644 index 00000000..c7d6e243 --- /dev/null +++ b/converters/salesforce/src/test/resources/examples/salesforceToOsi.json @@ -0,0 +1,354 @@ +{ + "apiName": "Customer_Orders_Model", + "label": "Customer Orders Model", + "description": "Example model demonstrating calculated dimensions with single and multiple data object dependencies", + "dataspace": "default", + "isQueryable": "Queryable", + "semanticDataObjects": [ + { + "apiName": "Customers", + "label": "Customers", + "description": "Customer master data", + "dataObjectName": "Customers__dll", + "dataObjectType": "Dlo", + "semanticDimensions": [ + { + "apiName": "customer_id", + "label": "Customer ID", + "description": "Unique customer identifier", + "dataObjectFieldName": "customer_id__c", + "dataType": "Text", + "displayCategory": "Discrete", + "isVisible": true, + "isPrimaryKey": true, + "sortOrder": "Ascending", + "semanticDataType": "Identifier" + }, + { + "apiName": "email", + "label": "Email", + "description": "Customer email address", + "dataObjectFieldName": "email__c", + "dataType": "Email", + "displayCategory": "Discrete", + "isVisible": true, + "sortOrder": "None" + } + ], + "semanticMeasurements": [ + { + "apiName": "total_purchases", + "label": "Total Purchases", + "description": "Lifetime purchase count", + "dataObjectFieldName": "total_purchases__c", + "dataType": "Number", + "aggregationType": "Sum", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": true, + "directionality": "Up", + "sentiment": "SentimentTypeUpIsGood", + "shouldTreatNullsAsZeros": true, + "decimalPlace": 0 + }, + { + "apiName": "lifetime_value", + "label": "Lifetime Value", + "description": "Total customer value", + "dataObjectFieldName": "lifetime_value__c", + "dataType": "Currency", + "aggregationType": "Sum", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": true, + "directionality": "Up", + "sentiment": "SentimentTypeUpIsGood", + "decimalPlace": 2 + } + ] + }, + { + "apiName": "Orders", + "label": "Orders", + "description": "Order transaction data", + "dataObjectName": "Orders__dll", + "dataObjectType": "Dlo", + "semanticDimensions": [ + { + "apiName": "order_id", + "label": "Order ID", + "description": "Unique order identifier", + "dataObjectFieldName": "order_id__c", + "dataType": "Text", + "displayCategory": "Discrete", + "isVisible": true, + "isPrimaryKey": true, + "sortOrder": "Ascending" + }, + { + "apiName": "order_date", + "label": "Order Date", + "description": "Date when order was placed", + "dataObjectFieldName": "order_date__c", + "dataType": "Date", + "displayCategory": "Discrete", + "isVisible": true, + "sortOrder": "Descending" + } + ], + "semanticMeasurements": [ + { + "apiName": "amount", + "label": "Amount", + "description": "Order amount in currency", + "dataObjectFieldName": "amount__c", + "dataType": "Currency", + "aggregationType": "Sum", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": true, + "directionality": "Up", + "sentiment": "SentimentTypeUpIsGood", + "decimalPlace": 2 + }, + { + "apiName": "quantity", + "label": "Quantity", + "description": "Number of items ordered", + "dataObjectFieldName": "quantity__c", + "dataType": "Number", + "aggregationType": "Sum", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": true, + "directionality": "Up", + "shouldTreatNullsAsZeros": true, + "decimalPlace": 0 + } + ] + }, + { + "apiName": "Products", + "label": "Products", + "description": "Product catalog data", + "dataObjectName": "Products__dll", + "dataObjectType": "Dlo", + "semanticDimensions": [ + { + "apiName": "product_id", + "label": "Product ID", + "description": "Unique product identifier", + "dataObjectFieldName": "product_id__c", + "dataType": "Text", + "displayCategory": "Discrete", + "isVisible": true, + "isPrimaryKey": true, + "sortOrder": "Ascending" + }, + { + "apiName": "product_name", + "label": "Product Name", + "description": "Product display name", + "dataObjectFieldName": "product_name__c", + "dataType": "Text", + "displayCategory": "Discrete", + "isVisible": true, + "sortOrder": "Ascending" + } + ], + "semanticMeasurements": [ + { + "apiName": "unit_price", + "label": "Unit Price", + "description": "Price per unit", + "dataObjectFieldName": "unit_price__c", + "dataType": "Currency", + "aggregationType": "Avg", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": true, + "decimalPlace": 2 + }, + { + "apiName": "stock_level", + "label": "Stock Level", + "description": "Current inventory count", + "dataObjectFieldName": "stock_level__c", + "dataType": "Number", + "aggregationType": "Sum", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": true, + "shouldTreatNullsAsZeros": false, + "decimalPlace": 0 + } + ] + } + ], + "semanticRelationships": [ + { + "apiName": "Customers_Orders_TableField", + "label": "Customers to Orders", + "description": "Test relationship with TableField types only", + "leftSemanticDefinitionApiName": "Customers", + "rightSemanticDefinitionApiName": "Orders", + "cardinality": "OneToMany", + "joinType": "Auto", + "isEnabled": true, + "criteria": [ + { + "leftSemanticFieldApiName": "customer_id", + "rightSemanticFieldApiName": "order_id", + "joinOperator": "Equals", + "leftFieldType": "TableField", + "rightFieldType": "TableField" + } + ] + }, + { + "apiName": "Customers_Orders_ConvertedCalcField", + "label": "Customers to Orders", + "description": "Test relationship with customer_email_domain", + "leftSemanticDefinitionApiName": "Customers", + "rightSemanticDefinitionApiName": "Orders", + "cardinality": "OneToMany", + "joinType": "Auto", + "isEnabled": true, + "criteria": [ + { + "leftSemanticFieldApiName": "customer_email_domain", + "rightSemanticFieldApiName": "order_id", + "joinOperator": "Equals", + "leftFieldType": "SemanticField", + "rightFieldType": "TableField" + } + ] + }, + { + "apiName": "Customers_Products_Formula", + "label": "Customers to Products - Formula Test", + "description": "Test relationship with Formula type", + "leftSemanticDefinitionApiName": "Customers", + "rightSemanticDefinitionApiName": "Products", + "cardinality": "ManyToMany", + "joinType": "Auto", + "isEnabled": true, + "criteria": [ + { + "leftSemanticFieldApiName": "customer_id", + "rightSemanticFieldApiName": "product_id", + "joinOperator": "Equals", + "leftFieldType": "Formula", + "rightFieldType": "TableField" + } + ] + } + ], + "semanticCalculatedDimensions": [ + { + "apiName": "customer_email_domain", + "label": "Customer Email Domain", + "description": "Extract domain from email", + "expression": "SUBSTRING([Customers].[email], CHARINDEX('@', [Customers].[email]) + 1, LEN([Customers].[email]))", + "dataType": "Text", + "displayCategory": "Discrete", + "isVisible": true, + "sortOrder": "Ascending", + "dependencies": [ + { + "dependentDefinitionApiName": "Customers", + "dependentFieldApiName": "email" + } + ] + }, + { + "apiName": "order_year", + "label": "Order Year", + "description": "Year extracted from order date", + "expression": "YEAR([Orders].[order_date])", + "dataType": "Number", + "displayCategory": "Discrete", + "isVisible": true, + "sortOrder": "Descending", + "dependencies": [ + { + "dependentDefinitionApiName": "Orders", + "dependentFieldApiName": "order_date" + } + ] + }, + { + "apiName": "customer_order_key", + "label": "Customer Order Key", + "description": "Unique key combining customer and order", + "expression": "CONCAT([Customers].[customer_id], '_', [Orders].[order_id])", + "dataType": "Text", + "displayCategory": "Discrete", + "isVisible": true, + "dependencies": [ + { + "dependentDefinitionApiName": "Customers", + "dependentFieldApiName": "customer_id" + }, + { + "dependentDefinitionApiName": "Orders", + "dependentFieldApiName": "order_id" + } + ] + }, + { + "apiName": "current_timestamp", + "label": "Current Timestamp", + "description": "Current date/time", + "expression": "CURRENT_TIMESTAMP()", + "dataType": "DateTime", + "displayCategory": "Discrete", + "isVisible": true, + "dependencies": [] + } + ], + "semanticCalculatedMeasurements": [ + { + "apiName": "total_revenue", + "label": "Total Revenue", + "description": "Sum of all order amounts", + "expression": "SUM([Orders].[amount])", + "dataType": "Currency", + "aggregationType": "Sum", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": true, + "directionality": "Up", + "sentiment": "SentimentTypeUpIsGood", + "decimalPlace": 2, + "level": "AggregateFunction", + "dependencies": [ + { + "dependentDefinitionApiName": "Orders", + "dependentFieldApiName": "amount" + } + ] + }, + { + "apiName": "avg_order_value", + "label": "Average Order Value", + "description": "Average amount per order", + "expression": "AVG([Orders].[amount])", + "dataType": "Currency", + "aggregationType": "Avg", + "displayCategory": "Continuous", + "isVisible": true, + "isAggregatable": false, + "directionality": "Up", + "sentiment": "SentimentTypeUpIsGood", + "decimalPlace": 2, + "level": "AggregateFunction", + "dependencies": [ + { + "dependentDefinitionApiName": "Orders", + "dependentFieldApiName": "amount" + } + ] + } + ] +}