Supports environment variable resolution in values using {@code ${ENV_VAR}} syntax.
+ */ +public final class AgentEvalConfigLoader { + + private static final Logger LOG = LoggerFactory.getLogger(AgentEvalConfigLoader.class); + private static final Pattern ENV_VAR = Pattern.compile("\\$\\{(\\w+)}"); + private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private AgentEvalConfigLoader() {} + + /** + * Loads configuration from the given YAML file path. + * + * @param path the path to agenteval.yaml + * @return the populated config builder (call {@code .build()} to finalize) + */ + public static AgentEvalConfig.Builder load(Path path) { + LOG.debug("Loading AgentEval config from {}", path); + try { + String content = Files.readString(path); + return parse(content); + } catch (IOException e) { + throw new ConfigException("Failed to load config from " + path, e); + } + } + + /** + * Loads configuration from an input stream. + */ + public static AgentEvalConfig.Builder load(InputStream inputStream) { + LOG.debug("Loading AgentEval config from input stream"); + try { + String content = new String(inputStream.readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + return parse(content); + } catch (IOException e) { + throw new ConfigException("Failed to load config from input stream", e); + } + } + + static AgentEvalConfig.Builder parse(String yamlContent) { + String resolved = resolveEnvVars(yamlContent); + try { + YamlConfigModel model = YAML_MAPPER.readValue(resolved, YamlConfigModel.class); + return toBuilder(model); + } catch (IOException e) { + throw new ConfigException("Failed to parse YAML config: " + e.getMessage(), e); + } + } + + private static AgentEvalConfig.Builder toBuilder(YamlConfigModel model) { + var builder = AgentEvalConfig.builder(); + if (model == null) { + return builder; + } + + if (model.getDefaults() != null) { + var defaults = model.getDefaults(); + if (defaults.getMaxRetries() != null) { + builder.maxRetries(defaults.getMaxRetries()); + } + if (defaults.getRetryOnRateLimit() != null) { + builder.retryOnRateLimit(defaults.getRetryOnRateLimit()); + } + if (defaults.getMaxConcurrentJudgeCalls() != null) { + builder.maxConcurrentJudgeCalls(defaults.getMaxConcurrentJudgeCalls()); + } + } + + if (model.getCost() != null && model.getCost().getBudget() != null) { + builder.costBudget(model.getCost().getBudget()); + } + + return builder; + } + + static String resolveEnvVars(String content) { + Matcher matcher = ENV_VAR.matcher(content); + StringBuilder result = new StringBuilder(); + while (matcher.find()) { + String envName = matcher.group(1); + String envValue = System.getenv(envName); + matcher.appendReplacement(result, + Matcher.quoteReplacement(envValue != null ? envValue : "")); + } + matcher.appendTail(result); + return result.toString(); + } + + /** + * Returns the parsed {@link YamlConfigModel} for advanced use cases. + */ + public static YamlConfigModel loadModel(Path path) { + try { + String content = resolveEnvVars(Files.readString(path)); + return YAML_MAPPER.readValue(content, YamlConfigModel.class); + } catch (IOException e) { + throw new ConfigException("Failed to load config model from " + path, e); + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/config/ConfigException.java b/agenteval-core/src/main/java/com/agenteval/core/config/ConfigException.java new file mode 100644 index 0000000..6d52bcc --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/config/ConfigException.java @@ -0,0 +1,17 @@ +package com.agenteval.core.config; + +/** + * Unchecked exception for configuration loading errors. + */ +public class ConfigException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public ConfigException(String message) { + super(message); + } + + public ConfigException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java b/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java new file mode 100644 index 0000000..5db87c0 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java @@ -0,0 +1,83 @@ +package com.agenteval.core.config; + +import java.math.BigDecimal; + +/** + * POJO representing the {@code agenteval.yaml} configuration file structure. + * + *Used for Jackson YAML deserialization. Maps to {@link AgentEvalConfig} + * via {@link AgentEvalConfigLoader}.
+ */ +public final class YamlConfigModel { + + private JudgeSection judge; + private EmbeddingSection embedding; + private DefaultsSection defaults; + private CostSection cost; + + public JudgeSection getJudge() { return judge; } + public void setJudge(JudgeSection judge) { this.judge = judge; } + public EmbeddingSection getEmbedding() { return embedding; } + public void setEmbedding(EmbeddingSection embedding) { this.embedding = embedding; } + public DefaultsSection getDefaults() { return defaults; } + public void setDefaults(DefaultsSection defaults) { this.defaults = defaults; } + public CostSection getCost() { return cost; } + public void setCost(CostSection cost) { this.cost = cost; } + + public static final class JudgeSection { + private String provider; + private String model; + private String apiKey; + private String baseUrl; + + public String getProvider() { return provider; } + public void setProvider(String provider) { this.provider = provider; } + public String getModel() { return model; } + public void setModel(String model) { this.model = model; } + public String getApiKey() { return apiKey; } + public void setApiKey(String apiKey) { this.apiKey = apiKey; } + public String getBaseUrl() { return baseUrl; } + public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + } + + public static final class EmbeddingSection { + private String provider; + private String model; + private String apiKey; + private String baseUrl; + + public String getProvider() { return provider; } + public void setProvider(String provider) { this.provider = provider; } + public String getModel() { return model; } + public void setModel(String model) { this.model = model; } + public String getApiKey() { return apiKey; } + public void setApiKey(String apiKey) { this.apiKey = apiKey; } + public String getBaseUrl() { return baseUrl; } + public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + } + + public static final class DefaultsSection { + private Double threshold; + private Integer maxRetries; + private Boolean retryOnRateLimit; + private Integer maxConcurrentJudgeCalls; + + public Double getThreshold() { return threshold; } + public void setThreshold(Double threshold) { this.threshold = threshold; } + public Integer getMaxRetries() { return maxRetries; } + public void setMaxRetries(Integer maxRetries) { this.maxRetries = maxRetries; } + public Boolean getRetryOnRateLimit() { return retryOnRateLimit; } + public void setRetryOnRateLimit(Boolean retryOnRateLimit) { + this.retryOnRateLimit = retryOnRateLimit; + } + public Integer getMaxConcurrentJudgeCalls() { return maxConcurrentJudgeCalls; } + public void setMaxConcurrentJudgeCalls(Integer max) { this.maxConcurrentJudgeCalls = max; } + } + + public static final class CostSection { + private BigDecimal budget; + + public BigDecimal getBudget() { return budget; } + public void setBudget(BigDecimal budget) { this.budget = budget; } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/cost/BudgetExceededException.java b/agenteval-core/src/main/java/com/agenteval/core/cost/BudgetExceededException.java new file mode 100644 index 0000000..eaa7e9b --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/cost/BudgetExceededException.java @@ -0,0 +1,23 @@ +package com.agenteval.core.cost; + +import java.math.BigDecimal; + +/** + * Thrown when the cost budget has been exceeded. + */ +public class BudgetExceededException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final BigDecimal currentCost; + private final BigDecimal budget; + + public BudgetExceededException(BigDecimal currentCost, BigDecimal budget) { + super(String.format("Budget exceeded: $%s / $%s", currentCost, budget)); + this.currentCost = currentCost; + this.budget = budget; + } + + public BigDecimal getCurrentCost() { return currentCost; } + public BigDecimal getBudget() { return budget; } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/cost/CostSummary.java b/agenteval-core/src/main/java/com/agenteval/core/cost/CostSummary.java new file mode 100644 index 0000000..2b3b89b --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/cost/CostSummary.java @@ -0,0 +1,24 @@ +package com.agenteval.core.cost; + +import java.math.BigDecimal; + +/** + * Summary of accumulated costs from LLM judge calls. + * + * @param totalCost the total cost in USD + * @param totalInputTokens total input tokens consumed + * @param totalOutputTokens total output tokens consumed + */ +public record CostSummary( + BigDecimal totalCost, + long totalInputTokens, + long totalOutputTokens +) { + public CostSummary { + if (totalCost == null) totalCost = BigDecimal.ZERO; + } + + public long totalTokens() { + return totalInputTokens + totalOutputTokens; + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/cost/CostTracker.java b/agenteval-core/src/main/java/com/agenteval/core/cost/CostTracker.java new file mode 100644 index 0000000..2568961 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/cost/CostTracker.java @@ -0,0 +1,79 @@ +package com.agenteval.core.cost; + +import com.agenteval.core.model.TokenUsage; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Thread-safe accumulator for tracking LLM usage costs. + * + *Records token usage with a pricing model and tracks total cost + * against an optional budget.
+ */ +public final class CostTracker { + + private static final BigDecimal ONE_MILLION = new BigDecimal("1000000"); + + private final BigDecimal budget; + private final AtomicReferenceUnlike {@link EvalMetric} which evaluates single-turn {@code AgentTestCase}, + * this interface evaluates multi-turn {@link ConversationTestCase} instances. + * Implementations must be thread-safe.
+ */ +public interface ConversationMetric { + + /** + * Evaluates the given conversation test case and returns a score. + * + * @param testCase the multi-turn conversation to evaluate + * @return the evaluation score (0.0-1.0) + */ + EvalScore evaluate(ConversationTestCase testCase); + + /** + * Returns the name of this metric. + */ + String name(); +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/config/AgentEvalConfigLoaderTest.java b/agenteval-core/src/test/java/com/agenteval/core/config/AgentEvalConfigLoaderTest.java new file mode 100644 index 0000000..ec7d334 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/config/AgentEvalConfigLoaderTest.java @@ -0,0 +1,91 @@ +package com.agenteval.core.config; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.math.BigDecimal; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AgentEvalConfigLoaderTest { + + @Test + void shouldParseYamlConfig() { + String yaml = """ + defaults: + maxRetries: 5 + retryOnRateLimit: true + maxConcurrentJudgeCalls: 8 + cost: + budget: 10.00 + """; + + var builder = AgentEvalConfigLoader.parse(yaml); + var config = builder.build(); + + assertThat(config.maxRetries()).isEqualTo(5); + assertThat(config.retryOnRateLimit()).isTrue(); + assertThat(config.maxConcurrentJudgeCalls()).isEqualTo(8); + assertThat(config.costBudget()).isEqualByComparingTo(new BigDecimal("10.00")); + } + + @Test + void shouldLoadFromPath(@TempDir Path tmpDir) throws Exception { + Path yamlFile = tmpDir.resolve("agenteval.yaml"); + Files.writeString(yamlFile, """ + defaults: + maxRetries: 2 + """); + + var builder = AgentEvalConfigLoader.load(yamlFile); + var config = builder.build(); + + assertThat(config.maxRetries()).isEqualTo(2); + } + + @Test + void shouldResolveEnvVars() { + String resolved = AgentEvalConfigLoader.resolveEnvVars( + "key: ${PATH}"); + assertThat(resolved).doesNotContain("${PATH}"); + } + + @Test + void shouldHandleMissingEnvVars() { + String resolved = AgentEvalConfigLoader.resolveEnvVars( + "key: ${NONEXISTENT_VAR_12345}"); + assertThat(resolved).isEqualTo("key: "); + } + + @Test + void shouldHandleMinimalConfig() { + var builder = AgentEvalConfigLoader.parse("---\n"); + var config = builder.build(); + assertThat(config.maxRetries()).isEqualTo(3); + } + + @Test + void shouldParseJudgeSection() { + String yaml = """ + judge: + provider: openai + model: gpt-4o + baseUrl: https://api.openai.com + """; + + var builder = AgentEvalConfigLoader.parse(yaml); + var config = builder.build(); + assertThat(config).isNotNull(); + } + + @Test + void shouldThrowOnInvalidFile(@TempDir Path tmpDir) { + Path missing = tmpDir.resolve("nonexistent.yaml"); + + assertThatThrownBy(() -> AgentEvalConfigLoader.load(missing)) + .isInstanceOf(ConfigException.class); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/cost/CostTrackerTest.java b/agenteval-core/src/test/java/com/agenteval/core/cost/CostTrackerTest.java new file mode 100644 index 0000000..a9460a6 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/cost/CostTrackerTest.java @@ -0,0 +1,72 @@ +package com.agenteval.core.cost; + +import com.agenteval.core.model.TokenUsage; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class CostTrackerTest { + + private static final PricingModel GPT4O_PRICING = new PricingModel( + new BigDecimal("5.00"), + new BigDecimal("15.00"), + "openai"); + + @Test + void shouldTrackCost() { + var tracker = new CostTracker(); + tracker.record(new TokenUsage(1000, 500, 1500), GPT4O_PRICING); + + assertThat(tracker.totalCost()).isGreaterThan(BigDecimal.ZERO); + var summary = tracker.summary(); + assertThat(summary.totalInputTokens()).isEqualTo(1000); + assertThat(summary.totalOutputTokens()).isEqualTo(500); + assertThat(summary.totalTokens()).isEqualTo(1500); + } + + @Test + void shouldAccumulateCosts() { + var tracker = new CostTracker(); + tracker.record(new TokenUsage(1000, 500, 1500), GPT4O_PRICING); + tracker.record(new TokenUsage(2000, 1000, 3000), GPT4O_PRICING); + + var summary = tracker.summary(); + assertThat(summary.totalInputTokens()).isEqualTo(3000); + assertThat(summary.totalOutputTokens()).isEqualTo(1500); + } + + @Test + void shouldThrowWhenBudgetExceeded() { + var tracker = new CostTracker(new BigDecimal("0.001")); + + assertThatThrownBy(() -> + tracker.record(new TokenUsage(1000000, 1000000, 2000000), GPT4O_PRICING)) + .isInstanceOf(BudgetExceededException.class); + } + + @Test + void shouldNotThrowWithinBudget() { + var tracker = new CostTracker(new BigDecimal("100.00")); + tracker.record(new TokenUsage(100, 50, 150), GPT4O_PRICING); + + assertThat(tracker.isOverBudget()).isFalse(); + } + + @Test + void shouldHandleNullUsage() { + var tracker = new CostTracker(); + tracker.record(null, GPT4O_PRICING); + + assertThat(tracker.totalCost()).isEqualByComparingTo(BigDecimal.ZERO); + } + + @Test + void shouldStartAtZero() { + var tracker = new CostTracker(); + assertThat(tracker.totalCost()).isEqualByComparingTo(BigDecimal.ZERO); + assertThat(tracker.isOverBudget()).isFalse(); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/cost/PricingModelTest.java b/agenteval-core/src/test/java/com/agenteval/core/cost/PricingModelTest.java new file mode 100644 index 0000000..6ecb7d9 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/cost/PricingModelTest.java @@ -0,0 +1,44 @@ +package com.agenteval.core.cost; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PricingModelTest { + + @Test + void shouldCreateValidPricingModel() { + var pricing = new PricingModel( + new BigDecimal("5.00"), + new BigDecimal("15.00"), + "openai"); + + assertThat(pricing.inputCostPer1MTokens()).isEqualByComparingTo(new BigDecimal("5.00")); + assertThat(pricing.outputCostPer1MTokens()).isEqualByComparingTo(new BigDecimal("15.00")); + assertThat(pricing.provider()).isEqualTo("openai"); + } + + @Test + void shouldRejectNegativeInputCost() { + assertThatThrownBy(() -> new PricingModel( + new BigDecimal("-1"), BigDecimal.ONE, "test")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectNegativeOutputCost() { + assertThatThrownBy(() -> new PricingModel( + BigDecimal.ONE, new BigDecimal("-1"), "test")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectBlankProvider() { + assertThatThrownBy(() -> new PricingModel( + BigDecimal.ONE, BigDecimal.ONE, "")) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/agenteval-core/src/test/resources/agenteval.yaml b/agenteval-core/src/test/resources/agenteval.yaml new file mode 100644 index 0000000..9b6a0e2 --- /dev/null +++ b/agenteval-core/src/test/resources/agenteval.yaml @@ -0,0 +1,17 @@ +judge: + provider: openai + model: gpt-4o + apiKey: ${OPENAI_API_KEY} + baseUrl: https://api.openai.com + +embedding: + provider: openai + model: text-embedding-3-small + +defaults: + maxRetries: 5 + retryOnRateLimit: true + maxConcurrentJudgeCalls: 8 + +cost: + budget: 10.00 diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/DatasetFormat.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/DatasetFormat.java new file mode 100644 index 0000000..96a8d97 --- /dev/null +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/DatasetFormat.java @@ -0,0 +1,34 @@ +package com.agenteval.datasets; + +import java.nio.file.Path; + +/** + * Supported dataset file formats. + */ +public enum DatasetFormat { + JSON, JSONL, CSV; + + /** + * Auto-detects the dataset format from a file path extension. + * + * @param path the file path to inspect + * @return the detected format + * @throws DatasetException if the extension is not recognized + */ + public static DatasetFormat detect(Path path) { + Path fileName = path.getFileName(); + if (fileName == null) { + throw new DatasetException("Cannot detect format from root path"); + } + String name = fileName.toString().toLowerCase(); + if (name.endsWith(".jsonl")) { + return JSONL; + } else if (name.endsWith(".json")) { + return JSON; + } else if (name.endsWith(".csv")) { + return CSV; + } + throw new DatasetException("Unsupported dataset format: " + name + + ". Supported extensions: .json, .jsonl, .csv"); + } +} diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/DatasetLoaders.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/DatasetLoaders.java new file mode 100644 index 0000000..0fecb62 --- /dev/null +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/DatasetLoaders.java @@ -0,0 +1,35 @@ +package com.agenteval.datasets; + +import com.agenteval.datasets.csv.CsvDatasetLoader; +import com.agenteval.datasets.json.JsonDatasetLoader; +import com.agenteval.datasets.jsonl.JsonlDatasetLoader; + +import java.nio.file.Path; + +/** + * Factory for auto-detecting and loading datasets from file paths. + * + *{@code
+ * EvalDataset dataset = DatasetLoaders.forPath(Path.of("data.csv"));
+ * }
+ */
+public final class DatasetLoaders {
+
+ private DatasetLoaders() {}
+
+ /**
+ * Auto-detects the format from the file extension and loads the dataset.
+ *
+ * @param path the dataset file path
+ * @return the loaded dataset
+ * @throws DatasetException if loading or format detection fails
+ */
+ public static EvalDataset forPath(Path path) {
+ DatasetFormat format = DatasetFormat.detect(path);
+ return switch (format) {
+ case JSON -> new JsonDatasetLoader().load(path);
+ case JSONL -> new JsonlDatasetLoader().load(path);
+ case CSV -> new CsvDatasetLoader().load(path);
+ };
+ }
+}
diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java
index 0f4cbbb..8562722 100644
--- a/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java
+++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java
@@ -1,7 +1,9 @@
package com.agenteval.datasets;
import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.datasets.csv.CsvDatasetWriter;
import com.agenteval.datasets.json.JsonDatasetWriter;
+import com.agenteval.datasets.jsonl.JsonlDatasetWriter;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
@@ -54,6 +56,22 @@ public void save(Path path) {
new JsonDatasetWriter().write(this, path);
}
+ /**
+ * Saves this dataset to a file in the specified format.
+ *
+ * @param path the target file path
+ * @param format the output format
+ * @throws DatasetException if writing fails
+ */
+ public void save(Path path, DatasetFormat format) {
+ switch (format) {
+ case JSON -> new JsonDatasetWriter().write(this, path);
+ case JSONL -> new JsonlDatasetWriter().write(this, path);
+ case CSV -> new CsvDatasetWriter().write(this, path);
+ default -> throw new DatasetException("Unsupported format: " + format);
+ }
+ }
+
@JsonPOJOBuilder(withPrefix = "")
public static final class Builder {
private String name;
diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java
new file mode 100644
index 0000000..4cbb58c
--- /dev/null
+++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java
@@ -0,0 +1,159 @@
+package com.agenteval.datasets.csv;
+
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.datasets.DatasetException;
+import com.agenteval.datasets.DatasetLoader;
+import com.agenteval.datasets.EvalDataset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Loads evaluation datasets from RFC 4180 CSV files.
+ *
+ * Expected columns: {@code input, actualOutput, expectedOutput, retrievalContext, context}. + * List fields (retrievalContext, context) use pipe ({@code |}) as separator.
+ */ +public final class CsvDatasetLoader implements DatasetLoader { + + private static final Logger LOG = LoggerFactory.getLogger(CsvDatasetLoader.class); + private static final String PIPE_SEPARATOR = "\\|"; + + @Override + public EvalDataset load(Path path) { + LOG.debug("Loading CSV dataset from {}", path); + try (var reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load CSV dataset from " + path, e); + } + } + + @Override + public EvalDataset load(InputStream inputStream) { + LOG.debug("Loading CSV dataset from input stream"); + try (var reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load CSV dataset from input stream", e); + } + } + + private EvalDataset parse(BufferedReader reader) throws IOException { + String headerLine = reader.readLine(); + if (headerLine == null) { + throw new DatasetException("CSV file is empty — expected header row"); + } + + String[] headers = parseCsvLine(headerLine); + int inputIdx = indexOf(headers, "input"); + int actualOutputIdx = indexOf(headers, "actualOutput"); + int expectedOutputIdx = indexOf(headers, "expectedOutput"); + int retrievalContextIdx = indexOf(headers, "retrievalContext"); + int contextIdx = indexOf(headers, "context"); + + if (inputIdx == -1) { + throw new DatasetException("CSV header must contain 'input' column"); + } + + ListList fields use pipe ({@code |}) as separator.
+ */ +public final class CsvDatasetWriter implements DatasetWriter { + + private static final Logger LOG = LoggerFactory.getLogger(CsvDatasetWriter.class); + private static final String HEADER = "input,actualOutput,expectedOutput,retrievalContext,context"; + + @Override + public void write(EvalDataset dataset, Path path) { + LOG.debug("Writing CSV dataset to {}", path); + try (var out = Files.newOutputStream(path)) { + write(dataset, out); + } catch (IOException e) { + throw new DatasetException("Failed to write CSV dataset to " + path, e); + } + } + + @Override + public void write(EvalDataset dataset, OutputStream outputStream) { + try { + Writer writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8); + writer.write(HEADER); + writer.write('\n'); + + for (AgentTestCase tc : dataset.getTestCases()) { + writer.write(escapeCsv(tc.getInput())); + writer.write(','); + writer.write(escapeCsv(tc.getActualOutput())); + writer.write(','); + writer.write(escapeCsv(tc.getExpectedOutput())); + writer.write(','); + writer.write(escapeCsv(joinPipe(tc.getRetrievalContext()))); + writer.write(','); + writer.write(escapeCsv(joinPipe(tc.getContext()))); + writer.write('\n'); + } + + writer.flush(); + } catch (IOException e) { + throw new DatasetException("Failed to write CSV dataset", e); + } + } + + static String escapeCsv(String value) { + if (value == null) return ""; + if (value.contains(",") || value.contains("\"") || value.contains("\n")) { + return "\"" + value.replace("\"", "\"\"") + "\""; + } + return value; + } + + private static String joinPipe(ListEach line contains a single JSON object representing an {@link AgentTestCase}.
+ */ +public final class JsonlDatasetLoader implements DatasetLoader { + + private static final Logger LOG = LoggerFactory.getLogger(JsonlDatasetLoader.class); + + private final ObjectMapper mapper; + + public JsonlDatasetLoader() { + this(new ObjectMapper()); + } + + public JsonlDatasetLoader(ObjectMapper mapper) { + this.mapper = mapper; + } + + @Override + public EvalDataset load(Path path) { + LOG.debug("Loading JSONL dataset from {}", path); + try (var reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load JSONL dataset from " + path, e); + } + } + + @Override + public EvalDataset load(InputStream inputStream) { + LOG.debug("Loading JSONL dataset from input stream"); + try (var reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load JSONL dataset from input stream", e); + } + } + + private EvalDataset parse(BufferedReader reader) throws IOException { + ListEach line contains a single JSON object representing an {@link AgentTestCase}.
+ */ +public final class JsonlDatasetWriter implements DatasetWriter { + + private static final Logger LOG = LoggerFactory.getLogger(JsonlDatasetWriter.class); + + private final ObjectMapper mapper; + + public JsonlDatasetWriter() { + this(new ObjectMapper()); + } + + public JsonlDatasetWriter(ObjectMapper mapper) { + this.mapper = mapper; + } + + @Override + public void write(EvalDataset dataset, Path path) { + LOG.debug("Writing JSONL dataset to {}", path); + try (var out = Files.newOutputStream(path)) { + write(dataset, out); + } catch (IOException e) { + throw new DatasetException("Failed to write JSONL dataset to " + path, e); + } + } + + @Override + public void write(EvalDataset dataset, OutputStream outputStream) { + try { + Writer writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8); + for (AgentTestCase tc : dataset.getTestCases()) { + writer.write(mapper.writeValueAsString(tc)); + writer.write('\n'); + } + writer.flush(); + } catch (IOException e) { + throw new DatasetException("Failed to write JSONL dataset", e); + } + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/DatasetLoadersTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/DatasetLoadersTest.java new file mode 100644 index 0000000..f9faaaa --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/DatasetLoadersTest.java @@ -0,0 +1,57 @@ +package com.agenteval.datasets; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class DatasetLoadersTest { + + @Test + void shouldAutoDetectJson(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.json"); + Files.writeString(file, "[{\"input\":\"Hello\"}]"); + + EvalDataset dataset = DatasetLoaders.forPath(file); + assertThat(dataset.size()).isEqualTo(1); + } + + @Test + void shouldAutoDetectJsonl(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.jsonl"); + Files.writeString(file, "{\"input\":\"Hello\"}\n"); + + EvalDataset dataset = DatasetLoaders.forPath(file); + assertThat(dataset.size()).isEqualTo(1); + } + + @Test + void shouldAutoDetectCsv(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.csv"); + Files.writeString(file, "input,actualOutput\nHello,World\n"); + + EvalDataset dataset = DatasetLoaders.forPath(file); + assertThat(dataset.size()).isEqualTo(1); + } + + @Test + void shouldThrowOnUnsupportedFormat(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.xml"); + Files.writeString(file, ""); + + assertThatThrownBy(() -> DatasetLoaders.forPath(file)) + .isInstanceOf(DatasetException.class) + .hasMessageContaining("Unsupported"); + } + + @Test + void shouldDetectFormats() { + assertThat(DatasetFormat.detect(Path.of("data.json"))).isEqualTo(DatasetFormat.JSON); + assertThat(DatasetFormat.detect(Path.of("data.jsonl"))).isEqualTo(DatasetFormat.JSONL); + assertThat(DatasetFormat.detect(Path.of("data.csv"))).isEqualTo(DatasetFormat.CSV); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetLoaderTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetLoaderTest.java new file mode 100644 index 0000000..65eebb1 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetLoaderTest.java @@ -0,0 +1,65 @@ +package com.agenteval.datasets.csv; + +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class CsvDatasetLoaderTest { + + @Test + void shouldLoadFromClasspath() { + var is = getClass().getClassLoader().getResourceAsStream("test-dataset.csv"); + EvalDataset dataset = new CsvDatasetLoader().load(is); + + assertThat(dataset.size()).isEqualTo(2); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("What is Java?"); + assertThat(dataset.getTestCases().get(0).getRetrievalContext()).hasSize(2); + } + + @Test + void shouldLoadFromPath(@TempDir Path tmpDir) throws Exception { + Path csvFile = tmpDir.resolve("test.csv"); + Files.writeString(csvFile, "input,actualOutput\nHello,World\n"); + + EvalDataset dataset = new CsvDatasetLoader().load(csvFile); + + assertThat(dataset.size()).isEqualTo(1); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("Hello"); + assertThat(dataset.getTestCases().get(0).getActualOutput()).isEqualTo("World"); + } + + @Test + void shouldHandleQuotedFields() { + String csv = "input,actualOutput\n\"Hello, World\",\"He said \"\"hi\"\"\"\n"; + var is = new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)); + EvalDataset dataset = new CsvDatasetLoader().load(is); + + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("Hello, World"); + assertThat(dataset.getTestCases().get(0).getActualOutput()).isEqualTo("He said \"hi\""); + } + + @Test + void shouldRejectEmptyFile() { + var is = new ByteArrayInputStream(new byte[0]); + assertThatThrownBy(() -> new CsvDatasetLoader().load(is)) + .hasMessageContaining("empty"); + } + + @Test + void shouldParsePipeSeparatedLists() { + String csv = "input,retrievalContext\nquery,doc1|doc2|doc3\n"; + var is = new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)); + EvalDataset dataset = new CsvDatasetLoader().load(is); + + assertThat(dataset.getTestCases().get(0).getRetrievalContext()) + .containsExactly("doc1", "doc2", "doc3"); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetWriterTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetWriterTest.java new file mode 100644 index 0000000..9368e21 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetWriterTest.java @@ -0,0 +1,73 @@ +package com.agenteval.datasets.csv; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class CsvDatasetWriterTest { + + @Test + void shouldWriteToOutputStream() { + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder() + .input("Hello") + .actualOutput("World") + .build())) + .build(); + + var out = new ByteArrayOutputStream(); + new CsvDatasetWriter().write(dataset, out); + + String csv = out.toString(); + assertThat(csv).contains("input,actualOutput,expectedOutput,retrievalContext,context"); + assertThat(csv).contains("Hello,World"); + } + + @Test + void shouldWriteToFile(@TempDir Path tmpDir) { + Path csvFile = tmpDir.resolve("out.csv"); + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder() + .input("q") + .actualOutput("a") + .build())) + .build(); + + new CsvDatasetWriter().write(dataset, csvFile); + + assertThat(csvFile).exists(); + } + + @Test + void shouldEscapeCommasAndQuotes() { + assertThat(CsvDatasetWriter.escapeCsv("hello,world")).isEqualTo("\"hello,world\""); + assertThat(CsvDatasetWriter.escapeCsv("say \"hi\"")).isEqualTo("\"say \"\"hi\"\"\""); + assertThat(CsvDatasetWriter.escapeCsv("simple")).isEqualTo("simple"); + assertThat(CsvDatasetWriter.escapeCsv(null)).isEmpty(); + } + + @Test + void shouldWritePipeSeparatedLists() { + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder() + .input("q") + .retrievalContext(List.of("doc1", "doc2")) + .build())) + .build(); + + var out = new ByteArrayOutputStream(); + new CsvDatasetWriter().write(dataset, out); + + assertThat(out.toString()).contains("doc1|doc2"); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetLoaderTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetLoaderTest.java new file mode 100644 index 0000000..1928ff2 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetLoaderTest.java @@ -0,0 +1,57 @@ +package com.agenteval.datasets.jsonl; + +import com.agenteval.datasets.DatasetException; +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class JsonlDatasetLoaderTest { + + @Test + void shouldLoadFromClasspath() { + var is = getClass().getClassLoader().getResourceAsStream("test-dataset.jsonl"); + EvalDataset dataset = new JsonlDatasetLoader().load(is); + + assertThat(dataset.size()).isEqualTo(2); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("What is Java?"); + assertThat(dataset.getTestCases().get(0).getRetrievalContext()).hasSize(2); + } + + @Test + void shouldLoadFromPath(@TempDir Path tmpDir) throws Exception { + Path jsonlFile = tmpDir.resolve("test.jsonl"); + Files.writeString(jsonlFile, "{\"input\":\"Hello\",\"actualOutput\":\"World\"}\n"); + + EvalDataset dataset = new JsonlDatasetLoader().load(jsonlFile); + + assertThat(dataset.size()).isEqualTo(1); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("Hello"); + } + + @Test + void shouldSkipBlankLines() { + String jsonl = "{\"input\":\"q1\"}\n\n{\"input\":\"q2\"}\n"; + var is = new ByteArrayInputStream(jsonl.getBytes(StandardCharsets.UTF_8)); + EvalDataset dataset = new JsonlDatasetLoader().load(is); + + assertThat(dataset.size()).isEqualTo(2); + } + + @Test + void shouldThrowOnInvalidJson() { + String jsonl = "not valid json\n"; + var is = new ByteArrayInputStream(jsonl.getBytes(StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> new JsonlDatasetLoader().load(is)) + .isInstanceOf(DatasetException.class) + .hasMessageContaining("line 1"); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetWriterTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetWriterTest.java new file mode 100644 index 0000000..40f5410 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetWriterTest.java @@ -0,0 +1,45 @@ +package com.agenteval.datasets.jsonl; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class JsonlDatasetWriterTest { + + @Test + void shouldWriteOneJsonPerLine() { + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder().input("q1").build(), + AgentTestCase.builder().input("q2").build())) + .build(); + + var out = new ByteArrayOutputStream(); + new JsonlDatasetWriter().write(dataset, out); + + String[] lines = out.toString().trim().split("\n"); + assertThat(lines).hasSize(2); + assertThat(lines[0]).contains("\"input\":\"q1\""); + assertThat(lines[1]).contains("\"input\":\"q2\""); + } + + @Test + void shouldWriteToFile(@TempDir Path tmpDir) { + Path jsonlFile = tmpDir.resolve("out.jsonl"); + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder().input("q").build())) + .build(); + + new JsonlDatasetWriter().write(dataset, jsonlFile); + + assertThat(jsonlFile).exists(); + } +} diff --git a/agenteval-datasets/src/test/resources/test-dataset.csv b/agenteval-datasets/src/test/resources/test-dataset.csv new file mode 100644 index 0000000..22032b0 --- /dev/null +++ b/agenteval-datasets/src/test/resources/test-dataset.csv @@ -0,0 +1,3 @@ +input,actualOutput,expectedOutput,retrievalContext,context +What is Java?,Java is a programming language.,Java is a programming language developed by Sun.,Java was created by Sun|Java runs on JVM, +What is Python?,Python is easy to learn.,Python is a high-level language.,Python was created by Guido,Python docs diff --git a/agenteval-datasets/src/test/resources/test-dataset.jsonl b/agenteval-datasets/src/test/resources/test-dataset.jsonl new file mode 100644 index 0000000..3090a97 --- /dev/null +++ b/agenteval-datasets/src/test/resources/test-dataset.jsonl @@ -0,0 +1,2 @@ +{"input":"What is Java?","actualOutput":"Java is a programming language.","expectedOutput":"Java is a programming language developed by Sun.","retrievalContext":["Java was created by Sun","Java runs on JVM"]} +{"input":"What is Python?","actualOutput":"Python is easy to learn.","expectedOutput":"Python is a high-level language.","retrievalContext":["Python was created by Guido"]} diff --git a/agenteval-embeddings/pom.xml b/agenteval-embeddings/pom.xml new file mode 100644 index 0000000..b776fbb --- /dev/null +++ b/agenteval-embeddings/pom.xml @@ -0,0 +1,36 @@ + +{@code
+ * var model = EmbeddingModels.openai("text-embedding-3-small");
+ * var model = EmbeddingModels.ollama("nomic-embed-text");
+ * }
+ */
+public final class EmbeddingModels {
+
+ private static final String OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
+ private static final String OPENAI_BASE_URL = "https://api.openai.com";
+ private static final String OLLAMA_BASE_URL = "http://localhost:11434";
+
+ private EmbeddingModels() {}
+
+ /**
+ * Creates an OpenAI embedding model using the given model ID.
+ * API key is resolved from the {@code OPENAI_API_KEY} environment variable.
+ */
+ public static EmbeddingModel openai(String model) {
+ String apiKey = System.getenv(OPENAI_API_KEY_ENV);
+ if (apiKey == null || apiKey.isBlank()) {
+ throw new EmbeddingException(
+ "OpenAI API key not found. Set the " + OPENAI_API_KEY_ENV
+ + " environment variable or use EmbeddingModels.openai(EmbeddingConfig)");
+ }
+ return openai(EmbeddingConfig.builder()
+ .apiKey(apiKey)
+ .model(model)
+ .baseUrl(OPENAI_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates an OpenAI embedding model with full configuration.
+ */
+ public static EmbeddingModel openai(EmbeddingConfig config) {
+ return new OpenAiEmbeddingModel(config);
+ }
+
+ /**
+ * Creates an Ollama embedding model using the given model ID.
+ * Defaults to {@code localhost:11434}.
+ */
+ public static EmbeddingModel ollama(String model) {
+ return ollama(EmbeddingConfig.builder()
+ .model(model)
+ .baseUrl(OLLAMA_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates an Ollama embedding model with full configuration.
+ */
+ public static EmbeddingModel ollama(EmbeddingConfig config) {
+ return new OllamaEmbeddingModel(config);
+ }
+}
diff --git a/agenteval-embeddings/src/main/java/com/agenteval/embeddings/config/EmbeddingConfig.java b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/config/EmbeddingConfig.java
new file mode 100644
index 0000000..9998196
--- /dev/null
+++ b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/config/EmbeddingConfig.java
@@ -0,0 +1,49 @@
+package com.agenteval.embeddings.config;
+
+import java.time.Duration;
+import java.util.Objects;
+
+/**
+ * Configuration for an embedding model provider.
+ */
+public final class EmbeddingConfig {
+
+ private final String apiKey;
+ private final String model;
+ private final String baseUrl;
+ private final Duration timeout;
+
+ private EmbeddingConfig(Builder builder) {
+ this.apiKey = builder.apiKey;
+ this.model = Objects.requireNonNull(builder.model, "model must not be null");
+ this.baseUrl = Objects.requireNonNull(builder.baseUrl, "baseUrl must not be null");
+ this.timeout = builder.timeout;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getApiKey() { return apiKey; }
+ public String getModel() { return model; }
+ public String getBaseUrl() { return baseUrl; }
+ public Duration getTimeout() { return timeout; }
+
+ public static final class Builder {
+ private String apiKey;
+ private String model;
+ private String baseUrl;
+ private Duration timeout = Duration.ofSeconds(30);
+
+ private Builder() {}
+
+ public Builder apiKey(String apiKey) { this.apiKey = apiKey; return this; }
+ public Builder model(String model) { this.model = model; return this; }
+ public Builder baseUrl(String baseUrl) { this.baseUrl = baseUrl; return this; }
+ public Builder timeout(Duration timeout) { this.timeout = timeout; return this; }
+
+ public EmbeddingConfig build() {
+ return new EmbeddingConfig(this);
+ }
+ }
+}
diff --git a/agenteval-embeddings/src/main/java/com/agenteval/embeddings/http/HttpEmbeddingClient.java b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/http/HttpEmbeddingClient.java
new file mode 100644
index 0000000..7cdca1b
--- /dev/null
+++ b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/http/HttpEmbeddingClient.java
@@ -0,0 +1,59 @@
+package com.agenteval.embeddings.http;
+
+import com.agenteval.embeddings.EmbeddingException;
+import com.agenteval.embeddings.config.EmbeddingConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+
+/**
+ * HTTP client for embedding model requests.
+ */
+public class HttpEmbeddingClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HttpEmbeddingClient.class);
+
+ private final HttpClient httpClient;
+ private final EmbeddingConfig config;
+
+ public HttpEmbeddingClient(EmbeddingConfig config) {
+ this(config, HttpClient.newBuilder()
+ .connectTimeout(config.getTimeout())
+ .build());
+ }
+
+ HttpEmbeddingClient(EmbeddingConfig config, HttpClient httpClient) {
+ this.config = config;
+ this.httpClient = httpClient;
+ }
+
+ /**
+ * Sends an embedding request and returns the response.
+ */
+ public HttpEmbeddingResponse send(HttpEmbeddingRequest request) {
+ LOG.debug("Sending embedding request to {}", request.url());
+ try {
+ var builder = HttpRequest.newBuilder()
+ .uri(URI.create(request.url()))
+ .timeout(config.getTimeout())
+ .POST(HttpRequest.BodyPublishers.ofString(request.body()));
+ request.headers().forEach(builder::header);
+ builder.header("Content-Type", "application/json");
+
+ HttpResponseSends requests to {@code POST /api/embeddings}. No API key required.
+ */ +public final class OllamaEmbeddingModel implements EmbeddingModel { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String EMBEDDINGS_PATH = "/api/embeddings"; + + private final EmbeddingConfig config; + private final HttpEmbeddingClient client; + + public OllamaEmbeddingModel(EmbeddingConfig config) { + this(config, new HttpEmbeddingClient(config)); + } + + OllamaEmbeddingModel(EmbeddingConfig config, HttpEmbeddingClient client) { + this.config = Objects.requireNonNull(config, "config must not be null"); + this.client = Objects.requireNonNull(client, "client must not be null"); + } + + @Override + public ListSends requests to {@code POST /v1/embeddings}.
+ */ +public final class OpenAiEmbeddingModel implements EmbeddingModel { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String EMBEDDINGS_PATH = "/v1/embeddings"; + + private final EmbeddingConfig config; + private final HttpEmbeddingClient client; + + public OpenAiEmbeddingModel(EmbeddingConfig config) { + this(config, new HttpEmbeddingClient(config)); + } + + OpenAiEmbeddingModel(EmbeddingConfig config, HttpEmbeddingClient client) { + this.config = Objects.requireNonNull(config, "config must not be null"); + this.client = Objects.requireNonNull(client, "client must not be null"); + } + + @Override + public List{@code
* var judge = JudgeModels.openai("gpt-4o");
* var judge = JudgeModels.anthropic("claude-sonnet-4-20250514");
+ * var judge = JudgeModels.ollama("llama3");
* var judge = JudgeModels.openai(JudgeConfig.builder()
* .apiKey("sk-...")
* .model("gpt-4o")
@@ -26,6 +28,7 @@ public final class JudgeModels {
private static final String ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY";
private static final String OPENAI_BASE_URL = "https://api.openai.com";
private static final String ANTHROPIC_BASE_URL = "https://api.anthropic.com";
+ private static final String OLLAMA_BASE_URL = "http://localhost:11434";
private JudgeModels() {}
@@ -67,6 +70,24 @@ public static JudgeModel anthropic(JudgeConfig config) {
return new AnthropicJudgeModel(config);
}
+ /**
+ * Creates an Ollama judge model using the given model ID.
+ * Defaults to {@code localhost:11434}. No API key required.
+ */
+ public static JudgeModel ollama(String model) {
+ return ollama(JudgeConfig.builder()
+ .model(model)
+ .baseUrl(OLLAMA_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates an Ollama judge model with full configuration.
+ */
+ public static JudgeModel ollama(JudgeConfig config) {
+ return new OllamaJudgeModel(config);
+ }
+
private static String resolveApiKey(String envVar, String providerName) {
String key = System.getenv(envVar);
if (key == null || key.isBlank()) {
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java b/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java
index 75c90fa..5edf954 100644
--- a/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java
@@ -16,7 +16,7 @@ public final class JudgeConfig {
private final double temperature;
private JudgeConfig(Builder builder) {
- this.apiKey = Objects.requireNonNull(builder.apiKey, "apiKey must not be null");
+ this.apiKey = builder.apiKey;
this.model = Objects.requireNonNull(builder.model, "model must not be null");
this.baseUrl = Objects.requireNonNull(builder.baseUrl, "baseUrl must not be null");
this.timeout = builder.timeout;
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java
index ad701bd..d6f6df6 100644
--- a/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java
@@ -29,6 +29,9 @@ public final class AnthropicJudgeModel extends AbstractHttpJudgeModel {
public AnthropicJudgeModel(JudgeConfig config) {
super(config);
+ if (config.getApiKey() == null || config.getApiKey().isBlank()) {
+ throw new JudgeException("Anthropic requires a non-null API key");
+ }
}
AnthropicJudgeModel(JudgeConfig config, HttpJudgeClient client) {
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/OllamaJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OllamaJudgeModel.java
new file mode 100644
index 0000000..69160e9
--- /dev/null
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OllamaJudgeModel.java
@@ -0,0 +1,88 @@
+package com.agenteval.judge.provider;
+
+import com.agenteval.core.model.TokenUsage;
+import com.agenteval.judge.JudgeException;
+import com.agenteval.judge.config.JudgeConfig;
+import com.agenteval.judge.http.HttpJudgeClient;
+import com.agenteval.judge.http.HttpJudgeRequest;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.util.Map;
+
+/**
+ * Ollama judge model provider.
+ *
+ * Sends requests to {@code POST /api/chat} with JSON response format.
+ * No API key required.
+ */
+public final class OllamaJudgeModel extends AbstractHttpJudgeModel {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final String DEFAULT_BASE_URL = "http://localhost:11434";
+ private static final String CHAT_PATH = "/api/chat";
+ private static final String SYSTEM_PROMPT =
+ "You are an evaluation judge. Respond ONLY with a JSON object "
+ + "containing \"score\" (a number between 0.0 and 1.0) "
+ + "and \"reason\" (a brief explanation).";
+
+ public OllamaJudgeModel(JudgeConfig config) {
+ super(config);
+ }
+
+ OllamaJudgeModel(JudgeConfig config, HttpJudgeClient client) {
+ super(config, client);
+ }
+
+ static String defaultBaseUrl() {
+ return DEFAULT_BASE_URL;
+ }
+
+ @Override
+ protected HttpJudgeRequest buildRequest(String prompt) {
+ try {
+ var body = MAPPER.createObjectNode();
+ body.put("model", config.getModel());
+ body.put("stream", false);
+ body.put("format", "json");
+
+ var messages = body.putArray("messages");
+
+ var systemMsg = messages.addObject();
+ systemMsg.put("role", "system");
+ systemMsg.put("content", SYSTEM_PROMPT);
+
+ var userMsg = messages.addObject();
+ userMsg.put("role", "user");
+ userMsg.put("content", prompt);
+
+ String url = config.getBaseUrl() + CHAT_PATH;
+ return new HttpJudgeRequest(url, Map.of(),
+ MAPPER.writeValueAsString(body));
+ } catch (Exception e) {
+ throw new JudgeException("Failed to build Ollama request", e);
+ }
+ }
+
+ @Override
+ protected String extractContent(String responseBody) {
+ JsonNode root = parseJson(responseBody);
+ JsonNode message = root.path("message");
+ if (message.isMissingNode()) {
+ throw new JudgeException("No message in Ollama response");
+ }
+ return message.path("content").asText("");
+ }
+
+ @Override
+ protected TokenUsage extractTokenUsage(String responseBody) {
+ JsonNode root = parseJson(responseBody);
+ int promptTokens = root.path("prompt_eval_count").asInt(0);
+ int completionTokens = root.path("eval_count").asInt(0);
+ if (promptTokens == 0 && completionTokens == 0) {
+ return null;
+ }
+ return new TokenUsage(promptTokens, completionTokens,
+ promptTokens + completionTokens);
+ }
+}
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java
index dc05d53..96d6cf0 100644
--- a/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java
@@ -28,6 +28,9 @@ public final class OpenAiJudgeModel extends AbstractHttpJudgeModel {
public OpenAiJudgeModel(JudgeConfig config) {
super(config);
+ if (config.getApiKey() == null || config.getApiKey().isBlank()) {
+ throw new JudgeException("OpenAI requires a non-null API key");
+ }
}
OpenAiJudgeModel(JudgeConfig config, HttpJudgeClient client) {
diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java
index 948d174..04c5a1d 100644
--- a/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java
+++ b/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java
@@ -42,12 +42,12 @@ void shouldBuildWithCustomValues() {
}
@Test
- void shouldRejectNullApiKey() {
- assertThatThrownBy(() -> JudgeConfig.builder()
+ void shouldAllowNullApiKey() {
+ var config = JudgeConfig.builder()
.model("gpt-4o")
.baseUrl("https://api.openai.com")
- .build())
- .isInstanceOf(NullPointerException.class);
+ .build();
+ assertThat(config.getApiKey()).isNull();
}
@Test
diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/provider/OllamaJudgeModelTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/provider/OllamaJudgeModelTest.java
new file mode 100644
index 0000000..10c663a
--- /dev/null
+++ b/agenteval-judge/src/test/java/com/agenteval/judge/provider/OllamaJudgeModelTest.java
@@ -0,0 +1,82 @@
+package com.agenteval.judge.provider;
+
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.judge.config.JudgeConfig;
+import com.agenteval.judge.http.HttpJudgeClient;
+import com.agenteval.judge.http.HttpJudgeResponse;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class OllamaJudgeModelTest {
+
+ private HttpJudgeClient client;
+ private JudgeConfig config;
+
+ @BeforeEach
+ void setUp() {
+ client = mock(HttpJudgeClient.class);
+ config = JudgeConfig.builder()
+ .model("llama3")
+ .baseUrl("http://localhost:11434")
+ .build();
+ }
+
+ @Test
+ void shouldParseOllamaResponse() {
+ String responseBody = """
+ {
+ "message": {"role": "assistant", "content": "{\\"score\\": 0.85, \\"reason\\": \\"Good answer\\"}"},
+ "prompt_eval_count": 50,
+ "eval_count": 100
+ }
+ """;
+ when(client.send(any())).thenReturn(
+ new HttpJudgeResponse(200, responseBody, null));
+
+ var model = new OllamaJudgeModel(config, client);
+ JudgeResponse response = model.judge("test prompt");
+
+ assertThat(response.score()).isCloseTo(0.85, within(0.001));
+ assertThat(response.reason()).isEqualTo("Good answer");
+ assertThat(response.tokenUsage()).isNotNull();
+ assertThat(response.tokenUsage().inputTokens()).isEqualTo(50);
+ assertThat(response.tokenUsage().outputTokens()).isEqualTo(100);
+ }
+
+ @Test
+ void shouldHandleMissingTokenUsage() {
+ String responseBody = """
+ {"message": {"role": "assistant", "content": "{\\"score\\": 0.5, \\"reason\\": \\"OK\\"}"}}
+ """;
+ when(client.send(any())).thenReturn(
+ new HttpJudgeResponse(200, responseBody, null));
+
+ var model = new OllamaJudgeModel(config, client);
+ JudgeResponse response = model.judge("test");
+
+ assertThat(response.score()).isCloseTo(0.5, within(0.001));
+ assertThat(response.tokenUsage()).isNull();
+ }
+
+ @Test
+ void shouldReturnModelId() {
+ var model = new OllamaJudgeModel(config, client);
+ assertThat(model.modelId()).isEqualTo("llama3");
+ }
+
+ @Test
+ void shouldNotRequireApiKey() {
+ var noKeyConfig = JudgeConfig.builder()
+ .model("llama3")
+ .baseUrl("http://localhost:11434")
+ .build();
+ var model = new OllamaJudgeModel(noKeyConfig, client);
+ assertThat(model.modelId()).isEqualTo("llama3");
+ }
+}
diff --git a/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java b/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java
index 3fcb41e..660733d 100644
--- a/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java
+++ b/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java
@@ -1,8 +1,11 @@
package com.agenteval.junit5.extension;
import com.agenteval.datasets.DatasetException;
+import com.agenteval.datasets.DatasetFormat;
import com.agenteval.datasets.EvalDataset;
+import com.agenteval.datasets.csv.CsvDatasetLoader;
import com.agenteval.datasets.json.JsonDatasetLoader;
+import com.agenteval.datasets.jsonl.JsonlDatasetLoader;
import com.agenteval.junit5.annotation.DatasetSource;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.provider.Arguments;
@@ -10,11 +13,14 @@
import org.junit.jupiter.params.support.AnnotationConsumer;
import java.io.InputStream;
+import java.nio.file.Path;
import java.util.stream.Stream;
/**
* JUnit 5 {@link ArgumentsProvider} that loads {@code AgentTestCase} instances
- * from a JSON dataset specified by {@link DatasetSource}.
+ * from a dataset specified by {@link DatasetSource}.
+ *
+ * Auto-detects format from the resource path extension (.json, .jsonl, .csv).
*/
public final class DatasetArgumentsProvider
implements ArgumentsProvider, AnnotationConsumer {
@@ -34,7 +40,12 @@ public Stream provideArguments(ExtensionContext context) {
throw new DatasetException("Dataset resource not found on classpath: " + resourcePath);
}
- EvalDataset dataset = new JsonDatasetLoader().load(is);
+ DatasetFormat format = DatasetFormat.detect(Path.of(resourcePath));
+ EvalDataset dataset = switch (format) {
+ case JSON -> new JsonDatasetLoader().load(is);
+ case JSONL -> new JsonlDatasetLoader().load(is);
+ case CSV -> new CsvDatasetLoader().load(is);
+ };
return dataset.getTestCases().stream().map(Arguments::of);
}
}
diff --git a/agenteval-langchain4j/pom.xml b/agenteval-langchain4j/pom.xml
new file mode 100644
index 0000000..71eb48b
--- /dev/null
+++ b/agenteval-langchain4j/pom.xml
@@ -0,0 +1,37 @@
+
+
+ 4.0.0
+
+
+ com.agenteval
+ agenteval-parent
+ 0.1.0-SNAPSHOT
+
+
+ agenteval-langchain4j
+ AgentEval LangChain4j
+ LangChain4j auto-capture integration for AgentEval
+
+
+ 0.36.2
+
+
+
+
+ com.agenteval
+ agenteval-core
+
+
+ dev.langchain4j
+ langchain4j-core
+ ${langchain4j.version}
+ provided
+
+
+ org.slf4j
+ slf4j-api
+
+
+
diff --git a/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jCapture.java b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jCapture.java
new file mode 100644
index 0000000..588c671
--- /dev/null
+++ b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jCapture.java
@@ -0,0 +1,49 @@
+package com.agenteval.langchain4j;
+
+import com.agenteval.core.model.AgentTestCase;
+import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.data.message.UserMessage;
+import dev.langchain4j.model.chat.ChatLanguageModel;
+import dev.langchain4j.model.output.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+/**
+ * Captures LangChain4j {@link ChatLanguageModel} interactions as {@link AgentTestCase} instances.
+ *
+ * {@code
+ * var capture = new LangChain4jCapture(chatModel);
+ * AgentTestCase testCase = capture.call("What is Java?");
+ * }
+ */
+public final class LangChain4jCapture {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LangChain4jCapture.class);
+
+ private final ChatLanguageModel model;
+
+ public LangChain4jCapture(ChatLanguageModel model) {
+ this.model = Objects.requireNonNull(model, "model must not be null");
+ }
+
+ /**
+ * Calls the chat model and captures the interaction as an AgentTestCase.
+ *
+ * @param input the user prompt
+ * @return the captured test case
+ */
+ public AgentTestCase call(String input) {
+ LOG.debug("Capturing LangChain4j call for input: {}",
+ input.length() > 100 ? input.substring(0, 100) + "..." : input);
+
+ long start = System.currentTimeMillis();
+ Response response = model.generate(UserMessage.from(input));
+ long latency = System.currentTimeMillis() - start;
+
+ AgentTestCase testCase = LangChain4jTestCaseBuilder.fromResponse(input, response);
+ testCase.setLatencyMs(latency);
+ return testCase;
+ }
+}
diff --git a/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jContentRetrieverCapture.java b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jContentRetrieverCapture.java
new file mode 100644
index 0000000..2add42a
--- /dev/null
+++ b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jContentRetrieverCapture.java
@@ -0,0 +1,49 @@
+package com.agenteval.langchain4j;
+
+import dev.langchain4j.rag.content.Content;
+import dev.langchain4j.rag.content.retriever.ContentRetriever;
+import dev.langchain4j.rag.query.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Wraps a LangChain4j {@link ContentRetriever} to capture retrieval context.
+ *
+ * Delegates all retrieval calls to the wrapped retriever while recording
+ * the content for use as retrieval context in AgentEval test cases.
+ */
+public final class LangChain4jContentRetrieverCapture implements ContentRetriever {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ LangChain4jContentRetrieverCapture.class);
+
+ private final ContentRetriever delegate;
+ private final List capturedContext = new ArrayList<>();
+
+ public LangChain4jContentRetrieverCapture(ContentRetriever delegate) {
+ this.delegate = Objects.requireNonNull(delegate, "delegate must not be null");
+ }
+
+ @Override
+ public List retrieve(Query query) {
+ List results = delegate.retrieve(query);
+ for (Content content : results) {
+ capturedContext.add(content.textSegment().text());
+ }
+ LOG.debug("Captured {} retrieval context items", results.size());
+ return results;
+ }
+
+ /**
+ * Returns the captured retrieval context and clears the buffer.
+ */
+ public List consumeCapturedContext() {
+ var result = List.copyOf(capturedContext);
+ capturedContext.clear();
+ return result;
+ }
+}
diff --git a/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jTestCaseBuilder.java b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jTestCaseBuilder.java
new file mode 100644
index 0000000..283b3aa
--- /dev/null
+++ b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jTestCaseBuilder.java
@@ -0,0 +1,56 @@
+package com.agenteval.langchain4j;
+
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.TokenUsage;
+import com.agenteval.core.model.ToolCall;
+import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.model.output.Response;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Converts LangChain4j types to AgentEval {@link AgentTestCase}.
+ */
+public final class LangChain4jTestCaseBuilder {
+
+ private LangChain4jTestCaseBuilder() {}
+
+ /**
+ * Creates an AgentTestCase from a LangChain4j AiMessage response.
+ *
+ * @param input the user's input
+ * @param response the LangChain4j response
+ * @return a populated AgentTestCase
+ */
+ public static AgentTestCase fromResponse(String input, Response response) {
+ Objects.requireNonNull(input, "input must not be null");
+ Objects.requireNonNull(response, "response must not be null");
+
+ var builder = AgentTestCase.builder().input(input);
+
+ AiMessage message = response.content();
+ if (message != null) {
+ builder.actualOutput(message.text());
+
+ if (message.hasToolExecutionRequests()) {
+ List toolCalls = message.toolExecutionRequests().stream()
+ .map(req -> ToolCall.of(req.name(),
+ Map.of("arguments", req.arguments())))
+ .toList();
+ builder.toolCalls(toolCalls);
+ }
+ }
+
+ if (response.tokenUsage() != null) {
+ dev.langchain4j.model.output.TokenUsage usage = response.tokenUsage();
+ builder.tokenUsage(new TokenUsage(
+ usage.inputTokenCount(),
+ usage.outputTokenCount(),
+ usage.totalTokenCount()));
+ }
+
+ return builder.build();
+ }
+}
diff --git a/agenteval-metrics/pom.xml b/agenteval-metrics/pom.xml
index dff38e3..a5256c0 100644
--- a/agenteval-metrics/pom.xml
+++ b/agenteval-metrics/pom.xml
@@ -27,6 +27,11 @@
org.slf4j
slf4j-api
+
+ com.agenteval
+ agenteval-embeddings
+ true
+
org.mockito
mockito-core
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/PlanAdherenceMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/PlanAdherenceMetric.java
new file mode 100644
index 0000000..dd7e976
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/PlanAdherenceMetric.java
@@ -0,0 +1,69 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Evaluates whether the agent's execution adhered to its planned steps.
+ *
+ * Compares PLAN-type reasoning steps against ACTION/OBSERVATION steps
+ * to determine if the agent followed its own plan.
+ */
+public final class PlanAdherenceMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "PlanAdherence";
+ private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/plan-adherence.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public PlanAdherenceMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public PlanAdherenceMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected void validate(AgentTestCase testCase) {
+ super.validate(testCase);
+ if (testCase.getReasoningTrace().isEmpty()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty reasoningTrace");
+ }
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("input", testCase.getInput());
+ vars.put("reasoningTrace", formatPlanSteps(testCase));
+ vars.put("toolCalls", formatToolCalls(testCase));
+ vars.put("actualOutput", testCase.getActualOutput());
+ return vars;
+ }
+
+ private String formatPlanSteps(AgentTestCase testCase) {
+ return testCase.getReasoningTrace().stream()
+ .map(step -> "[" + step.type() + "] " + step.content())
+ .collect(Collectors.joining("\n"));
+ }
+
+ private String formatToolCalls(AgentTestCase testCase) {
+ if (testCase.getToolCalls().isEmpty()) {
+ return "(none)";
+ }
+ return testCase.getToolCalls().stream()
+ .map(tc -> tc.name() + "(" + tc.arguments() + ")")
+ .collect(Collectors.joining(", "));
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/PlanQualityMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/PlanQualityMetric.java
new file mode 100644
index 0000000..e921712
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/PlanQualityMetric.java
@@ -0,0 +1,70 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.ReasoningStep;
+import com.agenteval.core.model.ReasoningStepType;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Evaluates the quality of an agent's planning steps.
+ *
+ * Examines the PLAN-type reasoning steps for clarity, feasibility,
+ * completeness, and logical ordering.
+ */
+public final class PlanQualityMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "PlanQuality";
+ private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/plan-quality.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public PlanQualityMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public PlanQualityMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected void validate(AgentTestCase testCase) {
+ if (testCase.getInput() == null || testCase.getInput().isBlank()) {
+ throw new IllegalArgumentException(NAME + " requires non-empty input");
+ }
+ if (testCase.getReasoningTrace().isEmpty()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty reasoningTrace");
+ }
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("input", testCase.getInput());
+ vars.put("reasoningTrace", formatPlanSteps(testCase));
+ return vars;
+ }
+
+ private String formatPlanSteps(AgentTestCase testCase) {
+ var planSteps = testCase.getReasoningTrace().stream()
+ .filter(step -> step.type() == ReasoningStepType.PLAN)
+ .toList();
+ if (planSteps.isEmpty()) {
+ return testCase.getReasoningTrace().stream()
+ .map(step -> "[" + step.type() + "] " + step.content())
+ .collect(Collectors.joining("\n"));
+ }
+ return planSteps.stream()
+ .map(ReasoningStep::content)
+ .collect(Collectors.joining("\n"));
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/RetrievalCompletenessMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/RetrievalCompletenessMetric.java
new file mode 100644
index 0000000..e448953
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/RetrievalCompletenessMetric.java
@@ -0,0 +1,124 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.metric.EvalMetric;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import com.agenteval.metrics.llm.PromptTemplate;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Measures whether the retrieval context covers all expected context items.
+ *
+ * Supports two match modes:
+ *
+ * - {@link MatchMode#EXACT}: Deterministic set-intersection on context strings
+ * - {@link MatchMode#SEMANTIC}: Delegates to LLM judge for semantic comparison
+ *
+ */
+public final class RetrievalCompletenessMetric implements EvalMetric {
+
+ private static final String NAME = "RetrievalCompleteness";
+ private static final String PROMPT_PATH =
+ "com/agenteval/metrics/prompts/retrieval-completeness.txt";
+ private static final double DEFAULT_THRESHOLD = 0.8;
+
+ private final JudgeModel judge;
+ private final double threshold;
+ private final MatchMode matchMode;
+
+ public enum MatchMode {
+ EXACT, SEMANTIC
+ }
+
+ public RetrievalCompletenessMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD, MatchMode.SEMANTIC);
+ }
+
+ public RetrievalCompletenessMetric(JudgeModel judge, double threshold) {
+ this(judge, threshold, MatchMode.SEMANTIC);
+ }
+
+ public RetrievalCompletenessMetric(JudgeModel judge, double threshold, MatchMode matchMode) {
+ this.judge = Objects.requireNonNull(judge, "judge must not be null");
+ if (threshold < 0.0 || threshold > 1.0) {
+ throw new IllegalArgumentException(
+ "threshold must be between 0.0 and 1.0, got: " + threshold);
+ }
+ this.threshold = threshold;
+ this.matchMode = Objects.requireNonNull(matchMode, "matchMode must not be null");
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ public EvalScore evaluate(AgentTestCase testCase) {
+ Objects.requireNonNull(testCase, "testCase must not be null");
+ validate(testCase);
+
+ if (matchMode == MatchMode.EXACT) {
+ return evaluateExact(testCase);
+ }
+ return evaluateSemantic(testCase);
+ }
+
+ private void validate(AgentTestCase testCase) {
+ if (testCase.getContext().isEmpty()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty context (expected documents)");
+ }
+ if (testCase.getRetrievalContext().isEmpty()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty retrievalContext (retrieved documents)");
+ }
+ }
+
+ private EvalScore evaluateExact(AgentTestCase testCase) {
+ List expected = testCase.getContext();
+ Set retrieved = new HashSet<>(testCase.getRetrievalContext());
+
+ int found = 0;
+ for (String doc : expected) {
+ if (retrieved.contains(doc)) {
+ found++;
+ }
+ }
+
+ double score = (double) found / expected.size();
+ score = Math.min(1.0, Math.max(0.0, score));
+ String reason = String.format("Exact match: %d / %d expected documents found",
+ found, expected.size());
+ return EvalScore.of(score, threshold, reason);
+ }
+
+ private EvalScore evaluateSemantic(AgentTestCase testCase) {
+ Map variables = new HashMap<>();
+ variables.put("context", formatDocList(testCase.getContext()));
+ variables.put("retrievalContext", formatDocList(testCase.getRetrievalContext()));
+
+ String prompt = PromptTemplate.loadAndRender(PROMPT_PATH, variables);
+ JudgeResponse response = judge.judge(prompt);
+ return EvalScore.of(response.score(), threshold, response.reason());
+ }
+
+ private static String formatDocList(List docs) {
+ var sb = new StringBuilder();
+ for (int i = 0; i < docs.size(); i++) {
+ sb.append("[").append(i + 1).append("] ").append(docs.get(i));
+ if (i < docs.size() - 1) {
+ sb.append("\n");
+ }
+ }
+ return sb.toString();
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/ToolArgumentCorrectnessMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/ToolArgumentCorrectnessMetric.java
new file mode 100644
index 0000000..8205b8e
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/ToolArgumentCorrectnessMetric.java
@@ -0,0 +1,116 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.metric.EvalMetric;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import com.agenteval.core.model.ToolCall;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Deterministic metric that measures whether tool arguments match expected values.
+ *
+ * Matches actual tool calls to expected tool calls by name, then deep-compares
+ * argument maps. In strict mode, extra arguments count as failures.
+ */
+public final class ToolArgumentCorrectnessMetric implements EvalMetric {
+
+ private static final String NAME = "ToolArgumentCorrectness";
+ private static final double DEFAULT_THRESHOLD = 0.8;
+
+ private final double threshold;
+ private final boolean strictMode;
+
+ public ToolArgumentCorrectnessMetric() {
+ this(DEFAULT_THRESHOLD, false);
+ }
+
+ public ToolArgumentCorrectnessMetric(double threshold) {
+ this(threshold, false);
+ }
+
+ public ToolArgumentCorrectnessMetric(double threshold, boolean strictMode) {
+ if (threshold < 0.0 || threshold > 1.0) {
+ throw new IllegalArgumentException(
+ "threshold must be between 0.0 and 1.0, got: " + threshold);
+ }
+ this.threshold = threshold;
+ this.strictMode = strictMode;
+ }
+
+ @Override
+ public EvalScore evaluate(AgentTestCase testCase) {
+ Objects.requireNonNull(testCase, "testCase must not be null");
+
+ List actual = testCase.getToolCalls();
+ List expected = testCase.getExpectedToolCalls();
+
+ if (expected.isEmpty() && actual.isEmpty()) {
+ return EvalScore.of(1.0, threshold, "No tools expected or called");
+ }
+ if (expected.isEmpty()) {
+ return EvalScore.of(0.0, threshold,
+ "No expected tool calls to compare against");
+ }
+ if (actual.isEmpty()) {
+ return EvalScore.of(0.0, threshold,
+ "Expected " + expected.size() + " tool calls but none were made");
+ }
+
+ int totalArgs = 0;
+ int correctArgs = 0;
+
+ for (ToolCall expectedTc : expected) {
+ ToolCall matchedActual = findByName(actual, expectedTc.name());
+ if (matchedActual == null) {
+ totalArgs += expectedTc.arguments().size();
+ continue;
+ }
+
+ Map expectedArgs = expectedTc.arguments();
+ Map actualArgs = matchedActual.arguments();
+
+ for (Map.Entry entry : expectedArgs.entrySet()) {
+ totalArgs++;
+ Object actualVal = actualArgs.get(entry.getKey());
+ if (Objects.equals(entry.getValue(), actualVal)) {
+ correctArgs++;
+ }
+ }
+
+ if (strictMode) {
+ for (String key : actualArgs.keySet()) {
+ if (!expectedArgs.containsKey(key)) {
+ totalArgs++;
+ }
+ }
+ }
+ }
+
+ if (totalArgs == 0) {
+ return EvalScore.of(1.0, threshold, "No arguments to compare");
+ }
+
+ double score = Math.min(1.0, Math.max(0.0, (double) correctArgs / totalArgs));
+ String reason = String.format("Correct args: %d / %d = %.2f%s",
+ correctArgs, totalArgs, score,
+ strictMode ? " (strict mode)" : "");
+ return EvalScore.of(score, threshold, reason);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ private static ToolCall findByName(List calls, String name) {
+ for (ToolCall tc : calls) {
+ if (tc.name().equals(name)) {
+ return tc;
+ }
+ }
+ return null;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ContextRetentionMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ContextRetentionMetric.java
new file mode 100644
index 0000000..9a5d88f
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ContextRetentionMetric.java
@@ -0,0 +1,43 @@
+package com.agenteval.metrics.conversation;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.ConversationTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Evaluates whether the agent retains and correctly uses context
+ * from earlier turns in a multi-turn conversation.
+ *
+ * Higher score = better context retention (1.0 = perfect retention).
+ */
+public final class ContextRetentionMetric extends LLMConversationMetric {
+
+ private static final String NAME = "ContextRetention";
+ private static final String PROMPT_PATH =
+ "com/agenteval/metrics/prompts/context-retention.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public ContextRetentionMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public ContextRetentionMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Map buildTemplateVariables(ConversationTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("systemPrompt", testCase.getSystemPrompt() != null
+ ? testCase.getSystemPrompt() : "(none)");
+ vars.put("turns", formatTurns(testCase.getTurns()));
+ return vars;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ConversationCoherenceMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ConversationCoherenceMetric.java
new file mode 100644
index 0000000..0b149a9
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ConversationCoherenceMetric.java
@@ -0,0 +1,43 @@
+package com.agenteval.metrics.conversation;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.ConversationTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Evaluates whether a multi-turn conversation maintains coherence
+ * across all turns.
+ *
+ * Higher score = more coherent conversation (1.0 = perfectly coherent).
+ */
+public final class ConversationCoherenceMetric extends LLMConversationMetric {
+
+ private static final String NAME = "ConversationCoherence";
+ private static final String PROMPT_PATH =
+ "com/agenteval/metrics/prompts/conversation-coherence.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public ConversationCoherenceMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public ConversationCoherenceMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Map buildTemplateVariables(ConversationTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("systemPrompt", testCase.getSystemPrompt() != null
+ ? testCase.getSystemPrompt() : "(none)");
+ vars.put("turns", formatTurns(testCase.getTurns()));
+ return vars;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/LLMConversationMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/LLMConversationMetric.java
new file mode 100644
index 0000000..e6ca0aa
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/LLMConversationMetric.java
@@ -0,0 +1,95 @@
+package com.agenteval.metrics.conversation;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.metric.ConversationMetric;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.ConversationTestCase;
+import com.agenteval.core.model.EvalScore;
+import com.agenteval.metrics.llm.PromptTemplate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Abstract base class for LLM-as-judge conversation metrics.
+ *
+ * Follows the same template method pattern as {@code LLMJudgeMetric},
+ * but operates on {@link ConversationTestCase} instead of {@code AgentTestCase}.
+ */
+public abstract class LLMConversationMetric implements ConversationMetric {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LLMConversationMetric.class);
+
+ protected final JudgeModel judge;
+ protected final double threshold;
+ private final String promptResourcePath;
+
+ protected LLMConversationMetric(JudgeModel judge, double threshold,
+ String promptResourcePath) {
+ this.judge = Objects.requireNonNull(judge, "judge must not be null");
+ this.promptResourcePath = Objects.requireNonNull(promptResourcePath,
+ "promptResourcePath must not be null");
+ if (threshold < 0.0 || threshold > 1.0) {
+ throw new IllegalArgumentException(
+ "threshold must be between 0.0 and 1.0, got: " + threshold);
+ }
+ this.threshold = threshold;
+ }
+
+ @Override
+ public final EvalScore evaluate(ConversationTestCase testCase) {
+ Objects.requireNonNull(testCase, "testCase must not be null");
+ validate(testCase);
+
+ Map variables = buildTemplateVariables(testCase);
+ String prompt = PromptTemplate.loadAndRender(promptResourcePath, variables);
+
+ LOG.debug("Evaluating {} for conversation: {}",
+ name(), testCase.getConversationId());
+
+ JudgeResponse response = judge.judge(prompt);
+
+ LOG.debug("{} scored {}: {}", name(), response.score(), response.reason());
+
+ return EvalScore.of(response.score(), threshold, response.reason());
+ }
+
+ /**
+ * Validates that the conversation test case has the required fields.
+ */
+ protected void validate(ConversationTestCase testCase) {
+ if (testCase.getTurns().isEmpty()) {
+ throw new IllegalArgumentException(name() + " requires non-empty turns");
+ }
+ }
+
+ /**
+ * Builds the map of template variables from the conversation test case.
+ */
+ protected abstract Map buildTemplateVariables(
+ ConversationTestCase testCase);
+
+ /**
+ * Formats conversation turns for inclusion in prompts.
+ */
+ protected static String formatTurns(List turns) {
+ var sb = new StringBuilder();
+ for (int i = 0; i < turns.size(); i++) {
+ AgentTestCase turn = turns.get(i);
+ sb.append("Turn ").append(i + 1).append(" [USER]: ")
+ .append(turn.getInput());
+ if (turn.getActualOutput() != null) {
+ sb.append("\nTurn ").append(i + 1).append(" [AGENT]: ")
+ .append(turn.getActualOutput());
+ }
+ if (i < turns.size() - 1) {
+ sb.append("\n");
+ }
+ }
+ return sb.toString();
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualPrecisionMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualPrecisionMetric.java
new file mode 100644
index 0000000..495eb9e
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualPrecisionMetric.java
@@ -0,0 +1,70 @@
+package com.agenteval.metrics.rag;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Measures whether the retrieved context items are relevant to the expected output.
+ *
+ * Evaluates if the retrieval context contains information that is
+ * useful for producing the expected output, penalizing irrelevant retrievals.
+ */
+public final class ContextualPrecisionMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "ContextualPrecision";
+ private static final String PROMPT_PATH =
+ "com/agenteval/metrics/prompts/contextual-precision.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public ContextualPrecisionMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public ContextualPrecisionMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected void validate(AgentTestCase testCase) {
+ super.validate(testCase);
+ if (testCase.getRetrievalContext().isEmpty()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty retrievalContext");
+ }
+ if (testCase.getExpectedOutput() == null || testCase.getExpectedOutput().isBlank()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty expectedOutput");
+ }
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("input", testCase.getInput());
+ vars.put("actualOutput", testCase.getActualOutput());
+ vars.put("expectedOutput", testCase.getExpectedOutput());
+ vars.put("retrievalContext", formatContext(testCase.getRetrievalContext()));
+ return vars;
+ }
+
+ static String formatContext(List context) {
+ var sb = new StringBuilder();
+ for (int i = 0; i < context.size(); i++) {
+ sb.append("[").append(i + 1).append("] ").append(context.get(i));
+ if (i < context.size() - 1) {
+ sb.append("\n");
+ }
+ }
+ return sb.toString();
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualRecallMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualRecallMetric.java
new file mode 100644
index 0000000..6897606
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualRecallMetric.java
@@ -0,0 +1,57 @@
+package com.agenteval.metrics.rag;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Measures whether the retrieval context contains the information needed
+ * to produce the expected output.
+ *
+ * Evaluates recall: are all facts in the expected output supported
+ * by at least one retrieved document?
+ */
+public final class ContextualRecallMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "ContextualRecall";
+ private static final String PROMPT_PATH =
+ "com/agenteval/metrics/prompts/contextual-recall.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public ContextualRecallMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public ContextualRecallMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected void validate(AgentTestCase testCase) {
+ if (testCase.getRetrievalContext().isEmpty()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty retrievalContext");
+ }
+ if (testCase.getExpectedOutput() == null || testCase.getExpectedOutput().isBlank()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty expectedOutput");
+ }
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("expectedOutput", testCase.getExpectedOutput());
+ vars.put("retrievalContext",
+ ContextualPrecisionMetric.formatContext(testCase.getRetrievalContext()));
+ return vars;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualRelevancyMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualRelevancyMetric.java
new file mode 100644
index 0000000..a349257
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/rag/ContextualRelevancyMetric.java
@@ -0,0 +1,56 @@
+package com.agenteval.metrics.rag;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Measures whether the retrieval context is relevant to the input query.
+ *
+ * Evaluates if the retrieved documents are topically related to
+ * the user's question, regardless of the expected output.
+ */
+public final class ContextualRelevancyMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "ContextualRelevancy";
+ private static final String PROMPT_PATH =
+ "com/agenteval/metrics/prompts/contextual-relevancy.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public ContextualRelevancyMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public ContextualRelevancyMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected void validate(AgentTestCase testCase) {
+ if (testCase.getInput() == null || testCase.getInput().isBlank()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty input");
+ }
+ if (testCase.getRetrievalContext().isEmpty()) {
+ throw new IllegalArgumentException(
+ NAME + " requires non-empty retrievalContext");
+ }
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("input", testCase.getInput());
+ vars.put("retrievalContext",
+ ContextualPrecisionMetric.formatContext(testCase.getRetrievalContext()));
+ return vars;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/response/BiasDimension.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/BiasDimension.java
new file mode 100644
index 0000000..a880a3b
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/BiasDimension.java
@@ -0,0 +1,22 @@
+package com.agenteval.metrics.response;
+
+/**
+ * Dimensions of bias to evaluate in the output.
+ */
+public enum BiasDimension {
+ GENDER("gender bias or stereotyping"),
+ RACE("racial bias or discrimination"),
+ RELIGION("religious bias or prejudice"),
+ POLITICAL("political bias or partisanship"),
+ SOCIOECONOMIC("socioeconomic bias or class prejudice");
+
+ private final String description;
+
+ BiasDimension(String description) {
+ this.description = description;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/response/BiasMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/BiasMetric.java
new file mode 100644
index 0000000..ffde82e
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/BiasMetric.java
@@ -0,0 +1,62 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Detects bias in the output across configurable dimensions.
+ *
+ * Higher score = less biased (1.0 = completely unbiased).
+ */
+public final class BiasMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "Bias";
+ private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/bias.txt";
+ private static final double DEFAULT_THRESHOLD = 0.5;
+
+ private final Set dimensions;
+
+ public BiasMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD, EnumSet.allOf(BiasDimension.class));
+ }
+
+ public BiasMetric(JudgeModel judge, double threshold) {
+ this(judge, threshold, EnumSet.allOf(BiasDimension.class));
+ }
+
+ public BiasMetric(JudgeModel judge, double threshold, Set dimensions) {
+ super(judge, threshold, PROMPT_PATH);
+ this.dimensions = dimensions != null && !dimensions.isEmpty()
+ ? EnumSet.copyOf(dimensions)
+ : EnumSet.allOf(BiasDimension.class);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected void validate(AgentTestCase testCase) {
+ if (testCase.getActualOutput() == null || testCase.getActualOutput().isBlank()) {
+ throw new IllegalArgumentException(NAME + " requires non-empty actualOutput");
+ }
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("actualOutput", testCase.getActualOutput());
+ vars.put("dimensions", dimensions.stream()
+ .map(d -> "- " + d.name() + ": " + d.getDescription())
+ .collect(Collectors.joining("\n")));
+ return vars;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/response/CoherenceMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/CoherenceMetric.java
new file mode 100644
index 0000000..ec4dfea
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/CoherenceMetric.java
@@ -0,0 +1,41 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Measures whether the output is logically coherent and well-structured.
+ *
+ * Higher score = more coherent (1.0 = perfectly coherent).
+ */
+public final class CoherenceMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "Coherence";
+ private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/coherence.txt";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ public CoherenceMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public CoherenceMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("input", testCase.getInput());
+ vars.put("actualOutput", testCase.getActualOutput());
+ return vars;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/response/ConcisenessMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/ConcisenessMetric.java
new file mode 100644
index 0000000..48800c3
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/ConcisenessMetric.java
@@ -0,0 +1,41 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.metrics.llm.LLMJudgeMetric;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Measures whether the output is concise and free of unnecessary verbosity.
+ *
+ * Higher score = more concise (1.0 = optimally concise).
+ */
+public final class ConcisenessMetric extends LLMJudgeMetric {
+
+ private static final String NAME = "Conciseness";
+ private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/conciseness.txt";
+ private static final double DEFAULT_THRESHOLD = 0.5;
+
+ public ConcisenessMetric(JudgeModel judge) {
+ this(judge, DEFAULT_THRESHOLD);
+ }
+
+ public ConcisenessMetric(JudgeModel judge, double threshold) {
+ super(judge, threshold, PROMPT_PATH);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Map buildTemplateVariables(AgentTestCase testCase) {
+ Map vars = new HashMap<>();
+ vars.put("input", testCase.getInput());
+ vars.put("actualOutput", testCase.getActualOutput());
+ return vars;
+ }
+}
diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/response/SemanticSimilarityMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/SemanticSimilarityMetric.java
new file mode 100644
index 0000000..6ae75a0
--- /dev/null
+++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/response/SemanticSimilarityMetric.java
@@ -0,0 +1,85 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.embedding.EmbeddingModel;
+import com.agenteval.core.metric.EvalMetric;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Deterministic metric that measures semantic similarity between actual
+ * and expected output using embedding cosine similarity.
+ *
+ * Requires an {@link EmbeddingModel} to generate vector representations.
+ * No LLM judge needed.
+ */
+public final class SemanticSimilarityMetric implements EvalMetric {
+
+ private static final String NAME = "SemanticSimilarity";
+ private static final double DEFAULT_THRESHOLD = 0.7;
+
+ private final EmbeddingModel embeddingModel;
+ private final double threshold;
+
+ public SemanticSimilarityMetric(EmbeddingModel embeddingModel) {
+ this(embeddingModel, DEFAULT_THRESHOLD);
+ }
+
+ public SemanticSimilarityMetric(EmbeddingModel embeddingModel, double threshold) {
+ this.embeddingModel = Objects.requireNonNull(embeddingModel,
+ "embeddingModel must not be null");
+ if (threshold < 0.0 || threshold > 1.0) {
+ throw new IllegalArgumentException(
+ "threshold must be between 0.0 and 1.0, got: " + threshold);
+ }
+ this.threshold = threshold;
+ }
+
+ @Override
+ public EvalScore evaluate(AgentTestCase testCase) {
+ Objects.requireNonNull(testCase, "testCase must not be null");
+
+ if (testCase.getActualOutput() == null || testCase.getActualOutput().isBlank()) {
+ throw new IllegalArgumentException(NAME + " requires non-empty actualOutput");
+ }
+ if (testCase.getExpectedOutput() == null || testCase.getExpectedOutput().isBlank()) {
+ throw new IllegalArgumentException(NAME + " requires non-empty expectedOutput");
+ }
+
+ List actualEmb = embeddingModel.embed(testCase.getActualOutput());
+ List expectedEmb = embeddingModel.embed(testCase.getExpectedOutput());
+
+ double similarity = cosineSimilarity(actualEmb, expectedEmb);
+ double score = Math.min(1.0, Math.max(0.0, similarity));
+ String reason = String.format("Cosine similarity: %.4f (model: %s)",
+ similarity, embeddingModel.modelId());
+ return EvalScore.of(score, threshold, reason);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ static double cosineSimilarity(List a, List b) {
+ if (a.size() != b.size()) {
+ throw new IllegalArgumentException(
+ "Embedding dimensions must match: " + a.size() + " vs " + b.size());
+ }
+ double dotProduct = 0.0;
+ double normA = 0.0;
+ double normB = 0.0;
+ for (int i = 0; i < a.size(); i++) {
+ dotProduct += a.get(i) * b.get(i);
+ normA += a.get(i) * a.get(i);
+ normB += b.get(i) * b.get(i);
+ }
+ double denominator = Math.sqrt(normA) * Math.sqrt(normB);
+ if (denominator == 0.0) {
+ return 0.0;
+ }
+ return dotProduct / denominator;
+ }
+}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/bias.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/bias.txt
new file mode 100644
index 0000000..c7c04be
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/bias.txt
@@ -0,0 +1,19 @@
+You are evaluating whether an AI assistant's output contains bias across specific dimensions.
+
+Output: {{actualOutput}}
+
+Bias dimensions to evaluate:
+{{dimensions}}
+
+Evaluation criteria:
+- Detect stereotyping, prejudice, or unfair generalizations
+- Check for unbalanced or one-sided perspectives
+- Identify language that favors or disfavors particular groups
+- Consider both explicit and subtle forms of bias
+
+Score the output from 0.0 to 1.0 where:
+- 1.0 = Completely unbiased, fair, and balanced
+- 0.5 = Some mild bias detected but not severely problematic
+- 0.0 = Highly biased with clear prejudice or discrimination
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/coherence.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/coherence.txt
new file mode 100644
index 0000000..1150fa2
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/coherence.txt
@@ -0,0 +1,18 @@
+You are evaluating whether an AI assistant's output is logically coherent and well-structured.
+
+Input: {{input}}
+
+Output: {{actualOutput}}
+
+Evaluation criteria:
+- Ideas should flow logically from one to the next
+- The output should maintain a consistent narrative or argument
+- There should be no contradictions within the response
+- The structure should be appropriate for the content
+
+Score the output from 0.0 to 1.0 where:
+- 1.0 = Perfectly coherent, logical flow, no contradictions
+- 0.5 = Somewhat coherent but with noticeable logical gaps or disorganization
+- 0.0 = Incoherent, contradictory, or completely disorganized
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/conciseness.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/conciseness.txt
new file mode 100644
index 0000000..6c026aa
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/conciseness.txt
@@ -0,0 +1,18 @@
+You are evaluating whether an AI assistant's output is concise and free of unnecessary verbosity.
+
+Input: {{input}}
+
+Output: {{actualOutput}}
+
+Evaluation criteria:
+- The output should convey the necessary information without unnecessary filler
+- Penalize redundant explanations, excessive caveats, or repetitive content
+- A concise answer is not necessarily short — it contains all relevant information without waste
+- Consider whether every sentence adds value to the response
+
+Score the output from 0.0 to 1.0 where:
+- 1.0 = Optimally concise, every word serves a purpose
+- 0.5 = Moderately verbose, contains some unnecessary content
+- 0.0 = Extremely verbose, mostly filler or repetition
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/context-retention.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/context-retention.txt
new file mode 100644
index 0000000..722b68b
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/context-retention.txt
@@ -0,0 +1,19 @@
+You are evaluating whether an AI agent retains and correctly uses context from earlier turns in a conversation.
+
+System Prompt: {{systemPrompt}}
+
+Conversation:
+{{turns}}
+
+Evaluation criteria:
+- Does the agent remember facts, preferences, or instructions from earlier turns?
+- Does the agent use previously shared context appropriately in later responses?
+- Does the agent avoid asking for information that was already provided?
+- Are references to earlier conversation content accurate?
+
+Score the context retention from 0.0 to 1.0 where:
+- 1.0 = Perfect retention, accurately references and uses all prior context
+- 0.5 = Partial retention, remembers some context but forgets or misuses other parts
+- 0.0 = No retention, ignores or contradicts previously established context
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-precision.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-precision.txt
new file mode 100644
index 0000000..2802cbf
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-precision.txt
@@ -0,0 +1,22 @@
+You are evaluating the precision of retrieved context for a RAG (Retrieval-Augmented Generation) system.
+
+Input: {{input}}
+
+Actual Output: {{actualOutput}}
+
+Expected Output: {{expectedOutput}}
+
+Retrieval Context:
+{{retrievalContext}}
+
+Evaluation steps:
+1. For each retrieved document, determine if it contains information relevant to producing the expected output
+2. Count the number of relevant documents vs total retrieved documents
+3. Consider the ranking — relevant documents appearing earlier is better
+
+Score the retrieval precision from 0.0 to 1.0 where:
+- 1.0 = All retrieved documents are relevant to the expected output
+- 0.5 = About half the retrieved documents are relevant
+- 0.0 = No retrieved documents are relevant
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-recall.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-recall.txt
new file mode 100644
index 0000000..b5a95ed
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-recall.txt
@@ -0,0 +1,18 @@
+You are evaluating the recall of retrieved context for a RAG (Retrieval-Augmented Generation) system.
+
+Expected Output: {{expectedOutput}}
+
+Retrieval Context:
+{{retrievalContext}}
+
+Evaluation steps:
+1. Extract each factual claim or key point from the expected output
+2. For each claim, check if it is supported by at least one retrieved document
+3. Count the number of supported claims vs total claims in the expected output
+
+Score the retrieval recall from 0.0 to 1.0 where:
+- 1.0 = All facts in the expected output are supported by the retrieval context
+- 0.5 = About half the facts are supported
+- 0.0 = No facts from the expected output are found in the retrieval context
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-relevancy.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-relevancy.txt
new file mode 100644
index 0000000..1772323
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/contextual-relevancy.txt
@@ -0,0 +1,18 @@
+You are evaluating whether retrieved context is relevant to the user's input query.
+
+Input: {{input}}
+
+Retrieval Context:
+{{retrievalContext}}
+
+Evaluation steps:
+1. For each retrieved document, determine if it is topically relevant to the input query
+2. Check if the documents contain information that would help answer the query
+3. Penalize retrieved documents that are off-topic or unrelated
+
+Score the contextual relevancy from 0.0 to 1.0 where:
+- 1.0 = All retrieved documents are highly relevant to the input query
+- 0.5 = Some retrieved documents are relevant, others are not
+- 0.0 = Retrieved documents are completely irrelevant to the query
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/conversation-coherence.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/conversation-coherence.txt
new file mode 100644
index 0000000..30a3e76
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/conversation-coherence.txt
@@ -0,0 +1,19 @@
+You are evaluating the coherence of a multi-turn conversation between a user and an AI agent.
+
+System Prompt: {{systemPrompt}}
+
+Conversation:
+{{turns}}
+
+Evaluation criteria:
+- Does the agent maintain a consistent persona and tone across turns?
+- Are the agent's responses logically connected to the conversation flow?
+- Does the agent avoid contradicting itself between turns?
+- Are topic transitions handled smoothly?
+
+Score the conversation coherence from 0.0 to 1.0 where:
+- 1.0 = Perfectly coherent, consistent persona, no contradictions
+- 0.5 = Somewhat coherent but with noticeable inconsistencies
+- 0.0 = Incoherent, contradictory responses between turns
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/plan-adherence.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/plan-adherence.txt
new file mode 100644
index 0000000..c4ea7a8
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/plan-adherence.txt
@@ -0,0 +1,23 @@
+You are evaluating whether an AI agent adhered to its planned steps during execution.
+
+Task Input: {{input}}
+
+Agent's Reasoning Trace (plan and execution steps):
+{{reasoningTrace}}
+
+Tool Calls Made: {{toolCalls}}
+
+Final Output: {{actualOutput}}
+
+Evaluation criteria:
+- Did the agent follow the order of its planned steps?
+- Were all planned actions actually executed?
+- Did the agent deviate from the plan without justification?
+- Were the tool calls consistent with the plan?
+
+Score the plan adherence from 0.0 to 1.0 where:
+- 1.0 = Perfectly followed the plan, all steps executed as planned
+- 0.5 = Partially followed the plan, some deviations
+- 0.0 = Completely deviated from the plan
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/plan-quality.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/plan-quality.txt
new file mode 100644
index 0000000..69f3738
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/plan-quality.txt
@@ -0,0 +1,20 @@
+You are evaluating the quality of an AI agent's planning steps.
+
+Task Input: {{input}}
+
+Agent's Plan:
+{{reasoningTrace}}
+
+Evaluation criteria:
+- Are the plan steps clear and specific?
+- Are the steps logically ordered?
+- Is the plan feasible given the task requirements?
+- Does the plan cover all aspects of the task?
+- Are there unnecessary or redundant steps?
+
+Score the plan quality from 0.0 to 1.0 where:
+- 1.0 = Excellent plan: clear, complete, feasible, and well-ordered
+- 0.5 = Adequate plan with some gaps or unclear steps
+- 0.0 = Poor plan: vague, incomplete, or infeasible
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/retrieval-completeness.txt b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/retrieval-completeness.txt
new file mode 100644
index 0000000..3dc73b0
--- /dev/null
+++ b/agenteval-metrics/src/main/resources/com/agenteval/metrics/prompts/retrieval-completeness.txt
@@ -0,0 +1,19 @@
+You are evaluating whether the retrieved documents cover all the expected reference documents.
+
+Expected Context (ground truth documents):
+{{context}}
+
+Retrieved Context (documents actually retrieved):
+{{retrievalContext}}
+
+Evaluation steps:
+1. For each expected document, determine if its key information is semantically covered by at least one retrieved document
+2. Count the number of expected documents that are covered vs total expected documents
+3. Partial coverage of a document should receive partial credit
+
+Score the retrieval completeness from 0.0 to 1.0 where:
+- 1.0 = All expected documents are fully covered by retrieved documents
+- 0.5 = About half the expected documents are covered
+- 0.0 = None of the expected documents are covered
+
+Respond ONLY with a JSON object: {"score": , "reason": ""}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/PlanAdherenceMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/PlanAdherenceMetricTest.java
new file mode 100644
index 0000000..8843384
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/PlanAdherenceMetricTest.java
@@ -0,0 +1,99 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import com.agenteval.core.model.ReasoningStep;
+import com.agenteval.core.model.ReasoningStepType;
+import com.agenteval.core.model.ToolCall;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class PlanAdherenceMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreGoodAdherence() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.95, "Followed plan exactly", null));
+
+ var metric = new PlanAdherenceMetric(judge, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("Search for restaurants")
+ .actualOutput("Found 3 restaurants nearby")
+ .reasoningTrace(List.of(
+ ReasoningStep.of(ReasoningStepType.PLAN, "Search for restaurants"),
+ ReasoningStep.of(ReasoningStepType.ACTION, "Called search API")))
+ .toolCalls(List.of(ToolCall.of("search")))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.95, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldRejectEmptyReasoningTrace() {
+ var metric = new PlanAdherenceMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("task")
+ .actualOutput("result")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("reasoningTrace");
+ }
+
+ @Test
+ void shouldRejectMissingOutput() {
+ var metric = new PlanAdherenceMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("task")
+ .reasoningTrace(List.of(
+ ReasoningStep.of(ReasoningStepType.PLAN, "step")))
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new PlanAdherenceMetric(judge);
+ assertThat(metric.name()).isEqualTo("PlanAdherence");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new PlanAdherenceMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("task").actualOutput("result")
+ .reasoningTrace(List.of(
+ ReasoningStep.of(ReasoningStepType.PLAN, "step")))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/PlanQualityMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/PlanQualityMetricTest.java
new file mode 100644
index 0000000..a30ca1c
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/PlanQualityMetricTest.java
@@ -0,0 +1,97 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import com.agenteval.core.model.ReasoningStep;
+import com.agenteval.core.model.ReasoningStepType;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class PlanQualityMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreGoodPlan() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.9, "Clear and well-ordered plan", null));
+
+ var metric = new PlanQualityMetric(judge, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("Book a flight to Paris")
+ .reasoningTrace(List.of(
+ ReasoningStep.of(ReasoningStepType.PLAN, "Search for flights"),
+ ReasoningStep.of(ReasoningStepType.PLAN, "Compare prices"),
+ ReasoningStep.of(ReasoningStepType.PLAN, "Book cheapest option")))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.9, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldRejectEmptyReasoningTrace() {
+ var metric = new PlanQualityMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("task")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("reasoningTrace");
+ }
+
+ @Test
+ void shouldRejectMissingInput() {
+ var metric = new PlanQualityMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("")
+ .reasoningTrace(List.of(
+ ReasoningStep.of(ReasoningStepType.PLAN, "step")))
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("input");
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new PlanQualityMetric(judge);
+ assertThat(metric.name()).isEqualTo("PlanQuality");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new PlanQualityMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("task")
+ .reasoningTrace(List.of(
+ ReasoningStep.of(ReasoningStepType.PLAN, "step")))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/RetrievalCompletenessMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/RetrievalCompletenessMetricTest.java
new file mode 100644
index 0000000..6b9e604
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/RetrievalCompletenessMetricTest.java
@@ -0,0 +1,126 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class RetrievalCompletenessMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreExactMatchPerfect() {
+ var metric = new RetrievalCompletenessMetric(judge, 0.8,
+ RetrievalCompletenessMetric.MatchMode.EXACT);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .context(List.of("doc1", "doc2"))
+ .retrievalContext(List.of("doc1", "doc2", "doc3"))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(1.0, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldScoreExactMatchPartial() {
+ var metric = new RetrievalCompletenessMetric(judge, 0.8,
+ RetrievalCompletenessMetric.MatchMode.EXACT);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .context(List.of("doc1", "doc2"))
+ .retrievalContext(List.of("doc1", "other"))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.5, within(0.001));
+ assertThat(score.passed()).isFalse();
+ }
+
+ @Test
+ void shouldScoreSemanticMode() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.85, "Good coverage", null));
+
+ var metric = new RetrievalCompletenessMetric(judge, 0.8,
+ RetrievalCompletenessMetric.MatchMode.SEMANTIC);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .context(List.of("expected doc"))
+ .retrievalContext(List.of("retrieved doc"))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.85, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldRejectMissingContext() {
+ var metric = new RetrievalCompletenessMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .retrievalContext(List.of("doc1"))
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("context");
+ }
+
+ @Test
+ void shouldRejectMissingRetrievalContext() {
+ var metric = new RetrievalCompletenessMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .context(List.of("doc1"))
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("retrievalContext");
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new RetrievalCompletenessMetric(judge);
+ assertThat(metric.name()).isEqualTo("RetrievalCompleteness");
+ }
+
+ @Test
+ void shouldDefaultToSemanticMode() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.8, "OK", null));
+
+ var metric = new RetrievalCompletenessMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .context(List.of("doc"))
+ .retrievalContext(List.of("doc"))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.8);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/ToolArgumentCorrectnessMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/ToolArgumentCorrectnessMetricTest.java
new file mode 100644
index 0000000..0875839
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/agent/ToolArgumentCorrectnessMetricTest.java
@@ -0,0 +1,110 @@
+package com.agenteval.metrics.agent;
+
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import com.agenteval.core.model.ToolCall;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+
+class ToolArgumentCorrectnessMetricTest {
+
+ @Test
+ void shouldScorePerfectMatch() {
+ var metric = new ToolArgumentCorrectnessMetric(0.8);
+ var testCase = AgentTestCase.builder()
+ .input("Search for Java tutorials")
+ .toolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "Java tutorials"))))
+ .expectedToolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "Java tutorials"))))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(1.0, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldScorePartialMatch() {
+ var metric = new ToolArgumentCorrectnessMetric(0.8);
+ var testCase = AgentTestCase.builder()
+ .input("Search query")
+ .toolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "wrong", "limit", 10))))
+ .expectedToolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "correct", "limit", 10))))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.5, within(0.001));
+ assertThat(score.passed()).isFalse();
+ }
+
+ @Test
+ void shouldCountExtraArgsInStrictMode() {
+ var metric = new ToolArgumentCorrectnessMetric(0.8, true);
+ var testCase = AgentTestCase.builder()
+ .input("Search query")
+ .toolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "test", "extra", "val"))))
+ .expectedToolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "test"))))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.5, within(0.001));
+ }
+
+ @Test
+ void shouldNotCountExtraArgsInNonStrictMode() {
+ var metric = new ToolArgumentCorrectnessMetric(0.8, false);
+ var testCase = AgentTestCase.builder()
+ .input("Search query")
+ .toolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "test", "extra", "val"))))
+ .expectedToolCalls(List.of(
+ ToolCall.of("search", Map.of("query", "test"))))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(1.0, within(0.001));
+ }
+
+ @Test
+ void shouldHandleBothEmpty() {
+ var metric = new ToolArgumentCorrectnessMetric();
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.value()).isCloseTo(1.0, within(0.001));
+ }
+
+ @Test
+ void shouldHandleMissingActualCalls() {
+ var metric = new ToolArgumentCorrectnessMetric();
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .expectedToolCalls(List.of(ToolCall.of("search", Map.of("q", "test"))))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.value()).isCloseTo(0.0, within(0.001));
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new ToolArgumentCorrectnessMetric();
+ assertThat(metric.name()).isEqualTo("ToolArgumentCorrectness");
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/conversation/ContextRetentionMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/conversation/ContextRetentionMetricTest.java
new file mode 100644
index 0000000..311196b
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/conversation/ContextRetentionMetricTest.java
@@ -0,0 +1,97 @@
+package com.agenteval.metrics.conversation;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.ConversationTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ContextRetentionMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreGoodRetention() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.95, "Excellent context retention", null));
+
+ var metric = new ContextRetentionMetric(judge, 0.7);
+ var testCase = ConversationTestCase.builder()
+ .turns(List.of(
+ AgentTestCase.builder()
+ .input("My name is Alice")
+ .actualOutput("Nice to meet you, Alice!")
+ .build(),
+ AgentTestCase.builder()
+ .input("What is my name?")
+ .actualOutput("Your name is Alice.")
+ .build()))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.95, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldScorePoorRetention() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.2, "Forgot user's name", null));
+
+ var metric = new ContextRetentionMetric(judge, 0.7);
+ var testCase = ConversationTestCase.builder()
+ .turns(List.of(
+ AgentTestCase.builder()
+ .input("My name is Alice")
+ .actualOutput("Nice to meet you!")
+ .build(),
+ AgentTestCase.builder()
+ .input("What is my name?")
+ .actualOutput("I don't know your name.")
+ .build()))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.2, within(0.001));
+ assertThat(score.passed()).isFalse();
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new ContextRetentionMetric(judge);
+ assertThat(metric.name()).isEqualTo("ContextRetention");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new ContextRetentionMetric(judge);
+ var testCase = ConversationTestCase.builder()
+ .turns(List.of(
+ AgentTestCase.builder()
+ .input("q").actualOutput("a").build()))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/conversation/ConversationCoherenceMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/conversation/ConversationCoherenceMetricTest.java
new file mode 100644
index 0000000..77448eb
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/conversation/ConversationCoherenceMetricTest.java
@@ -0,0 +1,117 @@
+package com.agenteval.metrics.conversation;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.ConversationTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.contains;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ConversationCoherenceMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreCoherentConversation() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.9, "Coherent conversation", null));
+
+ var metric = new ConversationCoherenceMetric(judge, 0.7);
+ var testCase = ConversationTestCase.builder()
+ .turns(List.of(
+ AgentTestCase.builder()
+ .input("What is Java?")
+ .actualOutput("Java is a programming language.")
+ .build(),
+ AgentTestCase.builder()
+ .input("What about Python?")
+ .actualOutput("Python is also a programming language.")
+ .build()))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.9, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldIncludeSystemPromptInEvaluation() {
+ when(judge.judge(contains("helpful assistant"))).thenReturn(
+ new JudgeResponse(0.8, "Consistent with system prompt", null));
+
+ var metric = new ConversationCoherenceMetric(judge);
+ var testCase = ConversationTestCase.builder()
+ .systemPrompt("You are a helpful assistant")
+ .turns(List.of(
+ AgentTestCase.builder()
+ .input("Hello")
+ .actualOutput("Hi there!")
+ .build()))
+ .build();
+
+ metric.evaluate(testCase);
+ }
+
+ @Test
+ void shouldHandleMissingSystemPrompt() {
+ when(judge.judge(contains("(none)"))).thenReturn(
+ new JudgeResponse(0.8, "OK", null));
+
+ var metric = new ConversationCoherenceMetric(judge);
+ var testCase = ConversationTestCase.builder()
+ .turns(List.of(
+ AgentTestCase.builder()
+ .input("Hello")
+ .actualOutput("Hi")
+ .build()))
+ .build();
+
+ metric.evaluate(testCase);
+ }
+
+ @Test
+ void shouldRejectNullTestCase() {
+ var metric = new ConversationCoherenceMetric(judge);
+ assertThatThrownBy(() -> metric.evaluate(null))
+ .isInstanceOf(NullPointerException.class);
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new ConversationCoherenceMetric(judge);
+ assertThat(metric.name()).isEqualTo("ConversationCoherence");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new ConversationCoherenceMetric(judge);
+ var testCase = ConversationTestCase.builder()
+ .turns(List.of(
+ AgentTestCase.builder()
+ .input("q").actualOutput("a").build()))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualPrecisionMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualPrecisionMetricTest.java
new file mode 100644
index 0000000..06f565a
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualPrecisionMetricTest.java
@@ -0,0 +1,101 @@
+package com.agenteval.metrics.rag;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ContextualPrecisionMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreHighPrecision() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.9, "All documents relevant", null));
+
+ var metric = new ContextualPrecisionMetric(judge, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("What is Java?")
+ .actualOutput("Java is a programming language.")
+ .expectedOutput("Java is a programming language developed by Sun Microsystems.")
+ .retrievalContext(List.of("Java was developed by Sun.", "Java uses JVM."))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.9, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldRejectMissingRetrievalContext() {
+ var metric = new ContextualPrecisionMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .actualOutput("answer")
+ .expectedOutput("expected")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("retrievalContext");
+ }
+
+ @Test
+ void shouldRejectMissingExpectedOutput() {
+ var metric = new ContextualPrecisionMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .actualOutput("answer")
+ .retrievalContext(List.of("doc1"))
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expectedOutput");
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new ContextualPrecisionMetric(judge);
+ assertThat(metric.name()).isEqualTo("ContextualPrecision");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new ContextualPrecisionMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("q").actualOutput("a").expectedOutput("e")
+ .retrievalContext(List.of("doc")).build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+
+ @Test
+ void shouldFormatContextWithNumberedSeparators() {
+ String formatted = ContextualPrecisionMetric.formatContext(
+ List.of("first doc", "second doc", "third doc"));
+ assertThat(formatted).isEqualTo("[1] first doc\n[2] second doc\n[3] third doc");
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualRecallMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualRecallMetricTest.java
new file mode 100644
index 0000000..a927ac1
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualRecallMetricTest.java
@@ -0,0 +1,91 @@
+package com.agenteval.metrics.rag;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ContextualRecallMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreHighRecall() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.95, "All expected facts found in context", null));
+
+ var metric = new ContextualRecallMetric(judge, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("any input")
+ .expectedOutput("Java was developed by Sun Microsystems and uses JVM.")
+ .retrievalContext(List.of("Java was developed by Sun.", "Java uses JVM."))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.95, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldRejectMissingRetrievalContext() {
+ var metric = new ContextualRecallMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .expectedOutput("expected")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("retrievalContext");
+ }
+
+ @Test
+ void shouldRejectMissingExpectedOutput() {
+ var metric = new ContextualRecallMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .retrievalContext(List.of("doc1"))
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expectedOutput");
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new ContextualRecallMetric(judge);
+ assertThat(metric.name()).isEqualTo("ContextualRecall");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new ContextualRecallMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("q").expectedOutput("e")
+ .retrievalContext(List.of("doc")).build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualRelevancyMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualRelevancyMetricTest.java
new file mode 100644
index 0000000..9012da1
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/rag/ContextualRelevancyMetricTest.java
@@ -0,0 +1,91 @@
+package com.agenteval.metrics.rag;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ContextualRelevancyMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreHighRelevancy() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.9, "All documents relevant to query", null));
+
+ var metric = new ContextualRelevancyMetric(judge, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("What is Java?")
+ .actualOutput("Java is a programming language.")
+ .retrievalContext(List.of("Java is a language.", "Java uses JVM."))
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.9, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldRejectMissingInput() {
+ var metric = new ContextualRelevancyMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("")
+ .retrievalContext(List.of("doc1"))
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("input");
+ }
+
+ @Test
+ void shouldRejectMissingRetrievalContext() {
+ var metric = new ContextualRelevancyMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .actualOutput("answer")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("retrievalContext");
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new ContextualRelevancyMetric(judge);
+ assertThat(metric.name()).isEqualTo("ContextualRelevancy");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new ContextualRelevancyMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("q").actualOutput("a")
+ .retrievalContext(List.of("doc")).build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/response/BiasMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/BiasMetricTest.java
new file mode 100644
index 0000000..83bbe9c
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/BiasMetricTest.java
@@ -0,0 +1,119 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.EnumSet;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.contains;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class BiasMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreUnbiasedOutput() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(1.0, "Completely unbiased", null));
+
+ var metric = new BiasMetric(judge, 0.5);
+ var testCase = AgentTestCase.builder()
+ .input("Tell me about programming languages")
+ .actualOutput("Python, Java, and JavaScript are all popular languages.")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(1.0, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldScoreBiasedOutput() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.2, "Contains gender bias", null));
+
+ var metric = new BiasMetric(judge, 0.5);
+ var testCase = AgentTestCase.builder()
+ .input("Who should be a nurse?")
+ .actualOutput("Biased content here")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.passed()).isFalse();
+ }
+
+ @Test
+ void shouldAcceptCustomDimensions() {
+ when(judge.judge(contains("GENDER"))).thenReturn(
+ new JudgeResponse(0.9, "No gender bias", null));
+
+ var metric = new BiasMetric(judge, 0.5,
+ EnumSet.of(BiasDimension.GENDER, BiasDimension.RACE));
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .actualOutput("answer")
+ .build();
+
+ metric.evaluate(testCase);
+ }
+
+ @Test
+ void shouldNotRequireInput() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(1.0, "Unbiased", null));
+
+ var metric = new BiasMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("any input")
+ .actualOutput("unbiased output")
+ .build();
+
+ metric.evaluate(testCase);
+ }
+
+ @Test
+ void shouldRejectMissingOutput() {
+ var metric = new BiasMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new BiasMetric(judge);
+ assertThat(metric.name()).isEqualTo("Bias");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.5, "OK", null));
+
+ var metric = new BiasMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("q").actualOutput("a").build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.5);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/response/CoherenceMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/CoherenceMetricTest.java
new file mode 100644
index 0000000..ffb1289
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/CoherenceMetricTest.java
@@ -0,0 +1,102 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class CoherenceMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreCoherentOutput() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.95, "Well-structured and logical", null));
+
+ var metric = new CoherenceMetric(judge, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("Explain photosynthesis")
+ .actualOutput("Photosynthesis is the process by which plants convert "
+ + "sunlight into energy. First, chlorophyll absorbs light. "
+ + "Then, this energy drives the conversion of CO2 and water into glucose.")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.95, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldScoreIncoherentOutput() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.1, "Contradictory and disorganized", null));
+
+ var metric = new CoherenceMetric(judge, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("Explain gravity")
+ .actualOutput("Gravity pulls things down. Also gravity pushes things up. "
+ + "Cats are nice. The moon is made of cheese.")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.passed()).isFalse();
+ }
+
+ @Test
+ void shouldRejectMissingInput() {
+ var metric = new CoherenceMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("")
+ .actualOutput("answer")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldRejectMissingOutput() {
+ var metric = new CoherenceMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new CoherenceMetric(judge);
+ assertThat(metric.name()).isEqualTo("Coherence");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.7, "OK", null));
+
+ var metric = new CoherenceMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("q").actualOutput("a").build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.7);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/response/ConcisenessMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/ConcisenessMetricTest.java
new file mode 100644
index 0000000..68ed860
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/ConcisenessMetricTest.java
@@ -0,0 +1,100 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.judge.JudgeModel;
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ConcisenessMetricTest {
+
+ private JudgeModel judge;
+
+ @BeforeEach
+ void setUp() {
+ judge = mock(JudgeModel.class);
+ }
+
+ @Test
+ void shouldScoreConciseOutput() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.95, "Very concise response", null));
+
+ var metric = new ConcisenessMetric(judge, 0.5);
+ var testCase = AgentTestCase.builder()
+ .input("What is 2+2?")
+ .actualOutput("4")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.95, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldScoreVerboseOutput() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.2, "Extremely verbose", null));
+
+ var metric = new ConcisenessMetric(judge, 0.5);
+ var testCase = AgentTestCase.builder()
+ .input("What is 2+2?")
+ .actualOutput("Well, let me think about that. The answer is 4. "
+ + "As you may know, addition is a mathematical operation...")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.passed()).isFalse();
+ }
+
+ @Test
+ void shouldRejectMissingInput() {
+ var metric = new ConcisenessMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("")
+ .actualOutput("answer")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldRejectMissingOutput() {
+ var metric = new ConcisenessMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("question")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new ConcisenessMetric(judge);
+ assertThat(metric.name()).isEqualTo("Conciseness");
+ }
+
+ @Test
+ void shouldUseDefaultThreshold() {
+ when(judge.judge(anyString())).thenReturn(
+ new JudgeResponse(0.5, "OK", null));
+
+ var metric = new ConcisenessMetric(judge);
+ var testCase = AgentTestCase.builder()
+ .input("q").actualOutput("a").build();
+
+ EvalScore score = metric.evaluate(testCase);
+ assertThat(score.threshold()).isEqualTo(0.5);
+ }
+}
diff --git a/agenteval-metrics/src/test/java/com/agenteval/metrics/response/SemanticSimilarityMetricTest.java b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/SemanticSimilarityMetricTest.java
new file mode 100644
index 0000000..21909d0
--- /dev/null
+++ b/agenteval-metrics/src/test/java/com/agenteval/metrics/response/SemanticSimilarityMetricTest.java
@@ -0,0 +1,108 @@
+package com.agenteval.metrics.response;
+
+import com.agenteval.core.embedding.EmbeddingModel;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class SemanticSimilarityMetricTest {
+
+ private EmbeddingModel embeddingModel;
+
+ @BeforeEach
+ void setUp() {
+ embeddingModel = mock(EmbeddingModel.class);
+ when(embeddingModel.modelId()).thenReturn("test-model");
+ }
+
+ @Test
+ void shouldScoreIdenticalEmbeddings() {
+ when(embeddingModel.embed(anyString()))
+ .thenReturn(List.of(1.0, 0.0, 0.0));
+
+ var metric = new SemanticSimilarityMetric(embeddingModel, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .actualOutput("same text")
+ .expectedOutput("same text")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(1.0, within(0.001));
+ assertThat(score.passed()).isTrue();
+ }
+
+ @Test
+ void shouldScoreOrthogonalEmbeddings() {
+ when(embeddingModel.embed("actual")).thenReturn(List.of(1.0, 0.0, 0.0));
+ when(embeddingModel.embed("expected")).thenReturn(List.of(0.0, 1.0, 0.0));
+
+ var metric = new SemanticSimilarityMetric(embeddingModel, 0.7);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .actualOutput("actual")
+ .expectedOutput("expected")
+ .build();
+
+ EvalScore score = metric.evaluate(testCase);
+
+ assertThat(score.value()).isCloseTo(0.0, within(0.001));
+ assertThat(score.passed()).isFalse();
+ }
+
+ @Test
+ void shouldRejectMissingActualOutput() {
+ var metric = new SemanticSimilarityMetric(embeddingModel);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .expectedOutput("expected")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldRejectMissingExpectedOutput() {
+ var metric = new SemanticSimilarityMetric(embeddingModel);
+ var testCase = AgentTestCase.builder()
+ .input("query")
+ .actualOutput("actual")
+ .build();
+
+ assertThatThrownBy(() -> metric.evaluate(testCase))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void shouldReturnCorrectName() {
+ var metric = new SemanticSimilarityMetric(embeddingModel);
+ assertThat(metric.name()).isEqualTo("SemanticSimilarity");
+ }
+
+ @Test
+ void shouldCalculateCosineSimilarity() {
+ double sim = SemanticSimilarityMetric.cosineSimilarity(
+ List.of(1.0, 0.0), List.of(1.0, 0.0));
+ assertThat(sim).isCloseTo(1.0, within(0.001));
+
+ sim = SemanticSimilarityMetric.cosineSimilarity(
+ List.of(1.0, 0.0), List.of(0.0, 1.0));
+ assertThat(sim).isCloseTo(0.0, within(0.001));
+
+ sim = SemanticSimilarityMetric.cosineSimilarity(
+ List.of(1.0, 1.0), List.of(1.0, 1.0));
+ assertThat(sim).isCloseTo(1.0, within(0.001));
+ }
+}
diff --git a/agenteval-reporting/pom.xml b/agenteval-reporting/pom.xml
index 964dd6c..afab760 100644
--- a/agenteval-reporting/pom.xml
+++ b/agenteval-reporting/pom.xml
@@ -23,5 +23,14 @@
org.slf4j
slf4j-api
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ org.mockito
+ mockito-core
+ test
+
diff --git a/agenteval-reporting/src/main/java/com/agenteval/reporting/JsonReporter.java b/agenteval-reporting/src/main/java/com/agenteval/reporting/JsonReporter.java
new file mode 100644
index 0000000..2d16b58
--- /dev/null
+++ b/agenteval-reporting/src/main/java/com/agenteval/reporting/JsonReporter.java
@@ -0,0 +1,75 @@
+package com.agenteval.reporting;
+
+import com.agenteval.core.eval.CaseResult;
+import com.agenteval.core.eval.EvalResult;
+import com.agenteval.core.model.EvalScore;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Reports evaluation results as a JSON file.
+ *
+ * Serializes the full {@link EvalResult} structure including per-case scores,
+ * metric summaries, and aggregate statistics.
+ */
+public final class JsonReporter implements EvalReporter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JsonReporter.class);
+ private static final ObjectMapper MAPPER = new ObjectMapper()
+ .enable(SerializationFeature.INDENT_OUTPUT);
+
+ private final Path outputPath;
+
+ public JsonReporter(Path outputPath) {
+ this.outputPath = Objects.requireNonNull(outputPath, "outputPath must not be null");
+ }
+
+ @Override
+ public void report(EvalResult result) {
+ LOG.debug("Writing JSON report to {}", outputPath);
+
+ ObjectNode root = MAPPER.createObjectNode();
+ root.put("averageScore", result.averageScore());
+ root.put("passRate", result.passRate());
+ root.put("totalCases", result.caseResults().size());
+ root.put("failedCases", result.failedCases().size());
+ root.put("durationMs", result.durationMs());
+
+ ObjectNode metricAverages = root.putObject("metricAverages");
+ result.averageScoresByMetric().forEach(metricAverages::put);
+
+ ArrayNode cases = root.putArray("caseResults");
+ for (CaseResult cr : result.caseResults()) {
+ ObjectNode caseNode = cases.addObject();
+ caseNode.put("input", cr.testCase().getInput());
+ caseNode.put("passed", cr.passed());
+ caseNode.put("averageScore", cr.averageScore());
+
+ ObjectNode scores = caseNode.putObject("scores");
+ for (Map.Entry entry : cr.scores().entrySet()) {
+ ObjectNode scoreNode = scores.putObject(entry.getKey());
+ scoreNode.put("value", entry.getValue().value());
+ scoreNode.put("threshold", entry.getValue().threshold());
+ scoreNode.put("passed", entry.getValue().passed());
+ scoreNode.put("reason", entry.getValue().reason());
+ }
+ }
+
+ try (OutputStream out = Files.newOutputStream(outputPath)) {
+ MAPPER.writeValue(out, root);
+ } catch (IOException e) {
+ throw new ReportException("Failed to write JSON report to " + outputPath, e);
+ }
+ }
+}
diff --git a/agenteval-reporting/src/test/java/com/agenteval/reporting/JsonReporterTest.java b/agenteval-reporting/src/test/java/com/agenteval/reporting/JsonReporterTest.java
new file mode 100644
index 0000000..8cf066d
--- /dev/null
+++ b/agenteval-reporting/src/test/java/com/agenteval/reporting/JsonReporterTest.java
@@ -0,0 +1,76 @@
+package com.agenteval.reporting;
+
+import com.agenteval.core.eval.CaseResult;
+import com.agenteval.core.eval.EvalResult;
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.EvalScore;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+
+class JsonReporterTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Test
+ void shouldWriteJsonReport(@TempDir Path tmpDir) throws Exception {
+ Path outputFile = tmpDir.resolve("report.json");
+
+ var testCase = AgentTestCase.builder()
+ .input("What is Java?")
+ .actualOutput("A programming language")
+ .build();
+ var score = EvalScore.of(0.9, 0.7, "Good answer");
+ var caseResult = new CaseResult(testCase, Map.of("Relevancy", score), true);
+ var result = EvalResult.of(List.of(caseResult), 150);
+
+ new JsonReporter(outputFile).report(result);
+
+ assertThat(outputFile).exists();
+ JsonNode root = MAPPER.readTree(Files.readString(outputFile));
+
+ assertThat(root.path("totalCases").asInt()).isEqualTo(1);
+ assertThat(root.path("failedCases").asInt()).isEqualTo(0);
+ assertThat(root.path("durationMs").asLong()).isEqualTo(150);
+ assertThat(root.path("passRate").asDouble()).isCloseTo(1.0, within(0.001));
+ assertThat(root.path("averageScore").asDouble()).isCloseTo(0.9, within(0.001));
+
+ JsonNode metricAvg = root.path("metricAverages");
+ assertThat(metricAvg.path("Relevancy").asDouble()).isCloseTo(0.9, within(0.001));
+
+ JsonNode cases = root.path("caseResults");
+ assertThat(cases).hasSize(1);
+ assertThat(cases.get(0).path("input").asText()).isEqualTo("What is Java?");
+ assertThat(cases.get(0).path("passed").asBoolean()).isTrue();
+
+ JsonNode scores = cases.get(0).path("scores").path("Relevancy");
+ assertThat(scores.path("value").asDouble()).isCloseTo(0.9, within(0.001));
+ assertThat(scores.path("reason").asText()).isEqualTo("Good answer");
+ }
+
+ @Test
+ void shouldReportFailedCases(@TempDir Path tmpDir) throws Exception {
+ Path outputFile = tmpDir.resolve("report.json");
+
+ var testCase = AgentTestCase.builder()
+ .input("Question").actualOutput("Bad answer").build();
+ var score = EvalScore.of(0.3, 0.7, "Irrelevant");
+ var caseResult = new CaseResult(testCase, Map.of("Relevancy", score), false);
+ var result = EvalResult.of(List.of(caseResult), 100);
+
+ new JsonReporter(outputFile).report(result);
+
+ JsonNode root = MAPPER.readTree(Files.readString(outputFile));
+ assertThat(root.path("failedCases").asInt()).isEqualTo(1);
+ assertThat(root.path("passRate").asDouble()).isCloseTo(0.0, within(0.001));
+ }
+}
diff --git a/agenteval-spring-ai/pom.xml b/agenteval-spring-ai/pom.xml
new file mode 100644
index 0000000..ab16767
--- /dev/null
+++ b/agenteval-spring-ai/pom.xml
@@ -0,0 +1,56 @@
+
+
+ 4.0.0
+
+
+ com.agenteval
+ agenteval-parent
+ 0.1.0-SNAPSHOT
+
+
+ agenteval-spring-ai
+ AgentEval Spring AI
+ Spring AI auto-capture integration for AgentEval
+
+
+ 1.0.0
+ 3.4.2
+
+
+
+
+ com.agenteval
+ agenteval-core
+
+
+ org.springframework.ai
+ spring-ai-model
+ ${spring-ai.version}
+ provided
+
+
+ org.springframework.ai
+ spring-ai-client-chat
+ ${spring-ai.version}
+ provided
+
+
+ org.springframework.ai
+ spring-ai-commons
+ ${spring-ai.version}
+ provided
+
+
+ org.springframework.boot
+ spring-boot-autoconfigure
+ ${spring-boot.version}
+ true
+
+
+ org.slf4j
+ slf4j-api
+
+
+
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
new file mode 100644
index 0000000..cd98561
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
@@ -0,0 +1,67 @@
+package com.agenteval.spring.ai;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.chat.client.ChatClientRequest;
+import org.springframework.ai.chat.client.ChatClientResponse;
+import org.springframework.ai.chat.client.advisor.api.CallAdvisor;
+import org.springframework.ai.chat.client.advisor.api.CallAdvisorChain;
+import org.springframework.ai.document.Document;
+import org.springframework.lang.NonNull;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Spring AI advisor interceptor that captures RAG retrieval context.
+ *
+ * Extracts document content from advised requests for use as
+ * retrieval context in AgentEval test cases.
+ */
+public final class SpringAiAdvisorInterceptor implements CallAdvisor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SpringAiAdvisorInterceptor.class);
+
+ private final List capturedContext = new ArrayList<>();
+
+ @NonNull
+ @Override
+ public ChatClientResponse adviseCall(@NonNull ChatClientRequest request,
+ @NonNull CallAdvisorChain chain) {
+ if (request.context() != null) {
+ var context = request.context();
+ Object docs = context.get("qa_advisor_retrieved_documents");
+ if (docs instanceof List docList) {
+ for (Object doc : docList) {
+ if (doc instanceof Document document) {
+ capturedContext.add(document.getText());
+ }
+ }
+ LOG.debug("Captured {} retrieval context documents",
+ capturedContext.size());
+ }
+ }
+
+ return chain.nextCall(request);
+ }
+
+ @NonNull
+ @Override
+ public String getName() {
+ return "AgentEvalAdvisorInterceptor";
+ }
+
+ @Override
+ public int getOrder() {
+ return 0;
+ }
+
+ /**
+ * Returns the captured retrieval context and clears the buffer.
+ */
+ public List consumeCapturedContext() {
+ var result = List.copyOf(capturedContext);
+ capturedContext.clear();
+ return result;
+ }
+}
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiCapture.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiCapture.java
new file mode 100644
index 0000000..0aa7abc
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiCapture.java
@@ -0,0 +1,48 @@
+package com.agenteval.spring.ai;
+
+import com.agenteval.core.model.AgentTestCase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.prompt.Prompt;
+
+import java.util.Objects;
+
+/**
+ * Captures Spring AI {@link ChatModel} interactions as {@link AgentTestCase} instances.
+ *
+ * {@code
+ * var capture = new SpringAiCapture(chatModel);
+ * AgentTestCase testCase = capture.call("What is Java?");
+ * }
+ */
+public final class SpringAiCapture {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SpringAiCapture.class);
+
+ private final ChatModel chatModel;
+
+ public SpringAiCapture(ChatModel chatModel) {
+ this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
+ }
+
+ /**
+ * Calls the chat model and captures the interaction as an AgentTestCase.
+ *
+ * @param input the user prompt
+ * @return the captured test case
+ */
+ public AgentTestCase call(String input) {
+ LOG.debug("Capturing Spring AI call for input: {}",
+ input.length() > 100 ? input.substring(0, 100) + "..." : input);
+
+ long start = System.currentTimeMillis();
+ ChatResponse response = chatModel.call(new Prompt(input));
+ long latency = System.currentTimeMillis() - start;
+
+ AgentTestCase testCase = SpringAiTestCaseBuilder.fromChatResponse(input, response);
+ testCase.setLatencyMs(latency);
+ return testCase;
+ }
+}
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiTestCaseBuilder.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiTestCaseBuilder.java
new file mode 100644
index 0000000..fff97dc
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiTestCaseBuilder.java
@@ -0,0 +1,47 @@
+package com.agenteval.spring.ai;
+
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.TokenUsage;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+
+import java.util.Objects;
+
+/**
+ * Converts Spring AI types to AgentEval {@link AgentTestCase}.
+ */
+public final class SpringAiTestCaseBuilder {
+
+ private SpringAiTestCaseBuilder() {}
+
+ /**
+ * Creates an AgentTestCase from a Spring AI ChatResponse.
+ *
+ * @param input the user's input prompt
+ * @param response the Spring AI chat response
+ * @return a populated AgentTestCase
+ */
+ public static AgentTestCase fromChatResponse(String input, ChatResponse response) {
+ Objects.requireNonNull(input, "input must not be null");
+ Objects.requireNonNull(response, "response must not be null");
+
+ var builder = AgentTestCase.builder().input(input);
+
+ if (response.getResult() != null) {
+ Generation result = response.getResult();
+ if (result.getOutput() != null) {
+ builder.actualOutput(result.getOutput().getText());
+ }
+ }
+
+ if (response.getMetadata() != null && response.getMetadata().getUsage() != null) {
+ var usage = response.getMetadata().getUsage();
+ builder.tokenUsage(new TokenUsage(
+ (int) usage.getPromptTokens(),
+ (int) usage.getCompletionTokens(),
+ (int) usage.getTotalTokens()));
+ }
+
+ return builder.build();
+ }
+}
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/autoconfigure/AgentEvalAutoConfiguration.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/autoconfigure/AgentEvalAutoConfiguration.java
new file mode 100644
index 0000000..e09c194
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/autoconfigure/AgentEvalAutoConfiguration.java
@@ -0,0 +1,21 @@
+package com.agenteval.spring.ai.autoconfigure;
+
+import com.agenteval.spring.ai.SpringAiAdvisorInterceptor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Spring Boot auto-configuration for AgentEval Spring AI integration.
+ *
+ * Registers the advisor interceptor for automatic RAG context capture.
+ */
+@Configuration
+@ConditionalOnClass(name = "org.springframework.ai.chat.model.ChatModel")
+public class AgentEvalAutoConfiguration {
+
+ @Bean
+ public SpringAiAdvisorInterceptor agentEvalAdvisorInterceptor() {
+ return new SpringAiAdvisorInterceptor();
+ }
+}
diff --git a/pom.xml b/pom.xml
index 9b545af..cd4a8bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,10 +23,13 @@
agenteval-core
agenteval-judge
+ agenteval-embeddings
agenteval-metrics
agenteval-datasets
agenteval-reporting
agenteval-junit5
+ agenteval-spring-ai
+ agenteval-langchain4j
@@ -110,6 +113,11 @@
agenteval-reporting
${project.version}
+
+ com.agenteval
+ agenteval-embeddings
+ ${project.version}
+
diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml
index 9ad3b3f..ecd1554 100644
--- a/spotbugs-exclude.xml
+++ b/spotbugs-exclude.xml
@@ -20,6 +20,10 @@
+
@@ -32,6 +36,10 @@
+
+
@@ -44,4 +52,16 @@
+
+
+
+
+
+
+
+