Higher score = less biased (1.0 = completely unbiased).
+ */ +public final class BiasMetric extends LLMJudgeMetric { + + private static final String NAME = "Bias"; + private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/bias.txt"; + private static final double DEFAULT_THRESHOLD = 0.5; + + private final SetHigher score = more coherent (1.0 = perfectly coherent).
+ */ +public final class CoherenceMetric extends LLMJudgeMetric { + + private static final String NAME = "Coherence"; + private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/coherence.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public CoherenceMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public CoherenceMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected MapHigher score = more concise (1.0 = optimally concise).
+ */ +public final class ConcisenessMetric extends LLMJudgeMetric { + + private static final String NAME = "Conciseness"; + private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/conciseness.txt"; + private static final double DEFAULT_THRESHOLD = 0.5; + + public ConcisenessMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public ConcisenessMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected MapEvaluates if the retrieval context contains information that is + * useful for producing the expected output, penalizing irrelevant retrievals.
+ */ +public final class ContextualPrecisionMetric extends LLMJudgeMetric { + + private static final String NAME = "ContextualPrecision"; + private static final String PROMPT_PATH = + "com/agenteval/metrics/prompts/contextual-precision.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public ContextualPrecisionMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public ContextualPrecisionMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected void validate(AgentTestCase testCase) { + super.validate(testCase); + if (testCase.getRetrievalContext().isEmpty()) { + throw new IllegalArgumentException( + NAME + " requires non-empty retrievalContext"); + } + if (testCase.getExpectedOutput() == null || testCase.getExpectedOutput().isBlank()) { + throw new IllegalArgumentException( + NAME + " requires non-empty expectedOutput"); + } + } + + @Override + protected MapEvaluates recall: are all facts in the expected output supported + * by at least one retrieved document?
+ */ +public final class ContextualRecallMetric extends LLMJudgeMetric { + + private static final String NAME = "ContextualRecall"; + private static final String PROMPT_PATH = + "com/agenteval/metrics/prompts/contextual-recall.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public ContextualRecallMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public ContextualRecallMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected void validate(AgentTestCase testCase) { + if (testCase.getRetrievalContext().isEmpty()) { + throw new IllegalArgumentException( + NAME + " requires non-empty retrievalContext"); + } + if (testCase.getExpectedOutput() == null || testCase.getExpectedOutput().isBlank()) { + throw new IllegalArgumentException( + NAME + " requires non-empty expectedOutput"); + } + } + + @Override + protected MapEvaluates if the retrieved documents are topically related to + * the user's question, regardless of the expected output.
+ */ +public final class ContextualRelevancyMetric extends LLMJudgeMetric { + + private static final String NAME = "ContextualRelevancy"; + private static final String PROMPT_PATH = + "com/agenteval/metrics/prompts/contextual-relevancy.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public ContextualRelevancyMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public ContextualRelevancyMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected void validate(AgentTestCase testCase) { + if (testCase.getInput() == null || testCase.getInput().isBlank()) { + throw new IllegalArgumentException( + NAME + " requires non-empty input"); + } + if (testCase.getRetrievalContext().isEmpty()) { + throw new IllegalArgumentException( + NAME + " requires non-empty retrievalContext"); + } + } + + @Override + protected MapCompares PLAN-type reasoning steps against ACTION/OBSERVATION steps + * to determine if the agent followed its own plan.
+ */ +public final class PlanAdherenceMetric extends LLMJudgeMetric { + + private static final String NAME = "PlanAdherence"; + private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/plan-adherence.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public PlanAdherenceMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public PlanAdherenceMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected void validate(AgentTestCase testCase) { + super.validate(testCase); + if (testCase.getReasoningTrace().isEmpty()) { + throw new IllegalArgumentException( + NAME + " requires non-empty reasoningTrace"); + } + } + + @Override + protected MapExamines the PLAN-type reasoning steps for clarity, feasibility, + * completeness, and logical ordering.
+ */ +public final class PlanQualityMetric extends LLMJudgeMetric { + + private static final String NAME = "PlanQuality"; + private static final String PROMPT_PATH = "com/agenteval/metrics/prompts/plan-quality.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public PlanQualityMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public PlanQualityMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected void validate(AgentTestCase testCase) { + if (testCase.getInput() == null || testCase.getInput().isBlank()) { + throw new IllegalArgumentException(NAME + " requires non-empty input"); + } + if (testCase.getReasoningTrace().isEmpty()) { + throw new IllegalArgumentException( + NAME + " requires non-empty reasoningTrace"); + } + } + + @Override + protected MapSupports two match modes:
+ *Matches actual tool calls to expected tool calls by name, then deep-compares + * argument maps. In strict mode, extra arguments count as failures.
+ */ +public final class ToolArgumentCorrectnessMetric implements EvalMetric { + + private static final String NAME = "ToolArgumentCorrectness"; + private static final double DEFAULT_THRESHOLD = 0.8; + + private final double threshold; + private final boolean strictMode; + + public ToolArgumentCorrectnessMetric() { + this(DEFAULT_THRESHOLD, false); + } + + public ToolArgumentCorrectnessMetric(double threshold) { + this(threshold, false); + } + + public ToolArgumentCorrectnessMetric(double threshold, boolean strictMode) { + if (threshold < 0.0 || threshold > 1.0) { + throw new IllegalArgumentException( + "threshold must be between 0.0 and 1.0, got: " + threshold); + } + this.threshold = threshold; + this.strictMode = strictMode; + } + + @Override + public EvalScore evaluate(AgentTestCase testCase) { + Objects.requireNonNull(testCase, "testCase must not be null"); + + ListUnlike {@link EvalMetric} which evaluates single-turn {@code AgentTestCase}, + * this interface evaluates multi-turn {@link ConversationTestCase} instances. + * Implementations must be thread-safe.
+ */ +public interface ConversationMetric { + + /** + * Evaluates the given conversation test case and returns a score. + * + * @param testCase the multi-turn conversation to evaluate + * @return the evaluation score (0.0-1.0) + */ + EvalScore evaluate(ConversationTestCase testCase); + + /** + * Returns the name of this metric. + */ + String name(); +} diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ContextRetentionMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ContextRetentionMetric.java new file mode 100644 index 0000000..9a5d88f --- /dev/null +++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/conversation/ContextRetentionMetric.java @@ -0,0 +1,43 @@ +package com.agenteval.metrics.conversation; + +import com.agenteval.core.judge.JudgeModel; +import com.agenteval.core.model.ConversationTestCase; + +import java.util.HashMap; +import java.util.Map; + +/** + * Evaluates whether the agent retains and correctly uses context + * from earlier turns in a multi-turn conversation. + * + *Higher score = better context retention (1.0 = perfect retention).
+ */ +public final class ContextRetentionMetric extends LLMConversationMetric { + + private static final String NAME = "ContextRetention"; + private static final String PROMPT_PATH = + "com/agenteval/metrics/prompts/context-retention.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public ContextRetentionMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public ContextRetentionMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected MapHigher score = more coherent conversation (1.0 = perfectly coherent).
+ */ +public final class ConversationCoherenceMetric extends LLMConversationMetric { + + private static final String NAME = "ConversationCoherence"; + private static final String PROMPT_PATH = + "com/agenteval/metrics/prompts/conversation-coherence.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + + public ConversationCoherenceMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD); + } + + public ConversationCoherenceMetric(JudgeModel judge, double threshold) { + super(judge, threshold, PROMPT_PATH); + } + + @Override + public String name() { + return NAME; + } + + @Override + protected MapFollows the same template method pattern as {@code LLMJudgeMetric}, + * but operates on {@link ConversationTestCase} instead of {@code AgentTestCase}.
+ */ +public abstract class LLMConversationMetric implements ConversationMetric { + + private static final Logger LOG = LoggerFactory.getLogger(LLMConversationMetric.class); + + protected final JudgeModel judge; + protected final double threshold; + private final String promptResourcePath; + + protected LLMConversationMetric(JudgeModel judge, double threshold, + String promptResourcePath) { + this.judge = Objects.requireNonNull(judge, "judge must not be null"); + this.promptResourcePath = Objects.requireNonNull(promptResourcePath, + "promptResourcePath must not be null"); + if (threshold < 0.0 || threshold > 1.0) { + throw new IllegalArgumentException( + "threshold must be between 0.0 and 1.0, got: " + threshold); + } + this.threshold = threshold; + } + + @Override + public final EvalScore evaluate(ConversationTestCase testCase) { + Objects.requireNonNull(testCase, "testCase must not be null"); + validate(testCase); + + Map{@code
+ * var model = EmbeddingModels.openai("text-embedding-3-small");
+ * var model = EmbeddingModels.ollama("nomic-embed-text");
+ * }
+ */
+public final class EmbeddingModels {
+
+ private static final String OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
+ private static final String OPENAI_BASE_URL = "https://api.openai.com";
+ private static final String OLLAMA_BASE_URL = "http://localhost:11434";
+
+ private EmbeddingModels() {}
+
+ /**
+ * Creates an OpenAI embedding model using the given model ID.
+ * API key is resolved from the {@code OPENAI_API_KEY} environment variable.
+ */
+ public static EmbeddingModel openai(String model) {
+ String apiKey = System.getenv(OPENAI_API_KEY_ENV);
+ if (apiKey == null || apiKey.isBlank()) {
+ throw new EmbeddingException(
+ "OpenAI API key not found. Set the " + OPENAI_API_KEY_ENV
+ + " environment variable or use EmbeddingModels.openai(EmbeddingConfig)");
+ }
+ return openai(EmbeddingConfig.builder()
+ .apiKey(apiKey)
+ .model(model)
+ .baseUrl(OPENAI_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates an OpenAI embedding model with full configuration.
+ */
+ public static EmbeddingModel openai(EmbeddingConfig config) {
+ return new OpenAiEmbeddingModel(config);
+ }
+
+ /**
+ * Creates an Ollama embedding model using the given model ID.
+ * Defaults to {@code localhost:11434}.
+ */
+ public static EmbeddingModel ollama(String model) {
+ return ollama(EmbeddingConfig.builder()
+ .model(model)
+ .baseUrl(OLLAMA_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates an Ollama embedding model with full configuration.
+ */
+ public static EmbeddingModel ollama(EmbeddingConfig config) {
+ return new OllamaEmbeddingModel(config);
+ }
+}
diff --git a/agenteval-embeddings/src/main/java/com/agenteval/embeddings/config/EmbeddingConfig.java b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/config/EmbeddingConfig.java
new file mode 100644
index 0000000..9998196
--- /dev/null
+++ b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/config/EmbeddingConfig.java
@@ -0,0 +1,49 @@
+package com.agenteval.embeddings.config;
+
+import java.time.Duration;
+import java.util.Objects;
+
+/**
+ * Configuration for an embedding model provider.
+ */
+public final class EmbeddingConfig {
+
+ private final String apiKey;
+ private final String model;
+ private final String baseUrl;
+ private final Duration timeout;
+
+ private EmbeddingConfig(Builder builder) {
+ this.apiKey = builder.apiKey;
+ this.model = Objects.requireNonNull(builder.model, "model must not be null");
+ this.baseUrl = Objects.requireNonNull(builder.baseUrl, "baseUrl must not be null");
+ this.timeout = builder.timeout;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getApiKey() { return apiKey; }
+ public String getModel() { return model; }
+ public String getBaseUrl() { return baseUrl; }
+ public Duration getTimeout() { return timeout; }
+
+ public static final class Builder {
+ private String apiKey;
+ private String model;
+ private String baseUrl;
+ private Duration timeout = Duration.ofSeconds(30);
+
+ private Builder() {}
+
+ public Builder apiKey(String apiKey) { this.apiKey = apiKey; return this; }
+ public Builder model(String model) { this.model = model; return this; }
+ public Builder baseUrl(String baseUrl) { this.baseUrl = baseUrl; return this; }
+ public Builder timeout(Duration timeout) { this.timeout = timeout; return this; }
+
+ public EmbeddingConfig build() {
+ return new EmbeddingConfig(this);
+ }
+ }
+}
diff --git a/agenteval-embeddings/src/main/java/com/agenteval/embeddings/http/HttpEmbeddingClient.java b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/http/HttpEmbeddingClient.java
new file mode 100644
index 0000000..7cdca1b
--- /dev/null
+++ b/agenteval-embeddings/src/main/java/com/agenteval/embeddings/http/HttpEmbeddingClient.java
@@ -0,0 +1,59 @@
+package com.agenteval.embeddings.http;
+
+import com.agenteval.embeddings.EmbeddingException;
+import com.agenteval.embeddings.config.EmbeddingConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+
+/**
+ * HTTP client for embedding model requests.
+ */
+public class HttpEmbeddingClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HttpEmbeddingClient.class);
+
+ private final HttpClient httpClient;
+ private final EmbeddingConfig config;
+
+ public HttpEmbeddingClient(EmbeddingConfig config) {
+ this(config, HttpClient.newBuilder()
+ .connectTimeout(config.getTimeout())
+ .build());
+ }
+
+ HttpEmbeddingClient(EmbeddingConfig config, HttpClient httpClient) {
+ this.config = config;
+ this.httpClient = httpClient;
+ }
+
+ /**
+ * Sends an embedding request and returns the response.
+ */
+ public HttpEmbeddingResponse send(HttpEmbeddingRequest request) {
+ LOG.debug("Sending embedding request to {}", request.url());
+ try {
+ var builder = HttpRequest.newBuilder()
+ .uri(URI.create(request.url()))
+ .timeout(config.getTimeout())
+ .POST(HttpRequest.BodyPublishers.ofString(request.body()));
+ request.headers().forEach(builder::header);
+ builder.header("Content-Type", "application/json");
+
+ HttpResponseSends requests to {@code POST /api/embeddings}. No API key required.
+ */ +public final class OllamaEmbeddingModel implements EmbeddingModel { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String EMBEDDINGS_PATH = "/api/embeddings"; + + private final EmbeddingConfig config; + private final HttpEmbeddingClient client; + + public OllamaEmbeddingModel(EmbeddingConfig config) { + this(config, new HttpEmbeddingClient(config)); + } + + OllamaEmbeddingModel(EmbeddingConfig config, HttpEmbeddingClient client) { + this.config = Objects.requireNonNull(config, "config must not be null"); + this.client = Objects.requireNonNull(client, "client must not be null"); + } + + @Override + public ListSends requests to {@code POST /v1/embeddings}.
+ */ +public final class OpenAiEmbeddingModel implements EmbeddingModel { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String EMBEDDINGS_PATH = "/v1/embeddings"; + + private final EmbeddingConfig config; + private final HttpEmbeddingClient client; + + public OpenAiEmbeddingModel(EmbeddingConfig config) { + this(config, new HttpEmbeddingClient(config)); + } + + OpenAiEmbeddingModel(EmbeddingConfig config, HttpEmbeddingClient client) { + this.config = Objects.requireNonNull(config, "config must not be null"); + this.client = Objects.requireNonNull(client, "client must not be null"); + } + + @Override + public ListRequires an {@link EmbeddingModel} to generate vector representations. + * No LLM judge needed.
+ */ +public final class SemanticSimilarityMetric implements EvalMetric { + + private static final String NAME = "SemanticSimilarity"; + private static final double DEFAULT_THRESHOLD = 0.7; + + private final EmbeddingModel embeddingModel; + private final double threshold; + + public SemanticSimilarityMetric(EmbeddingModel embeddingModel) { + this(embeddingModel, DEFAULT_THRESHOLD); + } + + public SemanticSimilarityMetric(EmbeddingModel embeddingModel, double threshold) { + this.embeddingModel = Objects.requireNonNull(embeddingModel, + "embeddingModel must not be null"); + if (threshold < 0.0 || threshold > 1.0) { + throw new IllegalArgumentException( + "threshold must be between 0.0 and 1.0, got: " + threshold); + } + this.threshold = threshold; + } + + @Override + public EvalScore evaluate(AgentTestCase testCase) { + Objects.requireNonNull(testCase, "testCase must not be null"); + + if (testCase.getActualOutput() == null || testCase.getActualOutput().isBlank()) { + throw new IllegalArgumentException(NAME + " requires non-empty actualOutput"); + } + if (testCase.getExpectedOutput() == null || testCase.getExpectedOutput().isBlank()) { + throw new IllegalArgumentException(NAME + " requires non-empty expectedOutput"); + } + + List{@code
+ * EvalDataset dataset = DatasetLoaders.forPath(Path.of("data.csv"));
+ * }
+ */
+public final class DatasetLoaders {
+
+ private DatasetLoaders() {}
+
+ /**
+ * Auto-detects the format from the file extension and loads the dataset.
+ *
+ * @param path the dataset file path
+ * @return the loaded dataset
+ * @throws DatasetException if loading or format detection fails
+ */
+ public static EvalDataset forPath(Path path) {
+ DatasetFormat format = DatasetFormat.detect(path);
+ return switch (format) {
+ case JSON -> new JsonDatasetLoader().load(path);
+ case JSONL -> new JsonlDatasetLoader().load(path);
+ case CSV -> new CsvDatasetLoader().load(path);
+ };
+ }
+}
diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java
index 0f4cbbb..8562722 100644
--- a/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java
+++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java
@@ -1,7 +1,9 @@
package com.agenteval.datasets;
import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.datasets.csv.CsvDatasetWriter;
import com.agenteval.datasets.json.JsonDatasetWriter;
+import com.agenteval.datasets.jsonl.JsonlDatasetWriter;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
@@ -54,6 +56,22 @@ public void save(Path path) {
new JsonDatasetWriter().write(this, path);
}
+ /**
+ * Saves this dataset to a file in the specified format.
+ *
+ * @param path the target file path
+ * @param format the output format
+ * @throws DatasetException if writing fails
+ */
+ public void save(Path path, DatasetFormat format) {
+ switch (format) {
+ case JSON -> new JsonDatasetWriter().write(this, path);
+ case JSONL -> new JsonlDatasetWriter().write(this, path);
+ case CSV -> new CsvDatasetWriter().write(this, path);
+ default -> throw new DatasetException("Unsupported format: " + format);
+ }
+ }
+
@JsonPOJOBuilder(withPrefix = "")
public static final class Builder {
private String name;
diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java
new file mode 100644
index 0000000..ae0b224
--- /dev/null
+++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java
@@ -0,0 +1,161 @@
+package com.agenteval.datasets.csv;
+
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.datasets.DatasetException;
+import com.agenteval.datasets.DatasetLoader;
+import com.agenteval.datasets.EvalDataset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Loads evaluation datasets from RFC 4180 CSV files.
+ *
+ * Expected columns: {@code input, actualOutput, expectedOutput, retrievalContext, context}. + * List fields (retrievalContext, context) use pipe ({@code |}) as separator.
+ */ +public final class CsvDatasetLoader implements DatasetLoader { + + private static final Logger LOG = LoggerFactory.getLogger(CsvDatasetLoader.class); + private static final String PIPE_SEPARATOR = "\\|"; + + @Override + public EvalDataset load(Path path) { + LOG.debug("Loading CSV dataset from {}", path); + try (var reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load CSV dataset from " + path, e); + } + } + + @Override + public EvalDataset load(InputStream inputStream) { + LOG.debug("Loading CSV dataset from input stream"); + try (var reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load CSV dataset from input stream", e); + } + } + + private EvalDataset parse(BufferedReader reader) throws IOException { + String headerLine = reader.readLine(); + if (headerLine == null) { + throw new DatasetException("CSV file is empty — expected header row"); + } + + String[] headers = parseCsvLine(headerLine); + int inputIdx = indexOf(headers, "input"); + int actualOutputIdx = indexOf(headers, "actualOutput"); + int expectedOutputIdx = indexOf(headers, "expectedOutput"); + int retrievalContextIdx = indexOf(headers, "retrievalContext"); + int contextIdx = indexOf(headers, "context"); + + if (inputIdx == -1) { + throw new DatasetException("CSV header must contain 'input' column"); + } + + ListList fields use pipe ({@code |}) as separator.
+ */ +public final class CsvDatasetWriter implements DatasetWriter { + + private static final Logger LOG = LoggerFactory.getLogger(CsvDatasetWriter.class); + private static final String HEADER = "input,actualOutput,expectedOutput,retrievalContext,context"; + + @Override + public void write(EvalDataset dataset, Path path) { + LOG.debug("Writing CSV dataset to {}", path); + try (var out = Files.newOutputStream(path)) { + write(dataset, out); + } catch (IOException e) { + throw new DatasetException("Failed to write CSV dataset to " + path, e); + } + } + + @Override + public void write(EvalDataset dataset, OutputStream outputStream) { + try { + Writer writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8); + writer.write(HEADER); + writer.write('\n'); + + for (AgentTestCase tc : dataset.getTestCases()) { + writer.write(escapeCsv(tc.getInput())); + writer.write(','); + writer.write(escapeCsv(tc.getActualOutput())); + writer.write(','); + writer.write(escapeCsv(tc.getExpectedOutput())); + writer.write(','); + writer.write(escapeCsv(joinPipe(tc.getRetrievalContext()))); + writer.write(','); + writer.write(escapeCsv(joinPipe(tc.getContext()))); + writer.write('\n'); + } + + writer.flush(); + } catch (IOException e) { + throw new DatasetException("Failed to write CSV dataset", e); + } + } + + static String escapeCsv(String value) { + if (value == null) return ""; + if (value.contains(",") || value.contains("\"") || value.contains("\n")) { + return "\"" + value.replace("\"", "\"\"") + "\""; + } + return value; + } + + private static String joinPipe(ListEach line contains a single JSON object representing an {@link AgentTestCase}.
+ */ +public final class JsonlDatasetLoader implements DatasetLoader { + + private static final Logger LOG = LoggerFactory.getLogger(JsonlDatasetLoader.class); + + private final ObjectMapper mapper; + + public JsonlDatasetLoader() { + this(new ObjectMapper()); + } + + public JsonlDatasetLoader(ObjectMapper mapper) { + this.mapper = mapper; + } + + @Override + public EvalDataset load(Path path) { + LOG.debug("Loading JSONL dataset from {}", path); + try (var reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load JSONL dataset from " + path, e); + } + } + + @Override + public EvalDataset load(InputStream inputStream) { + LOG.debug("Loading JSONL dataset from input stream"); + try (var reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return parse(reader); + } catch (IOException e) { + throw new DatasetException("Failed to load JSONL dataset from input stream", e); + } + } + + private EvalDataset parse(BufferedReader reader) throws IOException { + ListEach line contains a single JSON object representing an {@link AgentTestCase}.
+ */ +public final class JsonlDatasetWriter implements DatasetWriter { + + private static final Logger LOG = LoggerFactory.getLogger(JsonlDatasetWriter.class); + + private final ObjectMapper mapper; + + public JsonlDatasetWriter() { + this(new ObjectMapper()); + } + + public JsonlDatasetWriter(ObjectMapper mapper) { + this.mapper = mapper; + } + + @Override + public void write(EvalDataset dataset, Path path) { + LOG.debug("Writing JSONL dataset to {}", path); + try (var out = Files.newOutputStream(path)) { + write(dataset, out); + } catch (IOException e) { + throw new DatasetException("Failed to write JSONL dataset to " + path, e); + } + } + + @Override + public void write(EvalDataset dataset, OutputStream outputStream) { + try { + Writer writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8); + for (AgentTestCase tc : dataset.getTestCases()) { + writer.write(mapper.writeValueAsString(tc)); + writer.write('\n'); + } + writer.flush(); + } catch (IOException e) { + throw new DatasetException("Failed to write JSONL dataset", e); + } + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/DatasetLoadersTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/DatasetLoadersTest.java new file mode 100644 index 0000000..f9faaaa --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/DatasetLoadersTest.java @@ -0,0 +1,57 @@ +package com.agenteval.datasets; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class DatasetLoadersTest { + + @Test + void shouldAutoDetectJson(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.json"); + Files.writeString(file, "[{\"input\":\"Hello\"}]"); + + EvalDataset dataset = DatasetLoaders.forPath(file); + assertThat(dataset.size()).isEqualTo(1); + } + + @Test + void shouldAutoDetectJsonl(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.jsonl"); + Files.writeString(file, "{\"input\":\"Hello\"}\n"); + + EvalDataset dataset = DatasetLoaders.forPath(file); + assertThat(dataset.size()).isEqualTo(1); + } + + @Test + void shouldAutoDetectCsv(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.csv"); + Files.writeString(file, "input,actualOutput\nHello,World\n"); + + EvalDataset dataset = DatasetLoaders.forPath(file); + assertThat(dataset.size()).isEqualTo(1); + } + + @Test + void shouldThrowOnUnsupportedFormat(@TempDir Path tmpDir) throws Exception { + Path file = tmpDir.resolve("data.xml"); + Files.writeString(file, ""); + + assertThatThrownBy(() -> DatasetLoaders.forPath(file)) + .isInstanceOf(DatasetException.class) + .hasMessageContaining("Unsupported"); + } + + @Test + void shouldDetectFormats() { + assertThat(DatasetFormat.detect(Path.of("data.json"))).isEqualTo(DatasetFormat.JSON); + assertThat(DatasetFormat.detect(Path.of("data.jsonl"))).isEqualTo(DatasetFormat.JSONL); + assertThat(DatasetFormat.detect(Path.of("data.csv"))).isEqualTo(DatasetFormat.CSV); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetLoaderTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetLoaderTest.java new file mode 100644 index 0000000..65eebb1 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetLoaderTest.java @@ -0,0 +1,65 @@ +package com.agenteval.datasets.csv; + +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class CsvDatasetLoaderTest { + + @Test + void shouldLoadFromClasspath() { + var is = getClass().getClassLoader().getResourceAsStream("test-dataset.csv"); + EvalDataset dataset = new CsvDatasetLoader().load(is); + + assertThat(dataset.size()).isEqualTo(2); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("What is Java?"); + assertThat(dataset.getTestCases().get(0).getRetrievalContext()).hasSize(2); + } + + @Test + void shouldLoadFromPath(@TempDir Path tmpDir) throws Exception { + Path csvFile = tmpDir.resolve("test.csv"); + Files.writeString(csvFile, "input,actualOutput\nHello,World\n"); + + EvalDataset dataset = new CsvDatasetLoader().load(csvFile); + + assertThat(dataset.size()).isEqualTo(1); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("Hello"); + assertThat(dataset.getTestCases().get(0).getActualOutput()).isEqualTo("World"); + } + + @Test + void shouldHandleQuotedFields() { + String csv = "input,actualOutput\n\"Hello, World\",\"He said \"\"hi\"\"\"\n"; + var is = new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)); + EvalDataset dataset = new CsvDatasetLoader().load(is); + + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("Hello, World"); + assertThat(dataset.getTestCases().get(0).getActualOutput()).isEqualTo("He said \"hi\""); + } + + @Test + void shouldRejectEmptyFile() { + var is = new ByteArrayInputStream(new byte[0]); + assertThatThrownBy(() -> new CsvDatasetLoader().load(is)) + .hasMessageContaining("empty"); + } + + @Test + void shouldParsePipeSeparatedLists() { + String csv = "input,retrievalContext\nquery,doc1|doc2|doc3\n"; + var is = new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)); + EvalDataset dataset = new CsvDatasetLoader().load(is); + + assertThat(dataset.getTestCases().get(0).getRetrievalContext()) + .containsExactly("doc1", "doc2", "doc3"); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetWriterTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetWriterTest.java new file mode 100644 index 0000000..9368e21 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/csv/CsvDatasetWriterTest.java @@ -0,0 +1,73 @@ +package com.agenteval.datasets.csv; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class CsvDatasetWriterTest { + + @Test + void shouldWriteToOutputStream() { + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder() + .input("Hello") + .actualOutput("World") + .build())) + .build(); + + var out = new ByteArrayOutputStream(); + new CsvDatasetWriter().write(dataset, out); + + String csv = out.toString(); + assertThat(csv).contains("input,actualOutput,expectedOutput,retrievalContext,context"); + assertThat(csv).contains("Hello,World"); + } + + @Test + void shouldWriteToFile(@TempDir Path tmpDir) { + Path csvFile = tmpDir.resolve("out.csv"); + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder() + .input("q") + .actualOutput("a") + .build())) + .build(); + + new CsvDatasetWriter().write(dataset, csvFile); + + assertThat(csvFile).exists(); + } + + @Test + void shouldEscapeCommasAndQuotes() { + assertThat(CsvDatasetWriter.escapeCsv("hello,world")).isEqualTo("\"hello,world\""); + assertThat(CsvDatasetWriter.escapeCsv("say \"hi\"")).isEqualTo("\"say \"\"hi\"\"\""); + assertThat(CsvDatasetWriter.escapeCsv("simple")).isEqualTo("simple"); + assertThat(CsvDatasetWriter.escapeCsv(null)).isEmpty(); + } + + @Test + void shouldWritePipeSeparatedLists() { + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder() + .input("q") + .retrievalContext(List.of("doc1", "doc2")) + .build())) + .build(); + + var out = new ByteArrayOutputStream(); + new CsvDatasetWriter().write(dataset, out); + + assertThat(out.toString()).contains("doc1|doc2"); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetLoaderTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetLoaderTest.java new file mode 100644 index 0000000..1928ff2 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetLoaderTest.java @@ -0,0 +1,57 @@ +package com.agenteval.datasets.jsonl; + +import com.agenteval.datasets.DatasetException; +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class JsonlDatasetLoaderTest { + + @Test + void shouldLoadFromClasspath() { + var is = getClass().getClassLoader().getResourceAsStream("test-dataset.jsonl"); + EvalDataset dataset = new JsonlDatasetLoader().load(is); + + assertThat(dataset.size()).isEqualTo(2); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("What is Java?"); + assertThat(dataset.getTestCases().get(0).getRetrievalContext()).hasSize(2); + } + + @Test + void shouldLoadFromPath(@TempDir Path tmpDir) throws Exception { + Path jsonlFile = tmpDir.resolve("test.jsonl"); + Files.writeString(jsonlFile, "{\"input\":\"Hello\",\"actualOutput\":\"World\"}\n"); + + EvalDataset dataset = new JsonlDatasetLoader().load(jsonlFile); + + assertThat(dataset.size()).isEqualTo(1); + assertThat(dataset.getTestCases().get(0).getInput()).isEqualTo("Hello"); + } + + @Test + void shouldSkipBlankLines() { + String jsonl = "{\"input\":\"q1\"}\n\n{\"input\":\"q2\"}\n"; + var is = new ByteArrayInputStream(jsonl.getBytes(StandardCharsets.UTF_8)); + EvalDataset dataset = new JsonlDatasetLoader().load(is); + + assertThat(dataset.size()).isEqualTo(2); + } + + @Test + void shouldThrowOnInvalidJson() { + String jsonl = "not valid json\n"; + var is = new ByteArrayInputStream(jsonl.getBytes(StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> new JsonlDatasetLoader().load(is)) + .isInstanceOf(DatasetException.class) + .hasMessageContaining("line 1"); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetWriterTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetWriterTest.java new file mode 100644 index 0000000..40f5410 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/jsonl/JsonlDatasetWriterTest.java @@ -0,0 +1,45 @@ +package com.agenteval.datasets.jsonl; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class JsonlDatasetWriterTest { + + @Test + void shouldWriteOneJsonPerLine() { + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder().input("q1").build(), + AgentTestCase.builder().input("q2").build())) + .build(); + + var out = new ByteArrayOutputStream(); + new JsonlDatasetWriter().write(dataset, out); + + String[] lines = out.toString().trim().split("\n"); + assertThat(lines).hasSize(2); + assertThat(lines[0]).contains("\"input\":\"q1\""); + assertThat(lines[1]).contains("\"input\":\"q2\""); + } + + @Test + void shouldWriteToFile(@TempDir Path tmpDir) { + Path jsonlFile = tmpDir.resolve("out.jsonl"); + var dataset = EvalDataset.builder() + .testCases(List.of( + AgentTestCase.builder().input("q").build())) + .build(); + + new JsonlDatasetWriter().write(dataset, jsonlFile); + + assertThat(jsonlFile).exists(); + } +} diff --git a/agenteval-datasets/src/test/resources/test-dataset.csv b/agenteval-datasets/src/test/resources/test-dataset.csv new file mode 100644 index 0000000..22032b0 --- /dev/null +++ b/agenteval-datasets/src/test/resources/test-dataset.csv @@ -0,0 +1,3 @@ +input,actualOutput,expectedOutput,retrievalContext,context +What is Java?,Java is a programming language.,Java is a programming language developed by Sun.,Java was created by Sun|Java runs on JVM, +What is Python?,Python is easy to learn.,Python is a high-level language.,Python was created by Guido,Python docs diff --git a/agenteval-datasets/src/test/resources/test-dataset.jsonl b/agenteval-datasets/src/test/resources/test-dataset.jsonl new file mode 100644 index 0000000..3090a97 --- /dev/null +++ b/agenteval-datasets/src/test/resources/test-dataset.jsonl @@ -0,0 +1,2 @@ +{"input":"What is Java?","actualOutput":"Java is a programming language.","expectedOutput":"Java is a programming language developed by Sun.","retrievalContext":["Java was created by Sun","Java runs on JVM"]} +{"input":"What is Python?","actualOutput":"Python is easy to learn.","expectedOutput":"Python is a high-level language.","retrievalContext":["Python was created by Guido"]} diff --git a/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java b/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java index 3fcb41e..660733d 100644 --- a/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java +++ b/agenteval-junit5/src/main/java/com/agenteval/junit5/extension/DatasetArgumentsProvider.java @@ -1,8 +1,11 @@ package com.agenteval.junit5.extension; import com.agenteval.datasets.DatasetException; +import com.agenteval.datasets.DatasetFormat; import com.agenteval.datasets.EvalDataset; +import com.agenteval.datasets.csv.CsvDatasetLoader; import com.agenteval.datasets.json.JsonDatasetLoader; +import com.agenteval.datasets.jsonl.JsonlDatasetLoader; import com.agenteval.junit5.annotation.DatasetSource; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.provider.Arguments; @@ -10,11 +13,14 @@ import org.junit.jupiter.params.support.AnnotationConsumer; import java.io.InputStream; +import java.nio.file.Path; import java.util.stream.Stream; /** * JUnit 5 {@link ArgumentsProvider} that loads {@code AgentTestCase} instances - * from a JSON dataset specified by {@link DatasetSource}. + * from a dataset specified by {@link DatasetSource}. + * + *Auto-detects format from the resource path extension (.json, .jsonl, .csv).
*/ public final class DatasetArgumentsProvider implements ArgumentsProvider, AnnotationConsumerSerializes the full {@link EvalResult} structure including per-case scores, + * metric summaries, and aggregate statistics.
+ */ +public final class JsonReporter implements EvalReporter { + + private static final Logger LOG = LoggerFactory.getLogger(JsonReporter.class); + private static final ObjectMapper MAPPER = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); + + private final Path outputPath; + + public JsonReporter(Path outputPath) { + this.outputPath = Objects.requireNonNull(outputPath, "outputPath must not be null"); + } + + @Override + public void report(EvalResult result) { + LOG.debug("Writing JSON report to {}", outputPath); + + ObjectNode root = MAPPER.createObjectNode(); + root.put("averageScore", result.averageScore()); + root.put("passRate", result.passRate()); + root.put("totalCases", result.caseResults().size()); + root.put("failedCases", result.failedCases().size()); + root.put("durationMs", result.durationMs()); + + ObjectNode metricAverages = root.putObject("metricAverages"); + result.averageScoresByMetric().forEach(metricAverages::put); + + ArrayNode cases = root.putArray("caseResults"); + for (CaseResult cr : result.caseResults()) { + ObjectNode caseNode = cases.addObject(); + caseNode.put("input", cr.testCase().getInput()); + caseNode.put("passed", cr.passed()); + caseNode.put("averageScore", cr.averageScore()); + + ObjectNode scores = caseNode.putObject("scores"); + for (Map.EntrySupports environment variable resolution in values using {@code ${ENV_VAR}} syntax.
+ */ +public final class AgentEvalConfigLoader { + + private static final Logger LOG = LoggerFactory.getLogger(AgentEvalConfigLoader.class); + private static final Pattern ENV_VAR = Pattern.compile("\\$\\{(\\w+)}"); + private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private AgentEvalConfigLoader() {} + + /** + * Loads configuration from the given YAML file path. + * + * @param path the path to agenteval.yaml + * @return the populated config builder (call {@code .build()} to finalize) + */ + public static AgentEvalConfig.Builder load(Path path) { + LOG.debug("Loading AgentEval config from {}", path); + try { + String content = Files.readString(path); + return parse(content); + } catch (IOException e) { + throw new ConfigException("Failed to load config from " + path, e); + } + } + + /** + * Loads configuration from an input stream. + */ + public static AgentEvalConfig.Builder load(InputStream inputStream) { + LOG.debug("Loading AgentEval config from input stream"); + try { + String content = new String(inputStream.readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + return parse(content); + } catch (IOException e) { + throw new ConfigException("Failed to load config from input stream", e); + } + } + + static AgentEvalConfig.Builder parse(String yamlContent) { + String resolved = resolveEnvVars(yamlContent); + try { + YamlConfigModel model = YAML_MAPPER.readValue(resolved, YamlConfigModel.class); + return toBuilder(model); + } catch (IOException e) { + throw new ConfigException("Failed to parse YAML config: " + e.getMessage(), e); + } + } + + private static AgentEvalConfig.Builder toBuilder(YamlConfigModel model) { + var builder = AgentEvalConfig.builder(); + if (model == null) { + return builder; + } + + if (model.getDefaults() != null) { + var defaults = model.getDefaults(); + if (defaults.getMaxRetries() != null) { + builder.maxRetries(defaults.getMaxRetries()); + } + if (defaults.getRetryOnRateLimit() != null) { + builder.retryOnRateLimit(defaults.getRetryOnRateLimit()); + } + if (defaults.getMaxConcurrentJudgeCalls() != null) { + builder.maxConcurrentJudgeCalls(defaults.getMaxConcurrentJudgeCalls()); + } + } + + if (model.getCost() != null && model.getCost().getBudget() != null) { + builder.costBudget(model.getCost().getBudget()); + } + + return builder; + } + + static String resolveEnvVars(String content) { + Matcher matcher = ENV_VAR.matcher(content); + StringBuilder result = new StringBuilder(); + while (matcher.find()) { + String envName = matcher.group(1); + String envValue = System.getenv(envName); + matcher.appendReplacement(result, + Matcher.quoteReplacement(envValue != null ? envValue : "")); + } + matcher.appendTail(result); + return result.toString(); + } + + /** + * Returns the parsed {@link YamlConfigModel} for advanced use cases. + */ + public static YamlConfigModel loadModel(Path path) { + try { + String content = resolveEnvVars(Files.readString(path)); + return YAML_MAPPER.readValue(content, YamlConfigModel.class); + } catch (IOException e) { + throw new ConfigException("Failed to load config model from " + path, e); + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/config/ConfigException.java b/agenteval-core/src/main/java/com/agenteval/core/config/ConfigException.java new file mode 100644 index 0000000..6d52bcc --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/config/ConfigException.java @@ -0,0 +1,17 @@ +package com.agenteval.core.config; + +/** + * Unchecked exception for configuration loading errors. + */ +public class ConfigException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public ConfigException(String message) { + super(message); + } + + public ConfigException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java b/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java new file mode 100644 index 0000000..5db87c0 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java @@ -0,0 +1,83 @@ +package com.agenteval.core.config; + +import java.math.BigDecimal; + +/** + * POJO representing the {@code agenteval.yaml} configuration file structure. + * + *Used for Jackson YAML deserialization. Maps to {@link AgentEvalConfig} + * via {@link AgentEvalConfigLoader}.
+ */ +public final class YamlConfigModel { + + private JudgeSection judge; + private EmbeddingSection embedding; + private DefaultsSection defaults; + private CostSection cost; + + public JudgeSection getJudge() { return judge; } + public void setJudge(JudgeSection judge) { this.judge = judge; } + public EmbeddingSection getEmbedding() { return embedding; } + public void setEmbedding(EmbeddingSection embedding) { this.embedding = embedding; } + public DefaultsSection getDefaults() { return defaults; } + public void setDefaults(DefaultsSection defaults) { this.defaults = defaults; } + public CostSection getCost() { return cost; } + public void setCost(CostSection cost) { this.cost = cost; } + + public static final class JudgeSection { + private String provider; + private String model; + private String apiKey; + private String baseUrl; + + public String getProvider() { return provider; } + public void setProvider(String provider) { this.provider = provider; } + public String getModel() { return model; } + public void setModel(String model) { this.model = model; } + public String getApiKey() { return apiKey; } + public void setApiKey(String apiKey) { this.apiKey = apiKey; } + public String getBaseUrl() { return baseUrl; } + public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + } + + public static final class EmbeddingSection { + private String provider; + private String model; + private String apiKey; + private String baseUrl; + + public String getProvider() { return provider; } + public void setProvider(String provider) { this.provider = provider; } + public String getModel() { return model; } + public void setModel(String model) { this.model = model; } + public String getApiKey() { return apiKey; } + public void setApiKey(String apiKey) { this.apiKey = apiKey; } + public String getBaseUrl() { return baseUrl; } + public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + } + + public static final class DefaultsSection { + private Double threshold; + private Integer maxRetries; + private Boolean retryOnRateLimit; + private Integer maxConcurrentJudgeCalls; + + public Double getThreshold() { return threshold; } + public void setThreshold(Double threshold) { this.threshold = threshold; } + public Integer getMaxRetries() { return maxRetries; } + public void setMaxRetries(Integer maxRetries) { this.maxRetries = maxRetries; } + public Boolean getRetryOnRateLimit() { return retryOnRateLimit; } + public void setRetryOnRateLimit(Boolean retryOnRateLimit) { + this.retryOnRateLimit = retryOnRateLimit; + } + public Integer getMaxConcurrentJudgeCalls() { return maxConcurrentJudgeCalls; } + public void setMaxConcurrentJudgeCalls(Integer max) { this.maxConcurrentJudgeCalls = max; } + } + + public static final class CostSection { + private BigDecimal budget; + + public BigDecimal getBudget() { return budget; } + public void setBudget(BigDecimal budget) { this.budget = budget; } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/cost/BudgetExceededException.java b/agenteval-core/src/main/java/com/agenteval/core/cost/BudgetExceededException.java new file mode 100644 index 0000000..eaa7e9b --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/cost/BudgetExceededException.java @@ -0,0 +1,23 @@ +package com.agenteval.core.cost; + +import java.math.BigDecimal; + +/** + * Thrown when the cost budget has been exceeded. + */ +public class BudgetExceededException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final BigDecimal currentCost; + private final BigDecimal budget; + + public BudgetExceededException(BigDecimal currentCost, BigDecimal budget) { + super(String.format("Budget exceeded: $%s / $%s", currentCost, budget)); + this.currentCost = currentCost; + this.budget = budget; + } + + public BigDecimal getCurrentCost() { return currentCost; } + public BigDecimal getBudget() { return budget; } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/cost/CostSummary.java b/agenteval-core/src/main/java/com/agenteval/core/cost/CostSummary.java new file mode 100644 index 0000000..2b3b89b --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/cost/CostSummary.java @@ -0,0 +1,24 @@ +package com.agenteval.core.cost; + +import java.math.BigDecimal; + +/** + * Summary of accumulated costs from LLM judge calls. + * + * @param totalCost the total cost in USD + * @param totalInputTokens total input tokens consumed + * @param totalOutputTokens total output tokens consumed + */ +public record CostSummary( + BigDecimal totalCost, + long totalInputTokens, + long totalOutputTokens +) { + public CostSummary { + if (totalCost == null) totalCost = BigDecimal.ZERO; + } + + public long totalTokens() { + return totalInputTokens + totalOutputTokens; + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/cost/CostTracker.java b/agenteval-core/src/main/java/com/agenteval/core/cost/CostTracker.java new file mode 100644 index 0000000..2568961 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/cost/CostTracker.java @@ -0,0 +1,79 @@ +package com.agenteval.core.cost; + +import com.agenteval.core.model.TokenUsage; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Thread-safe accumulator for tracking LLM usage costs. + * + *Records token usage with a pricing model and tracks total cost + * against an optional budget.
+ */ +public final class CostTracker { + + private static final BigDecimal ONE_MILLION = new BigDecimal("1000000"); + + private final BigDecimal budget; + private final AtomicReference{@code
* var judge = JudgeModels.openai("gpt-4o");
* var judge = JudgeModels.anthropic("claude-sonnet-4-20250514");
+ * var judge = JudgeModels.ollama("llama3");
* var judge = JudgeModels.openai(JudgeConfig.builder()
* .apiKey("sk-...")
* .model("gpt-4o")
@@ -26,6 +28,7 @@ public final class JudgeModels {
private static final String ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY";
private static final String OPENAI_BASE_URL = "https://api.openai.com";
private static final String ANTHROPIC_BASE_URL = "https://api.anthropic.com";
+ private static final String OLLAMA_BASE_URL = "http://localhost:11434";
private JudgeModels() {}
@@ -67,6 +70,24 @@ public static JudgeModel anthropic(JudgeConfig config) {
return new AnthropicJudgeModel(config);
}
+ /**
+ * Creates an Ollama judge model using the given model ID.
+ * Defaults to {@code localhost:11434}. No API key required.
+ */
+ public static JudgeModel ollama(String model) {
+ return ollama(JudgeConfig.builder()
+ .model(model)
+ .baseUrl(OLLAMA_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates an Ollama judge model with full configuration.
+ */
+ public static JudgeModel ollama(JudgeConfig config) {
+ return new OllamaJudgeModel(config);
+ }
+
private static String resolveApiKey(String envVar, String providerName) {
String key = System.getenv(envVar);
if (key == null || key.isBlank()) {
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java b/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java
index 75c90fa..5edf954 100644
--- a/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/config/JudgeConfig.java
@@ -16,7 +16,7 @@ public final class JudgeConfig {
private final double temperature;
private JudgeConfig(Builder builder) {
- this.apiKey = Objects.requireNonNull(builder.apiKey, "apiKey must not be null");
+ this.apiKey = builder.apiKey;
this.model = Objects.requireNonNull(builder.model, "model must not be null");
this.baseUrl = Objects.requireNonNull(builder.baseUrl, "baseUrl must not be null");
this.timeout = builder.timeout;
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java
index ad701bd..d6f6df6 100644
--- a/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/AnthropicJudgeModel.java
@@ -29,6 +29,9 @@ public final class AnthropicJudgeModel extends AbstractHttpJudgeModel {
public AnthropicJudgeModel(JudgeConfig config) {
super(config);
+ if (config.getApiKey() == null || config.getApiKey().isBlank()) {
+ throw new JudgeException("Anthropic requires a non-null API key");
+ }
}
AnthropicJudgeModel(JudgeConfig config, HttpJudgeClient client) {
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/OllamaJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OllamaJudgeModel.java
new file mode 100644
index 0000000..69160e9
--- /dev/null
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OllamaJudgeModel.java
@@ -0,0 +1,88 @@
+package com.agenteval.judge.provider;
+
+import com.agenteval.core.model.TokenUsage;
+import com.agenteval.judge.JudgeException;
+import com.agenteval.judge.config.JudgeConfig;
+import com.agenteval.judge.http.HttpJudgeClient;
+import com.agenteval.judge.http.HttpJudgeRequest;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.util.Map;
+
+/**
+ * Ollama judge model provider.
+ *
+ * Sends requests to {@code POST /api/chat} with JSON response format.
+ * No API key required.
+ */
+public final class OllamaJudgeModel extends AbstractHttpJudgeModel {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final String DEFAULT_BASE_URL = "http://localhost:11434";
+ private static final String CHAT_PATH = "/api/chat";
+ private static final String SYSTEM_PROMPT =
+ "You are an evaluation judge. Respond ONLY with a JSON object "
+ + "containing \"score\" (a number between 0.0 and 1.0) "
+ + "and \"reason\" (a brief explanation).";
+
+ public OllamaJudgeModel(JudgeConfig config) {
+ super(config);
+ }
+
+ OllamaJudgeModel(JudgeConfig config, HttpJudgeClient client) {
+ super(config, client);
+ }
+
+ static String defaultBaseUrl() {
+ return DEFAULT_BASE_URL;
+ }
+
+ @Override
+ protected HttpJudgeRequest buildRequest(String prompt) {
+ try {
+ var body = MAPPER.createObjectNode();
+ body.put("model", config.getModel());
+ body.put("stream", false);
+ body.put("format", "json");
+
+ var messages = body.putArray("messages");
+
+ var systemMsg = messages.addObject();
+ systemMsg.put("role", "system");
+ systemMsg.put("content", SYSTEM_PROMPT);
+
+ var userMsg = messages.addObject();
+ userMsg.put("role", "user");
+ userMsg.put("content", prompt);
+
+ String url = config.getBaseUrl() + CHAT_PATH;
+ return new HttpJudgeRequest(url, Map.of(),
+ MAPPER.writeValueAsString(body));
+ } catch (Exception e) {
+ throw new JudgeException("Failed to build Ollama request", e);
+ }
+ }
+
+ @Override
+ protected String extractContent(String responseBody) {
+ JsonNode root = parseJson(responseBody);
+ JsonNode message = root.path("message");
+ if (message.isMissingNode()) {
+ throw new JudgeException("No message in Ollama response");
+ }
+ return message.path("content").asText("");
+ }
+
+ @Override
+ protected TokenUsage extractTokenUsage(String responseBody) {
+ JsonNode root = parseJson(responseBody);
+ int promptTokens = root.path("prompt_eval_count").asInt(0);
+ int completionTokens = root.path("eval_count").asInt(0);
+ if (promptTokens == 0 && completionTokens == 0) {
+ return null;
+ }
+ return new TokenUsage(promptTokens, completionTokens,
+ promptTokens + completionTokens);
+ }
+}
diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java
index dc05d53..96d6cf0 100644
--- a/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java
+++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/OpenAiJudgeModel.java
@@ -28,6 +28,9 @@ public final class OpenAiJudgeModel extends AbstractHttpJudgeModel {
public OpenAiJudgeModel(JudgeConfig config) {
super(config);
+ if (config.getApiKey() == null || config.getApiKey().isBlank()) {
+ throw new JudgeException("OpenAI requires a non-null API key");
+ }
}
OpenAiJudgeModel(JudgeConfig config, HttpJudgeClient client) {
diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java
index 948d174..04c5a1d 100644
--- a/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java
+++ b/agenteval-judge/src/test/java/com/agenteval/judge/config/JudgeConfigTest.java
@@ -42,12 +42,12 @@ void shouldBuildWithCustomValues() {
}
@Test
- void shouldRejectNullApiKey() {
- assertThatThrownBy(() -> JudgeConfig.builder()
+ void shouldAllowNullApiKey() {
+ var config = JudgeConfig.builder()
.model("gpt-4o")
.baseUrl("https://api.openai.com")
- .build())
- .isInstanceOf(NullPointerException.class);
+ .build();
+ assertThat(config.getApiKey()).isNull();
}
@Test
diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/provider/OllamaJudgeModelTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/provider/OllamaJudgeModelTest.java
new file mode 100644
index 0000000..10c663a
--- /dev/null
+++ b/agenteval-judge/src/test/java/com/agenteval/judge/provider/OllamaJudgeModelTest.java
@@ -0,0 +1,82 @@
+package com.agenteval.judge.provider;
+
+import com.agenteval.core.judge.JudgeResponse;
+import com.agenteval.judge.config.JudgeConfig;
+import com.agenteval.judge.http.HttpJudgeClient;
+import com.agenteval.judge.http.HttpJudgeResponse;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class OllamaJudgeModelTest {
+
+ private HttpJudgeClient client;
+ private JudgeConfig config;
+
+ @BeforeEach
+ void setUp() {
+ client = mock(HttpJudgeClient.class);
+ config = JudgeConfig.builder()
+ .model("llama3")
+ .baseUrl("http://localhost:11434")
+ .build();
+ }
+
+ @Test
+ void shouldParseOllamaResponse() {
+ String responseBody = """
+ {
+ "message": {"role": "assistant", "content": "{\\"score\\": 0.85, \\"reason\\": \\"Good answer\\"}"},
+ "prompt_eval_count": 50,
+ "eval_count": 100
+ }
+ """;
+ when(client.send(any())).thenReturn(
+ new HttpJudgeResponse(200, responseBody, null));
+
+ var model = new OllamaJudgeModel(config, client);
+ JudgeResponse response = model.judge("test prompt");
+
+ assertThat(response.score()).isCloseTo(0.85, within(0.001));
+ assertThat(response.reason()).isEqualTo("Good answer");
+ assertThat(response.tokenUsage()).isNotNull();
+ assertThat(response.tokenUsage().inputTokens()).isEqualTo(50);
+ assertThat(response.tokenUsage().outputTokens()).isEqualTo(100);
+ }
+
+ @Test
+ void shouldHandleMissingTokenUsage() {
+ String responseBody = """
+ {"message": {"role": "assistant", "content": "{\\"score\\": 0.5, \\"reason\\": \\"OK\\"}"}}
+ """;
+ when(client.send(any())).thenReturn(
+ new HttpJudgeResponse(200, responseBody, null));
+
+ var model = new OllamaJudgeModel(config, client);
+ JudgeResponse response = model.judge("test");
+
+ assertThat(response.score()).isCloseTo(0.5, within(0.001));
+ assertThat(response.tokenUsage()).isNull();
+ }
+
+ @Test
+ void shouldReturnModelId() {
+ var model = new OllamaJudgeModel(config, client);
+ assertThat(model.modelId()).isEqualTo("llama3");
+ }
+
+ @Test
+ void shouldNotRequireApiKey() {
+ var noKeyConfig = JudgeConfig.builder()
+ .model("llama3")
+ .baseUrl("http://localhost:11434")
+ .build();
+ var model = new OllamaJudgeModel(noKeyConfig, client);
+ assertThat(model.modelId()).isEqualTo("llama3");
+ }
+}
From 91e61142e33c68dc76529438129e81e42e68bc82 Mon Sep 17 00:00:00 2001
From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com>
Date: Thu, 12 Mar 2026 18:51:18 +0530
Subject: [PATCH 8/9] Add Spring AI and LangChain4j framework integrations
agenteval-spring-ai:
- SpringAiCapture: wraps ChatModel calls as AgentTestCase
- SpringAiTestCaseBuilder: converts ChatResponse to AgentTestCase
- SpringAiAdvisorInterceptor: CallAdvisor capturing RAG retrieval context
- AgentEvalAutoConfiguration for Spring Boot auto-config
- Uses Spring AI 1.0 GA artifacts (spring-ai-model, spring-ai-client-chat)
agenteval-langchain4j:
- LangChain4jCapture: wraps ChatLanguageModel calls as AgentTestCase
- LangChain4jTestCaseBuilder: converts AiMessage response to AgentTestCase
- LangChain4jContentRetrieverCapture: wraps ContentRetriever for context
Both modules use provided-scope dependencies so users bring their own
framework version.
---
.../datasets/csv/CsvDatasetLoader.java | 2 -
agenteval-langchain4j/pom.xml | 37 +++++++++++
.../langchain4j/LangChain4jCapture.java | 49 ++++++++++++++
.../LangChain4jContentRetrieverCapture.java | 49 ++++++++++++++
.../LangChain4jTestCaseBuilder.java | 56 ++++++++++++++++
agenteval-spring-ai/pom.xml | 56 ++++++++++++++++
.../spring/ai/SpringAiAdvisorInterceptor.java | 64 +++++++++++++++++++
.../agenteval/spring/ai/SpringAiCapture.java | 48 ++++++++++++++
.../spring/ai/SpringAiTestCaseBuilder.java | 47 ++++++++++++++
.../AgentEvalAutoConfiguration.java | 21 ++++++
10 files changed, 427 insertions(+), 2 deletions(-)
create mode 100644 agenteval-langchain4j/pom.xml
create mode 100644 agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jCapture.java
create mode 100644 agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jContentRetrieverCapture.java
create mode 100644 agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jTestCaseBuilder.java
create mode 100644 agenteval-spring-ai/pom.xml
create mode 100644 agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
create mode 100644 agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiCapture.java
create mode 100644 agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiTestCaseBuilder.java
create mode 100644 agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/autoconfigure/AgentEvalAutoConfiguration.java
diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java
index ae0b224..4cbb58c 100644
--- a/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java
+++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/csv/CsvDatasetLoader.java
@@ -69,9 +69,7 @@ private EvalDataset parse(BufferedReader reader) throws IOException {
List testCases = new ArrayList<>();
String line;
- int lineNum = 1;
while ((line = reader.readLine()) != null) {
- lineNum++;
if (line.isBlank()) continue;
String[] fields = parseCsvLine(line);
diff --git a/agenteval-langchain4j/pom.xml b/agenteval-langchain4j/pom.xml
new file mode 100644
index 0000000..71eb48b
--- /dev/null
+++ b/agenteval-langchain4j/pom.xml
@@ -0,0 +1,37 @@
+
+
+ 4.0.0
+
+
+ com.agenteval
+ agenteval-parent
+ 0.1.0-SNAPSHOT
+
+
+ agenteval-langchain4j
+ AgentEval LangChain4j
+ LangChain4j auto-capture integration for AgentEval
+
+
+ 0.36.2
+
+
+
+
+ com.agenteval
+ agenteval-core
+
+
+ dev.langchain4j
+ langchain4j-core
+ ${langchain4j.version}
+ provided
+
+
+ org.slf4j
+ slf4j-api
+
+
+
diff --git a/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jCapture.java b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jCapture.java
new file mode 100644
index 0000000..588c671
--- /dev/null
+++ b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jCapture.java
@@ -0,0 +1,49 @@
+package com.agenteval.langchain4j;
+
+import com.agenteval.core.model.AgentTestCase;
+import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.data.message.UserMessage;
+import dev.langchain4j.model.chat.ChatLanguageModel;
+import dev.langchain4j.model.output.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+/**
+ * Captures LangChain4j {@link ChatLanguageModel} interactions as {@link AgentTestCase} instances.
+ *
+ * {@code
+ * var capture = new LangChain4jCapture(chatModel);
+ * AgentTestCase testCase = capture.call("What is Java?");
+ * }
+ */
+public final class LangChain4jCapture {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LangChain4jCapture.class);
+
+ private final ChatLanguageModel model;
+
+ public LangChain4jCapture(ChatLanguageModel model) {
+ this.model = Objects.requireNonNull(model, "model must not be null");
+ }
+
+ /**
+ * Calls the chat model and captures the interaction as an AgentTestCase.
+ *
+ * @param input the user prompt
+ * @return the captured test case
+ */
+ public AgentTestCase call(String input) {
+ LOG.debug("Capturing LangChain4j call for input: {}",
+ input.length() > 100 ? input.substring(0, 100) + "..." : input);
+
+ long start = System.currentTimeMillis();
+ Response response = model.generate(UserMessage.from(input));
+ long latency = System.currentTimeMillis() - start;
+
+ AgentTestCase testCase = LangChain4jTestCaseBuilder.fromResponse(input, response);
+ testCase.setLatencyMs(latency);
+ return testCase;
+ }
+}
diff --git a/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jContentRetrieverCapture.java b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jContentRetrieverCapture.java
new file mode 100644
index 0000000..2add42a
--- /dev/null
+++ b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jContentRetrieverCapture.java
@@ -0,0 +1,49 @@
+package com.agenteval.langchain4j;
+
+import dev.langchain4j.rag.content.Content;
+import dev.langchain4j.rag.content.retriever.ContentRetriever;
+import dev.langchain4j.rag.query.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Wraps a LangChain4j {@link ContentRetriever} to capture retrieval context.
+ *
+ * Delegates all retrieval calls to the wrapped retriever while recording
+ * the content for use as retrieval context in AgentEval test cases.
+ */
+public final class LangChain4jContentRetrieverCapture implements ContentRetriever {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ LangChain4jContentRetrieverCapture.class);
+
+ private final ContentRetriever delegate;
+ private final List capturedContext = new ArrayList<>();
+
+ public LangChain4jContentRetrieverCapture(ContentRetriever delegate) {
+ this.delegate = Objects.requireNonNull(delegate, "delegate must not be null");
+ }
+
+ @Override
+ public List retrieve(Query query) {
+ List results = delegate.retrieve(query);
+ for (Content content : results) {
+ capturedContext.add(content.textSegment().text());
+ }
+ LOG.debug("Captured {} retrieval context items", results.size());
+ return results;
+ }
+
+ /**
+ * Returns the captured retrieval context and clears the buffer.
+ */
+ public List consumeCapturedContext() {
+ var result = List.copyOf(capturedContext);
+ capturedContext.clear();
+ return result;
+ }
+}
diff --git a/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jTestCaseBuilder.java b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jTestCaseBuilder.java
new file mode 100644
index 0000000..283b3aa
--- /dev/null
+++ b/agenteval-langchain4j/src/main/java/com/agenteval/langchain4j/LangChain4jTestCaseBuilder.java
@@ -0,0 +1,56 @@
+package com.agenteval.langchain4j;
+
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.TokenUsage;
+import com.agenteval.core.model.ToolCall;
+import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.model.output.Response;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Converts LangChain4j types to AgentEval {@link AgentTestCase}.
+ */
+public final class LangChain4jTestCaseBuilder {
+
+ private LangChain4jTestCaseBuilder() {}
+
+ /**
+ * Creates an AgentTestCase from a LangChain4j AiMessage response.
+ *
+ * @param input the user's input
+ * @param response the LangChain4j response
+ * @return a populated AgentTestCase
+ */
+ public static AgentTestCase fromResponse(String input, Response response) {
+ Objects.requireNonNull(input, "input must not be null");
+ Objects.requireNonNull(response, "response must not be null");
+
+ var builder = AgentTestCase.builder().input(input);
+
+ AiMessage message = response.content();
+ if (message != null) {
+ builder.actualOutput(message.text());
+
+ if (message.hasToolExecutionRequests()) {
+ List toolCalls = message.toolExecutionRequests().stream()
+ .map(req -> ToolCall.of(req.name(),
+ Map.of("arguments", req.arguments())))
+ .toList();
+ builder.toolCalls(toolCalls);
+ }
+ }
+
+ if (response.tokenUsage() != null) {
+ dev.langchain4j.model.output.TokenUsage usage = response.tokenUsage();
+ builder.tokenUsage(new TokenUsage(
+ usage.inputTokenCount(),
+ usage.outputTokenCount(),
+ usage.totalTokenCount()));
+ }
+
+ return builder.build();
+ }
+}
diff --git a/agenteval-spring-ai/pom.xml b/agenteval-spring-ai/pom.xml
new file mode 100644
index 0000000..ab16767
--- /dev/null
+++ b/agenteval-spring-ai/pom.xml
@@ -0,0 +1,56 @@
+
+
+ 4.0.0
+
+
+ com.agenteval
+ agenteval-parent
+ 0.1.0-SNAPSHOT
+
+
+ agenteval-spring-ai
+ AgentEval Spring AI
+ Spring AI auto-capture integration for AgentEval
+
+
+ 1.0.0
+ 3.4.2
+
+
+
+
+ com.agenteval
+ agenteval-core
+
+
+ org.springframework.ai
+ spring-ai-model
+ ${spring-ai.version}
+ provided
+
+
+ org.springframework.ai
+ spring-ai-client-chat
+ ${spring-ai.version}
+ provided
+
+
+ org.springframework.ai
+ spring-ai-commons
+ ${spring-ai.version}
+ provided
+
+
+ org.springframework.boot
+ spring-boot-autoconfigure
+ ${spring-boot.version}
+ true
+
+
+ org.slf4j
+ slf4j-api
+
+
+
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
new file mode 100644
index 0000000..76a52da
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
@@ -0,0 +1,64 @@
+package com.agenteval.spring.ai;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.chat.client.ChatClientRequest;
+import org.springframework.ai.chat.client.ChatClientResponse;
+import org.springframework.ai.chat.client.advisor.api.CallAdvisor;
+import org.springframework.ai.chat.client.advisor.api.CallAdvisorChain;
+import org.springframework.ai.document.Document;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Spring AI advisor interceptor that captures RAG retrieval context.
+ *
+ * Extracts document content from advised requests for use as
+ * retrieval context in AgentEval test cases.
+ */
+public final class SpringAiAdvisorInterceptor implements CallAdvisor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SpringAiAdvisorInterceptor.class);
+
+ private final List capturedContext = new ArrayList<>();
+
+ @Override
+ public ChatClientResponse adviseCall(ChatClientRequest request,
+ CallAdvisorChain chain) {
+ if (request.context() != null) {
+ var context = request.context();
+ Object docs = context.get("qa_advisor_retrieved_documents");
+ if (docs instanceof List docList) {
+ for (Object doc : docList) {
+ if (doc instanceof Document document) {
+ capturedContext.add(document.getText());
+ }
+ }
+ LOG.debug("Captured {} retrieval context documents",
+ capturedContext.size());
+ }
+ }
+
+ return chain.nextCall(request);
+ }
+
+ @Override
+ public String getName() {
+ return "AgentEvalAdvisorInterceptor";
+ }
+
+ @Override
+ public int getOrder() {
+ return 0;
+ }
+
+ /**
+ * Returns the captured retrieval context and clears the buffer.
+ */
+ public List consumeCapturedContext() {
+ var result = List.copyOf(capturedContext);
+ capturedContext.clear();
+ return result;
+ }
+}
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiCapture.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiCapture.java
new file mode 100644
index 0000000..0aa7abc
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiCapture.java
@@ -0,0 +1,48 @@
+package com.agenteval.spring.ai;
+
+import com.agenteval.core.model.AgentTestCase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.prompt.Prompt;
+
+import java.util.Objects;
+
+/**
+ * Captures Spring AI {@link ChatModel} interactions as {@link AgentTestCase} instances.
+ *
+ * {@code
+ * var capture = new SpringAiCapture(chatModel);
+ * AgentTestCase testCase = capture.call("What is Java?");
+ * }
+ */
+public final class SpringAiCapture {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SpringAiCapture.class);
+
+ private final ChatModel chatModel;
+
+ public SpringAiCapture(ChatModel chatModel) {
+ this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
+ }
+
+ /**
+ * Calls the chat model and captures the interaction as an AgentTestCase.
+ *
+ * @param input the user prompt
+ * @return the captured test case
+ */
+ public AgentTestCase call(String input) {
+ LOG.debug("Capturing Spring AI call for input: {}",
+ input.length() > 100 ? input.substring(0, 100) + "..." : input);
+
+ long start = System.currentTimeMillis();
+ ChatResponse response = chatModel.call(new Prompt(input));
+ long latency = System.currentTimeMillis() - start;
+
+ AgentTestCase testCase = SpringAiTestCaseBuilder.fromChatResponse(input, response);
+ testCase.setLatencyMs(latency);
+ return testCase;
+ }
+}
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiTestCaseBuilder.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiTestCaseBuilder.java
new file mode 100644
index 0000000..fff97dc
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiTestCaseBuilder.java
@@ -0,0 +1,47 @@
+package com.agenteval.spring.ai;
+
+import com.agenteval.core.model.AgentTestCase;
+import com.agenteval.core.model.TokenUsage;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+
+import java.util.Objects;
+
+/**
+ * Converts Spring AI types to AgentEval {@link AgentTestCase}.
+ */
+public final class SpringAiTestCaseBuilder {
+
+ private SpringAiTestCaseBuilder() {}
+
+ /**
+ * Creates an AgentTestCase from a Spring AI ChatResponse.
+ *
+ * @param input the user's input prompt
+ * @param response the Spring AI chat response
+ * @return a populated AgentTestCase
+ */
+ public static AgentTestCase fromChatResponse(String input, ChatResponse response) {
+ Objects.requireNonNull(input, "input must not be null");
+ Objects.requireNonNull(response, "response must not be null");
+
+ var builder = AgentTestCase.builder().input(input);
+
+ if (response.getResult() != null) {
+ Generation result = response.getResult();
+ if (result.getOutput() != null) {
+ builder.actualOutput(result.getOutput().getText());
+ }
+ }
+
+ if (response.getMetadata() != null && response.getMetadata().getUsage() != null) {
+ var usage = response.getMetadata().getUsage();
+ builder.tokenUsage(new TokenUsage(
+ (int) usage.getPromptTokens(),
+ (int) usage.getCompletionTokens(),
+ (int) usage.getTotalTokens()));
+ }
+
+ return builder.build();
+ }
+}
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/autoconfigure/AgentEvalAutoConfiguration.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/autoconfigure/AgentEvalAutoConfiguration.java
new file mode 100644
index 0000000..e09c194
--- /dev/null
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/autoconfigure/AgentEvalAutoConfiguration.java
@@ -0,0 +1,21 @@
+package com.agenteval.spring.ai.autoconfigure;
+
+import com.agenteval.spring.ai.SpringAiAdvisorInterceptor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Spring Boot auto-configuration for AgentEval Spring AI integration.
+ *
+ * Registers the advisor interceptor for automatic RAG context capture.
+ */
+@Configuration
+@ConditionalOnClass(name = "org.springframework.ai.chat.model.ChatModel")
+public class AgentEvalAutoConfiguration {
+
+ @Bean
+ public SpringAiAdvisorInterceptor agentEvalAdvisorInterceptor() {
+ return new SpringAiAdvisorInterceptor();
+ }
+}
From 6c458066328ca05c843ce6af17a0dd7418cd40a1 Mon Sep 17 00:00:00 2001
From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com>
Date: Thu, 12 Mar 2026 19:36:15 +0530
Subject: [PATCH 9/9] Fix @NonNull annotation warnings in
SpringAiAdvisorInterceptor
Match Spring AI's CallAdvisor and Advisor interface null contracts.
---
.../agenteval/spring/ai/SpringAiAdvisorInterceptor.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
index 76a52da..cd98561 100644
--- a/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
+++ b/agenteval-spring-ai/src/main/java/com/agenteval/spring/ai/SpringAiAdvisorInterceptor.java
@@ -7,6 +7,7 @@
import org.springframework.ai.chat.client.advisor.api.CallAdvisor;
import org.springframework.ai.chat.client.advisor.api.CallAdvisorChain;
import org.springframework.ai.document.Document;
+import org.springframework.lang.NonNull;
import java.util.ArrayList;
import java.util.List;
@@ -23,9 +24,10 @@ public final class SpringAiAdvisorInterceptor implements CallAdvisor {
private final List capturedContext = new ArrayList<>();
+ @NonNull
@Override
- public ChatClientResponse adviseCall(ChatClientRequest request,
- CallAdvisorChain chain) {
+ public ChatClientResponse adviseCall(@NonNull ChatClientRequest request,
+ @NonNull CallAdvisorChain chain) {
if (request.context() != null) {
var context = request.context();
Object docs = context.get("qa_advisor_retrieved_documents");
@@ -43,6 +45,7 @@ public ChatClientResponse adviseCall(ChatClientRequest request,
return chain.nextCall(request);
}
+ @NonNull
@Override
public String getName() {
return "AgentEvalAdvisorInterceptor";