From 3f0320054b4e7cf4bf91fe6e46c3f32110aec87d Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:42:58 +0530 Subject: [PATCH] Add agenteval-core module with data model, metrics, and eval engine Root Maven multi-module POM with Java 21, JUnit 5, Jackson, SLF4J. Core module includes: AgentTestCase, ConversationTestCase, ToolCall, ReasoningStep, TokenUsage, EvalScore records/classes, EvalMetric and CompositeMetric, JudgeModel/EmbeddingModel SPI interfaces, AgentEval evaluation engine, and AgentEvalConfig. Includes Checkstyle and SpotBugs lint enforcement. 71 tests passing. --- .gitignore | 7 + agent-eval-feature-spec.md | 761 ++++++++++++++++++ agenteval-core/pom.xml | 27 + .../core/config/AgentEvalConfig.java | 87 ++ .../core/embedding/EmbeddingModel.java | 25 + .../com/agenteval/core/eval/AgentEval.java | 91 +++ .../com/agenteval/core/eval/CaseResult.java | 42 + .../com/agenteval/core/eval/EvalResult.java | 88 ++ .../com/agenteval/core/judge/JudgeModel.java | 24 + .../agenteval/core/judge/JudgeResponse.java | 21 + .../core/metric/CompositeMetric.java | 139 ++++ .../core/metric/CompositeStrategy.java | 25 + .../com/agenteval/core/metric/EvalMetric.java | 26 + .../agenteval/core/model/AgentTestCase.java | 119 +++ .../core/model/ConversationTestCase.java | 53 ++ .../com/agenteval/core/model/EvalScore.java | 56 ++ .../agenteval/core/model/ReasoningStep.java | 31 + .../core/model/ReasoningStepType.java | 11 + .../com/agenteval/core/model/TokenUsage.java | 20 + .../com/agenteval/core/model/ToolCall.java | 33 + .../core/config/AgentEvalConfigTest.java | 60 ++ .../agenteval/core/eval/AgentEvalTest.java | 145 ++++ .../agenteval/core/eval/EvalResultTest.java | 80 ++ .../core/metric/CompositeMetricTest.java | 150 ++++ .../core/model/AgentTestCaseTest.java | 128 +++ .../core/model/ConversationTestCaseTest.java | 72 ++ .../agenteval/core/model/EvalScoreTest.java | 90 +++ .../core/model/ReasoningStepTest.java | 63 ++ .../agenteval/core/model/TokenUsageTest.java | 51 ++ .../agenteval/core/model/ToolCallTest.java | 63 ++ checkstyle.xml | 83 ++ pom.xml | 188 +++++ spotbugs-exclude.xml | 16 + 33 files changed, 2875 insertions(+) create mode 100644 agent-eval-feature-spec.md create mode 100644 agenteval-core/pom.xml create mode 100644 agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfig.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/embedding/EmbeddingModel.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/eval/AgentEval.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/eval/CaseResult.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/eval/EvalResult.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/judge/JudgeModel.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/judge/JudgeResponse.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/metric/CompositeMetric.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/metric/CompositeStrategy.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/metric/EvalMetric.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/model/AgentTestCase.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/model/ConversationTestCase.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/model/EvalScore.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStep.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStepType.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/model/TokenUsage.java create mode 100644 agenteval-core/src/main/java/com/agenteval/core/model/ToolCall.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/config/AgentEvalConfigTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/eval/AgentEvalTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/eval/EvalResultTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/metric/CompositeMetricTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/model/AgentTestCaseTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/model/ConversationTestCaseTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/model/EvalScoreTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/model/ReasoningStepTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/model/TokenUsageTest.java create mode 100644 agenteval-core/src/test/java/com/agenteval/core/model/ToolCallTest.java create mode 100644 checkstyle.xml create mode 100644 pom.xml create mode 100644 spotbugs-exclude.xml diff --git a/.gitignore b/.gitignore index b832405..ee2f435 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,13 @@ hs_err_pid* replay_pid* +# Build output +target/ + +# IDE +.idea/ +*.iml + # Misc CLUADE.md .claude/ diff --git a/agent-eval-feature-spec.md b/agent-eval-feature-spec.md new file mode 100644 index 0000000..e9dc2ca --- /dev/null +++ b/agent-eval-feature-spec.md @@ -0,0 +1,761 @@ +# Java AI Agent Evaluation & Testing Library — Feature Specification + +> **Working Name:** TBD (candidates: `agenteval`, `agentest`, `jeval`, `evalkit`) +> **Language:** Java 21+ (LTS baseline) +> **License:** Apache 2.0 +> **Build:** Maven Central artifact, Gradle/Maven compatible +> **Philosophy:** Library, not framework. JUnit-native. Local-first. Framework-agnostic. + +--- + +## 1. Core Test Case Model + +The foundational data model that captures everything needed to evaluate an AI interaction. + +### 1.1 `AgentTestCase` — Single-Turn Evaluation + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `input` | `String` | Yes | The user query or prompt sent to the agent | +| `actualOutput` | `String` | Yes | The agent's actual response | +| `expectedOutput` | `String` | No | Ground truth / ideal response for comparison | +| `retrievalContext` | `List` | No | Documents/chunks retrieved by RAG pipeline | +| `context` | `List` | No | Ground truth context (what should have been retrieved) | +| `toolCalls` | `List` | No | Tools invoked by the agent during execution | +| `expectedToolCalls` | `List` | No | Expected tool invocations for comparison | +| `reasoningTrace` | `List` | No | Agent's chain-of-thought / planning steps | +| `latencyMs` | `long` | No | End-to-end execution time | +| `tokenUsage` | `TokenUsage` | No | Input/output/total token counts | +| `cost` | `BigDecimal` | No | Estimated cost of the interaction | +| `metadata` | `Map` | No | Arbitrary key-value pairs for filtering/grouping | + +### 1.2 `ConversationTestCase` — Multi-Turn Evaluation + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `turns` | `List` | Yes | Ordered list of conversation turns | +| `conversationId` | `String` | No | Identifier for the conversation session | +| `systemPrompt` | `String` | No | System prompt used across all turns | + +### 1.3 Supporting Types + +- **`ToolCall`** — `name: String`, `arguments: Map`, `result: String`, `durationMs: long` +- **`ReasoningStep`** — `type: enum(PLAN, THOUGHT, OBSERVATION, ACTION)`, `content: String`, `toolCall: ToolCall?` +- **`TokenUsage`** — `inputTokens: int`, `outputTokens: int`, `totalTokens: int` +- **`EvalScore`** — `value: double` (0.0–1.0), `threshold: double`, `passed: boolean`, `reason: String`, `metricName: String` + +### 1.4 Builder Pattern + +All test case types use immutable builders: + +```java +var testCase = AgentTestCase.builder() + .input("What is our refund policy?") + .actualOutput(agent.run("What is our refund policy?")) + .expectedOutput("Full refund within 30 days of purchase.") + .retrievalContext(List.of(doc1, doc2)) + .build(); +``` + +--- + +## 2. Metrics — Response Quality + +Metrics that evaluate the quality of the agent's text output. All metrics implement the `EvalMetric` interface and return an `EvalScore` (0.0–1.0). + +### 2.1 Answer Relevancy + +- Measures whether the output is relevant to the input question +- Uses LLM-as-judge to generate synthetic questions from the output, then measures similarity to the original input +- Penalizes off-topic content and irrelevant information +- **Config:** threshold (default 0.7), strictMode (boolean) + +### 2.2 Faithfulness + +- Measures whether claims in the output are supported by the retrieval context +- Extracts individual claims from the output, then verifies each against the provided context +- Core metric for RAG pipelines — catches hallucinated facts that aren't grounded in source documents +- **Config:** threshold (default 0.7) + +### 2.3 Hallucination Detection + +- Measures whether the output contains fabricated information relative to provided context +- Different from faithfulness: specifically targets invented entities, false statistics, non-existent citations +- Can operate with or without ground truth context +- **Config:** threshold (default 0.5), contextRequired (boolean) + +### 2.4 Correctness (G-Eval) + +- General-purpose metric using the G-Eval framework +- Takes custom evaluation criteria as natural language instructions +- Compares actual output against expected output using LLM-as-judge with chain-of-thought +- The most flexible metric — can be configured for any evaluation dimension +- **Config:** criteria (String), evaluationSteps (List), threshold (default 0.5) + +### 2.5 Semantic Similarity + +- Measures embedding-based cosine similarity between actual and expected output +- Does not require LLM-as-judge (uses embedding model only) +- Fast and deterministic — good for regression testing +- **Config:** threshold (default 0.7), embeddingModel (configurable) + +### 2.6 Toxicity + +- Detects harmful, offensive, or inappropriate content in the output +- Covers categories: hate speech, threats, sexual content, self-harm, profanity +- Uses LLM-as-judge with specialized safety rubric +- **Config:** threshold (default 0.5), categories (Set) + +### 2.7 Bias Detection + +- Detects biased content across dimensions: gender, race, religion, political, socioeconomic +- Evaluates whether the output unfairly favors or discriminates against any group +- **Config:** threshold (default 0.5), dimensions (Set) + +### 2.8 Conciseness + +- Measures whether the output is appropriately concise without losing essential information +- Penalizes verbosity, repetition, and filler content +- **Config:** threshold (default 0.5) + +### 2.9 Coherence + +- Measures logical flow, consistency, and readability of the output +- Checks for contradictions within the response +- **Config:** threshold (default 0.7) + +--- + +## 3. Metrics — RAG-Specific + +Metrics specifically designed for evaluating Retrieval-Augmented Generation pipelines. + +### 3.1 Contextual Precision + +- Measures whether the retrieved documents that are actually relevant are ranked higher than irrelevant ones +- Requires ground truth expected output to determine relevance +- Higher score = relevant documents appear earlier in the retrieval results +- **Config:** threshold (default 0.7) + +### 3.2 Contextual Recall + +- Measures whether all relevant information needed to produce the expected output was actually retrieved +- Aligns sentences in the expected output to sentences in the retrieval context +- Low score = the retrieval pipeline missed important source documents +- **Config:** threshold (default 0.7) + +### 3.3 Contextual Relevancy + +- Measures what proportion of the retrieved context is actually relevant to the input +- Penalizes retrieval of irrelevant/noisy documents that dilute useful context +- **Config:** threshold (default 0.7) + +### 3.4 Retrieval Completeness + +- Checks whether all ground truth context documents were retrieved +- Set-based comparison: were the right documents fetched? +- Supports both exact match and fuzzy/semantic matching modes +- **Config:** threshold (default 0.8), matchMode (EXACT | SEMANTIC) + +--- + +## 4. Metrics — Agent-Specific + +Metrics designed for evaluating autonomous agent behavior, including tool use and multi-step reasoning. + +### 4.1 Tool Selection Accuracy + +- Measures whether the agent selected the correct tools for the task +- Compares actual tool calls against expected tool calls (by name) +- Handles cases where tool order matters and where it doesn't +- **Config:** threshold (default 0.8), orderMatters (boolean, default false) + +### 4.2 Tool Argument Correctness + +- Measures whether the arguments passed to tools were correct +- Supports type-safe generic assertions: `assertToolArg(SearchTool.class, args -> args.query().contains("refund"))` +- Deep comparison of argument maps against expected values +- **Config:** threshold (default 0.8), strictMode (boolean — fail on extra args) + +### 4.3 Tool Result Utilization + +- Measures whether the agent actually used the results returned by tool calls in its final output +- Detects cases where the agent calls a tool but ignores its response +- **Config:** threshold (default 0.7) + +### 4.4 Plan Quality + +- Evaluates whether the agent's generated plan is logical, complete, and efficient +- Checks: does the plan address all aspects of the task? Are steps in a sensible order? Are there redundant steps? +- Requires reasoning trace capture from the agent +- **Config:** threshold (default 0.7) + +### 4.5 Plan Adherence + +- Evaluates whether the agent followed its own plan during execution +- Compares the planned steps against the actual execution trace +- Detects deviations, skipped steps, and unplanned actions +- **Config:** threshold (default 0.7) + +### 4.6 Task Completion + +- Binary + graded evaluation of whether the agent accomplished the stated goal +- LLM-as-judge determines if the final outcome satisfies the original task +- Can incorporate custom success criteria +- **Config:** threshold (default 0.5), successCriteria (String, optional natural language) + +### 4.7 Trajectory Optimality + +- Measures whether the agent took an efficient path to the solution +- Compares the number and type of steps taken against a reference optimal trajectory +- Penalizes unnecessary tool calls, redundant LLM invocations, and circular reasoning +- **Config:** threshold (default 0.5), maxSteps (int, optional) + +### 4.8 Step-Level Error Localization + +- Identifies which specific step in the agent's execution chain caused a failure +- Evaluates each reasoning step and tool call individually, flagging the first point of divergence +- Produces a diagnostic report pointing to the root cause +- **Config:** threshold (default 0.5) + +--- + +## 5. Metrics — Conversation-Specific + +Metrics for evaluating multi-turn agent interactions. + +### 5.1 Conversation Coherence + +- Measures whether responses maintain logical consistency across turns +- Detects self-contradictions between earlier and later responses +- **Config:** threshold (default 0.7) + +### 5.2 Context Retention + +- Measures whether the agent remembers and correctly uses information from earlier turns +- Tests: does the agent recall user preferences, prior answers, and established facts? +- **Config:** threshold (default 0.7) + +### 5.3 Topic Drift Detection + +- Measures whether the agent stays on topic across the conversation +- Detects when responses diverge from the user's original intent +- **Config:** threshold (default 0.5) + +### 5.4 Conversation Resolution + +- Evaluates whether the multi-turn conversation reached a satisfactory conclusion +- Determines if the user's original goal was ultimately accomplished +- **Config:** threshold (default 0.5), successCriteria (String) + +--- + +## 6. Custom Metrics + +Infrastructure for users to define their own evaluation metrics. + +### 6.1 G-Eval Custom Metric Builder + +- Define arbitrary evaluation criteria in natural language +- The library generates chain-of-thought evaluation prompts automatically +- Supports specifying which test case fields to include in evaluation + +```java +var customMetric = GEval.builder() + .name("TechnicalAccuracy") + .criteria("Evaluate whether the response contains technically accurate Java code examples") + .evaluationSteps(List.of( + "Check if code snippets compile", + "Verify API usage is correct for the stated Java version", + "Check for deprecated method usage" + )) + .threshold(0.8) + .build(); +``` + +### 6.2 Deterministic Custom Metrics + +- Implement the `EvalMetric` interface for rule-based checks +- No LLM calls required — pure Java logic +- Useful for: regex matching, schema validation, JSON structure checks, keyword presence + +```java +public class ContainsDisclaimerMetric implements EvalMetric { + @Override + public EvalScore evaluate(AgentTestCase testCase) { + boolean hasDisclaimer = testCase.getActualOutput() + .contains("not financial advice"); + return EvalScore.of(hasDisclaimer ? 1.0 : 0.0, 0.5, "Disclaimer check"); + } +} +``` + +### 6.3 Composite Metrics + +- Combine multiple metrics with weighted averaging +- Define pass/fail logic: ALL must pass, ANY must pass, or WEIGHTED average + +```java +var composite = CompositeMetric.builder() + .name("OverallQuality") + .add(new AnswerRelevancy(), 0.4) + .add(new Faithfulness(), 0.4) + .add(new Conciseness(), 0.2) + .strategy(CompositeStrategy.WEIGHTED_AVERAGE) + .threshold(0.7) + .build(); +``` + +--- + +## 7. LLM-as-Judge Engine + +The evaluation backbone — manages LLM calls used by metrics to score test cases. + +### 7.1 Provider Support + +- **OpenAI** — GPT-4o, GPT-4o-mini, GPT-4.1, etc. +- **Anthropic** — Claude Sonnet, Claude Haiku +- **Google** — Gemini Flash, Gemini Pro +- **Ollama** — Any locally hosted model (llama3, mistral, etc.) +- **Azure OpenAI** — Enterprise endpoint support +- **Amazon Bedrock** — AWS-native model access +- **Custom** — Implement `JudgeModel` interface for any HTTP-compatible LLM + +### 7.2 Configuration + +```java +AgentEval.configure() + .judgeModel(JudgeModels.anthropic("claude-sonnet-4-20250514")) + .embeddingModel(EmbeddingModels.openai("text-embedding-3-small")) + .maxConcurrentJudgeCalls(4) + .retryOnRateLimit(true, maxRetries: 3) + .cachJudgeResults(true) // avoid re-evaluating identical test cases + .build(); +``` + +### 7.3 Cost Management + +- Token usage tracking per metric evaluation +- Estimated cost reporting per test run +- Budget limits — abort eval run if cost exceeds threshold +- Judge result caching to avoid redundant LLM calls across runs + +### 7.4 Judge Prompt Templates + +- Research-backed prompt templates for each metric (G-Eval, etc.) +- Templates are open and customizable — override any metric's judge prompt +- Chain-of-thought prompting with structured output parsing +- All templates available as resources for inspection and modification + +--- + +## 8. JUnit 5 Integration + +First-class integration with JUnit 5, the standard Java testing framework. + +### 8.1 Extension + +- `@ExtendWith(AgentEvalExtension.class)` on test classes +- Automatically collects results, manages lifecycle, generates reports +- Integrates with JUnit's test lifecycle hooks (beforeAll, afterAll, etc.) + +### 8.2 Annotations + +| Annotation | Target | Description | +|-----------|--------|-------------| +| `@AgentTest` | Method | Marks a test method as an agent evaluation | +| `@Metric` | Method | Applies a metric with configurable threshold | +| `@Metrics` | Method | Container for multiple `@Metric` annotations | +| `@DatasetSource` | Method | Loads test cases from a file (JSON/CSV) | +| `@GoldenSet` | Parameter | Injects golden dataset into parameterized test | +| `@JudgeModel` | Class/Method | Overrides the judge LLM for specific tests | +| `@EvalTimeout` | Method | Sets max time for evaluation to complete | +| `@Tag("eval")` | Class/Method | Standard JUnit tag for selective execution | + +### 8.3 Assertion API + +Fluent assertion API compatible with AssertJ style: + +```java +AgentAssertions.assertThat(testCase) + .meetsMetric(new AnswerRelevancy(0.7)) + .meetsMetric(new Faithfulness(0.8)) + .hasToolCalls() + .toolCallCount(2) + .calledTool("SearchOrders") + .neverCalledTool("DeleteOrder") + .outputContains("refund") + .outputMatchesSchema(RefundResponse.class); +``` + +### 8.4 Parameterized Dataset Tests + +```java +@ParameterizedTest +@DatasetSource("src/test/resources/golden-set.json") +@Metric(value = AnswerRelevancy.class, threshold = 0.7) +@Metric(value = Faithfulness.class, threshold = 0.8) +void evaluateAcrossDataset(AgentTestCase testCase) { + var response = agent.run(testCase.getInput()); + testCase.setActualOutput(response); +} +``` + +### 8.5 Selective Execution + +- Run only eval tests: `mvn test -Dgroups=eval` +- Run only fast (deterministic) metrics: `mvn test -Dgroups=eval-fast` +- Skip eval tests in quick builds: `mvn test -DexcludeGroups=eval` + +--- + +## 9. Standalone Evaluation Runner + +For batch evaluation outside of JUnit (scripting, notebooks, CI scripts). + +### 9.1 Programmatic API + +```java +var results = AgentEval.evaluate( + dataset, + List.of( + new AnswerRelevancy(0.7), + new Faithfulness(0.8), + new ToolSelectionAccuracy(0.9) + ) +); + +results.summary(); // Console summary +results.averageScore(); // Overall average +results.passRate(); // Percentage of test cases that passed all metrics +results.failedCases(); // Stream of failed test cases with reasons +``` + +### 9.2 Parallel Execution + +- Evaluates multiple test cases concurrently using virtual threads +- Configurable concurrency limit (default: number of available processors) +- Thread-safe metric implementations + +### 9.3 Progress Reporting + +- Real-time console progress bar for long-running evaluations +- Callback interface for custom progress handling +- Estimated time remaining based on throughput + +--- + +## 10. Dataset Management + +Infrastructure for managing evaluation datasets. + +### 10.1 Format Support + +| Format | Read | Write | Notes | +|--------|------|-------|-------| +| JSON | Yes | Yes | Primary format, supports full test case model | +| CSV | Yes | Yes | Flat structure, good for simple input/output pairs | +| JSONL | Yes | Yes | Streaming-friendly, one test case per line | +| YAML | Yes | No | Human-readable alternative to JSON | + +### 10.2 Golden Set Management + +- Load from files, classpath resources, or URLs +- Filter and slice datasets by metadata tags +- Split into train/test for prompt optimization workflows +- Version tracking — associate datasets with commit hashes or release tags + +### 10.3 Dataset Builder + +```java +var dataset = EvalDataset.builder() + .name("refund-queries-v2") + .addCase(AgentTestCase.builder() + .input("How do I get a refund?") + .expectedOutput("You can request a refund within 30 days...") + .metadata(Map.of("category", "refund", "difficulty", "easy")) + .build()) + .addCase(...) + .build(); + +dataset.save("src/test/resources/refund-queries-v2.json"); +``` + +### 10.4 Synthetic Dataset Generation (P2) + +- Generate test cases from existing documents using LLM +- Produce variations of existing golden set entries (paraphrasing, edge cases) +- Generate adversarial inputs designed to expose weaknesses + +--- + +## 11. Reporting & Output + +### 11.1 Console Report + +- Summary table: metric name, average score, pass rate, min/max +- Failed test case details with LLM-judge explanations +- Color-coded output (pass=green, fail=red, warning=yellow) +- Execution time and cost summary + +### 11.2 JUnit XML Report + +- Standard JUnit XML format compatible with all CI systems +- Each metric evaluation maps to a test case in the report +- Jenkins, GitHub Actions, GitLab CI all render these natively + +### 11.3 JSON Report + +- Machine-readable full results export +- Contains all scores, explanations, metadata, and configuration +- Useful for custom dashboards or historical tracking + +### 11.4 HTML Report (P2) + +- Self-contained single-file HTML report +- Metric scorecards with distribution charts +- Drill-down into individual test cases +- Side-by-side comparison of actual vs expected output + +### 11.5 Regression Comparison (P2) + +- Compare two evaluation runs side-by-side +- Identify metrics that improved or degraded +- Highlight specific test cases that changed pass/fail status +- Useful for validating prompt changes or model swaps + +--- + +## 12. Framework Integrations + +Optional modules that auto-capture agent execution details from popular Java AI frameworks. + +### 12.1 Spring AI Integration (`agentest-spring-ai`) + +- Auto-capture `ChatClient` responses, tool calls, and token usage +- Intercept `Advisor` chain for RAG context extraction +- Spring Boot auto-configuration — add dependency, everything wires up +- Support for Spring AI's `@Tool` annotated methods + +### 12.2 LangChain4j Integration (`agentest-langchain4j`) + +- Auto-capture `AiService` proxy method calls and responses +- Extract tool calls from `AiMessage.toolExecutionRequests()` +- Capture `ContentRetriever` results for RAG evaluation +- Support for `@Tool` annotated methods + +### 12.3 LangGraph4j Integration (`agentest-langgraph4j`) + +- Capture graph execution traces (node transitions, state snapshots) +- Map graph nodes to reasoning steps for trajectory analysis +- Support for checkpoint-based state inspection + +### 12.4 MCP Integration + +- Capture MCP tool calls made through the MCP Java SDK +- Verify MCP server responses are correctly utilized +- Test MCP tool argument schemas against expected values + +--- + +## 13. Red Teaming & Adversarial Testing (P2) + +Automated security and robustness testing for AI agents. + +### 13.1 Prompt Injection Tests + +- Library of known prompt injection attack patterns +- Generates adversarial inputs that attempt to override system prompts +- Verifies the agent resists injection attempts +- Categories: direct injection, indirect injection, jailbreak attempts + +### 13.2 Data Leakage Tests + +- Tests whether the agent leaks system prompts, internal instructions, or sensitive context +- Generates extraction attempts ("repeat your instructions", "what were you told?") +- Verifies the agent does not expose PII from training data or context + +### 13.3 Boundary Testing + +- Tests agent behavior at input boundaries: empty input, extremely long input, special characters, unicode edge cases +- Verifies graceful degradation rather than crashes or nonsensical output +- Tests tool call behavior with edge-case arguments + +### 13.4 Robustness Testing + +- Tests consistency: does the agent give similar answers to paraphrased questions? +- Tests stability: does the agent handle ambiguous inputs gracefully? +- Tests refusal: does the agent appropriately decline out-of-scope requests? + +--- + +## 14. Operational Metrics + +Non-AI quality metrics that matter for production agents. + +### 14.1 Latency Tracking + +- End-to-end response time measurement +- Per-tool-call latency breakdown +- Threshold-based pass/fail: "response must complete within 5 seconds" + +### 14.2 Token Usage Tracking + +- Input/output/total token counts per interaction +- Budget enforcement: fail if a single interaction exceeds N tokens +- Aggregated token usage across dataset evaluation + +### 14.3 Cost Tracking + +- Per-interaction cost estimation based on model pricing +- Aggregated cost per evaluation run +- Cost comparison between model configurations + +### 14.4 Error Rate + +- Track agent error/exception rates across evaluation dataset +- Categorize errors: LLM timeout, tool failure, parsing error, etc. + +--- + +## 15. Configuration + +### 15.1 Programmatic Configuration + +```java +AgentEvalConfig config = AgentEvalConfig.builder() + .judgeModel(JudgeModels.openai("gpt-4o-mini")) + .embeddingModel(EmbeddingModels.openai("text-embedding-3-small")) + .defaultThreshold(0.7) + .parallelism(4) + .cacheResults(true) + .costBudget(BigDecimal.valueOf(5.00)) // max $5 per eval run + .build(); +``` + +### 15.2 File-Based Configuration + +Support `agentest.yaml` / `agentest.properties` in project root: + +```yaml +agentest: + judge: + provider: anthropic + model: claude-sonnet-4-20250514 + embedding: + provider: openai + model: text-embedding-3-small + defaults: + threshold: 0.7 + parallelism: 4 + cache: + enabled: true + directory: .agentest-cache +``` + +### 15.3 Environment Variable Overrides + +- `AGENTEST_JUDGE_PROVIDER` / `AGENTEST_JUDGE_MODEL` +- `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` — standard env vars +- CI-friendly: no config files needed if env vars are set + +--- + +## 16. Module Structure + +``` +agentest-bom/ — Bill of Materials for dependency management +agentest-core/ — Test case model, metric interfaces, scoring engine +agentest-metrics/ — All built-in metric implementations +agentest-judge/ — LLM-as-judge engine, provider integrations +agentest-embeddings/ — Embedding model integrations (for semantic metrics) +agentest-junit5/ — JUnit 5 extension, annotations, assertion API +agentest-datasets/ — Dataset loading, management, generation +agentest-reporting/ — Console, JUnit XML, JSON, HTML report generation +agentest-spring-ai/ — Spring AI auto-capture integration +agentest-langchain4j/ — LangChain4j auto-capture integration +agentest-langgraph4j/ — LangGraph4j graph execution capture +agentest-mcp/ — MCP Java SDK tool call capture +agentest-redteam/ — Adversarial testing and prompt injection library +``` + +--- + +## 17. Priority Tiers + +### P0 — MVP (First Release) + +- Core test case model (`AgentTestCase`, `ToolCall`, `EvalScore`) +- 5 response quality metrics: AnswerRelevancy, Faithfulness, Correctness (G-Eval), Hallucination, Toxicity +- 2 agent metrics: ToolSelectionAccuracy, TaskCompletion +- LLM-as-judge engine with OpenAI + Anthropic support +- JUnit 5 extension with `@AgentTest`, `@Metric` annotations +- Fluent assertion API +- JSON dataset loading +- Console + JUnit XML reporting +- Programmatic configuration + +### P1 — Fast Follow + +- Remaining response quality metrics: Bias, Conciseness, Coherence, SemanticSimilarity +- All RAG metrics: ContextualPrecision, ContextualRecall, ContextualRelevancy +- Remaining agent metrics: ToolArgumentCorrectness, PlanQuality, PlanAdherence, TrajectoryOptimality +- Conversation metrics: Coherence, ContextRetention +- Ollama / local model support for judge +- Spring AI integration module +- LangChain4j integration module +- CSV/JSONL dataset support +- JSON report output +- Cost tracking and budget limits +- File-based configuration (YAML) + +### P2 — Growth + +- LangGraph4j integration +- MCP integration +- HTML report generation +- Regression comparison (diff two runs) +- Synthetic dataset generation +- Red teaming / adversarial testing module +- Custom embedding model support +- Conversation metrics: TopicDrift, Resolution +- Step-level error localization +- Tool result utilization metric +- Parallel evaluation with virtual threads +- Progress bar and ETA for batch runs + +### P3 — Ecosystem + +- Maven plugin for `mvn agentest:evaluate` +- Gradle plugin +- GitHub Actions integration (post results as PR comments) +- Golden set versioning with Git integration +- Benchmark mode (compare multiple models/prompts across same dataset) +- Multi-model judge (evaluate with multiple judges, take consensus) +- Snapshot testing (save and compare outputs across releases) +- IntelliJ IDEA plugin (run eval tests with inline metric results) + +--- + +## 18. Non-Goals + +Things this library explicitly does **not** aim to do: + +- **Not an observability platform.** No dashboards, no production monitoring, no trace storage. Use Langfuse, LangSmith, or OpenTelemetry for that. +- **Not a cloud service.** No accounts, no SaaS, no data leaves your machine. Optional future: export to external platforms. +- **Not a framework for building agents.** Use Spring AI, LangChain4j, or LangGraph4j for that. This library tests what you've already built. +- **Not a benchmark suite.** Spring AI Bench benchmarks coding agents on Spring tasks. This library tests your custom agents against your custom criteria. +- **Not a training/fine-tuning tool.** This evaluates, it doesn't train. Evaluation results can inform fine-tuning decisions, but that's a different tool. + +--- + +## 19. Technical Constraints + +- **Java 21+ baseline** — leverages records, sealed interfaces, pattern matching, virtual threads +- **Zero required runtime dependencies** on Spring or LangChain4j — core is standalone +- **JUnit 5.10+** for extension API +- **Jackson** for JSON serialization (aligned with MCP Java SDK choice) +- **SLF4J** for logging (no specific implementation required) +- **Java HTTP Client** (built-in `java.net.http`) for LLM API calls — no OkHttp/Apache HttpClient dependency +- **No reflection-heavy magic** — prefer explicit configuration over classpath scanning diff --git a/agenteval-core/pom.xml b/agenteval-core/pom.xml new file mode 100644 index 0000000..8aee9d2 --- /dev/null +++ b/agenteval-core/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + + com.agenteval + agenteval-parent + 0.1.0-SNAPSHOT + + + agenteval-core + AgentEval Core + Core data model, metric interfaces, and scoring engine + + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + diff --git a/agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfig.java b/agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfig.java new file mode 100644 index 0000000..e6ce2dd --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfig.java @@ -0,0 +1,87 @@ +package com.agenteval.core.config; + +import com.agenteval.core.embedding.EmbeddingModel; +import com.agenteval.core.judge.JudgeModel; + +/** + * Configuration for the AgentEval evaluation engine. + * + * 
{@code
+ * var config = AgentEvalConfig.builder()
+ *     .judgeModel(myJudgeModel)
+ *     .maxConcurrentJudgeCalls(4)
+ *     .retryOnRateLimit(true)
+ *     .maxRetries(3)
+ *     .cacheJudgeResults(true)
+ *     .build();
+ * }
+ */ +public final class AgentEvalConfig { + + private final JudgeModel judgeModel; + private final EmbeddingModel embeddingModel; + private final int maxConcurrentJudgeCalls; + private final boolean retryOnRateLimit; + private final int maxRetries; + private final boolean cacheJudgeResults; + + private AgentEvalConfig(Builder builder) { + this.judgeModel = builder.judgeModel; + this.embeddingModel = builder.embeddingModel; + this.maxConcurrentJudgeCalls = builder.maxConcurrentJudgeCalls; + this.retryOnRateLimit = builder.retryOnRateLimit; + this.maxRetries = builder.maxRetries; + this.cacheJudgeResults = builder.cacheJudgeResults; + } + + public JudgeModel judgeModel() { return judgeModel; } + public EmbeddingModel embeddingModel() { return embeddingModel; } + public int maxConcurrentJudgeCalls() { return maxConcurrentJudgeCalls; } + public boolean retryOnRateLimit() { return retryOnRateLimit; } + public int maxRetries() { return maxRetries; } + public boolean cacheJudgeResults() { return cacheJudgeResults; } + + public static Builder builder() { + return new Builder(); + } + + /** + * Returns a default configuration (no judge or embedding model configured). + */ + public static AgentEvalConfig defaults() { + return new Builder().build(); + } + + public static final class Builder { + private JudgeModel judgeModel; + private EmbeddingModel embeddingModel; + private int maxConcurrentJudgeCalls = Runtime.getRuntime().availableProcessors(); + private boolean retryOnRateLimit = true; + private int maxRetries = 3; + private boolean cacheJudgeResults = false; + + private Builder() {} + + public Builder judgeModel(JudgeModel judgeModel) { this.judgeModel = judgeModel; return this; } + public Builder embeddingModel(EmbeddingModel embeddingModel) { + this.embeddingModel = embeddingModel; + return this; + } + public Builder maxConcurrentJudgeCalls(int max) { + if (max < 1) throw new IllegalArgumentException("maxConcurrentJudgeCalls must be >= 1"); + this.maxConcurrentJudgeCalls = max; + return this; + } + public Builder retryOnRateLimit(boolean retry) { this.retryOnRateLimit = retry; return this; } + public Builder maxRetries(int maxRetries) { + if (maxRetries < 0) throw new IllegalArgumentException("maxRetries must be >= 0"); + this.maxRetries = maxRetries; + return this; + } + public Builder cacheJudgeResults(boolean cache) { this.cacheJudgeResults = cache; return this; } + + public AgentEvalConfig build() { + return new AgentEvalConfig(this); + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/embedding/EmbeddingModel.java b/agenteval-core/src/main/java/com/agenteval/core/embedding/EmbeddingModel.java new file mode 100644 index 0000000..3834e37 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/embedding/EmbeddingModel.java @@ -0,0 +1,25 @@ +package com.agenteval.core.embedding; + +import java.util.List; + +/** + * SPI interface for embedding model providers. + * + *

Implementations are provided by {@code agenteval-embeddings} module. + * Used by metrics like Semantic Similarity that require vector embeddings.

+ */ +public interface EmbeddingModel { + + /** + * Generates an embedding vector for the given text. + * + * @param text the input text to embed + * @return the embedding vector as a list of doubles + */ + List embed(String text); + + /** + * Returns the model identifier (e.g., "text-embedding-3-small"). + */ + String modelId(); +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/eval/AgentEval.java b/agenteval-core/src/main/java/com/agenteval/core/eval/AgentEval.java new file mode 100644 index 0000000..7014767 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/eval/AgentEval.java @@ -0,0 +1,91 @@ +package com.agenteval.core.eval; + +import com.agenteval.core.config.AgentEvalConfig; +import com.agenteval.core.metric.EvalMetric; +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Static entry point for running evaluations. + * + * 
{@code
+ * var results = AgentEval.evaluate(
+ *     testCases,
+ *     List.of(new AnswerRelevancy(0.7), new Faithfulness(0.8))
+ * );
+ * results.summary();
+ * }
+ */ +public final class AgentEval { + + private static final Logger LOG = LoggerFactory.getLogger(AgentEval.class); + + private AgentEval() {} + + /** + * Evaluates a list of test cases against the given metrics using default configuration. + */ + public static EvalResult evaluate(List testCases, List metrics) { + return evaluate(testCases, metrics, AgentEvalConfig.defaults()); + } + + /** + * Evaluates a list of test cases against the given metrics with the provided configuration. + */ + public static EvalResult evaluate(List testCases, List metrics, + AgentEvalConfig config) { + Objects.requireNonNull(testCases, "testCases must not be null"); + Objects.requireNonNull(metrics, "metrics must not be null"); + Objects.requireNonNull(config, "config must not be null"); + + LOG.info("Starting evaluation: {} test cases, {} metrics", testCases.size(), metrics.size()); + long startTime = System.currentTimeMillis(); + + List caseResults = testCases.stream() + .map(tc -> evaluateCase(tc, metrics)) + .toList(); + + long durationMs = System.currentTimeMillis() - startTime; + LOG.info("Evaluation complete in {}ms", durationMs); + + return EvalResult.of(caseResults, durationMs); + } + + private static CaseResult evaluateCase(AgentTestCase testCase, List metrics) { + Map scores = new LinkedHashMap<>(); + boolean allPassed = true; + + for (EvalMetric metric : metrics) { + try { + EvalScore score = metric.evaluate(testCase); + score = score.withMetricName(metric.name()); + scores.put(metric.name(), score); + if (!score.passed()) { + allPassed = false; + } + } catch (Exception e) { + LOG.error("Metric '{}' failed for test case: {}", metric.name(), e.getMessage(), e); + EvalScore errorScore = EvalScore.fail("Metric evaluation error: " + e.getMessage()) + .withMetricName(metric.name()); + scores.put(metric.name(), errorScore); + allPassed = false; + } + } + + return new CaseResult(testCase, scores, allPassed); + } + + /** + * Creates a new configuration builder. + */ + public static AgentEvalConfig.Builder configure() { + return AgentEvalConfig.builder(); + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/eval/CaseResult.java b/agenteval-core/src/main/java/com/agenteval/core/eval/CaseResult.java new file mode 100644 index 0000000..19989ae --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/eval/CaseResult.java @@ -0,0 +1,42 @@ +package com.agenteval.core.eval; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Evaluation result for a single test case across all metrics. + */ +public record CaseResult( + AgentTestCase testCase, + Map scores, + boolean passed +) { + public CaseResult { + Objects.requireNonNull(testCase, "testCase must not be null"); + scores = scores == null ? Map.of() : Map.copyOf(scores); + } + + /** + * Returns the list of scores that failed their thresholds. + */ + public List failedScores() { + return scores.values().stream() + .filter(s -> !s.passed()) + .toList(); + } + + /** + * Returns the average score across all metrics for this case. + */ + public double averageScore() { + if (scores.isEmpty()) return 0.0; + return scores.values().stream() + .mapToDouble(EvalScore::value) + .average() + .orElse(0.0); + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/eval/EvalResult.java b/agenteval-core/src/main/java/com/agenteval/core/eval/EvalResult.java new file mode 100644 index 0000000..f12e304 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/eval/EvalResult.java @@ -0,0 +1,88 @@ +package com.agenteval.core.eval; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Aggregated evaluation result across all test cases and metrics. + */ +public final class EvalResult { + + private final List caseResults; + private final long durationMs; + + private EvalResult(List caseResults, long durationMs) { + Objects.requireNonNull(caseResults, "caseResults must not be null"); + this.caseResults = List.copyOf(caseResults); + this.durationMs = durationMs; + } + + public static EvalResult of(List caseResults, long durationMs) { + return new EvalResult(caseResults, durationMs); + } + + public List caseResults() { return caseResults; } + public long durationMs() { return durationMs; } + + /** + * Returns the overall average score across all cases and metrics. + */ + public double averageScore() { + return caseResults.stream() + .mapToDouble(CaseResult::averageScore) + .average() + .orElse(0.0); + } + + /** + * Returns the pass rate as a percentage (0.0–1.0). + */ + public double passRate() { + if (caseResults.isEmpty()) return 0.0; + long passed = caseResults.stream().filter(CaseResult::passed).count(); + return (double) passed / caseResults.size(); + } + + /** + * Returns the test cases that failed at least one metric. + */ + public List failedCases() { + return caseResults.stream() + .filter(cr -> !cr.passed()) + .toList(); + } + + /** + * Returns average scores per metric across all cases. + */ + public Map averageScoresByMetric() { + return caseResults.stream() + .flatMap(cr -> cr.scores().entrySet().stream()) + .collect(Collectors.groupingBy( + Map.Entry::getKey, + Collectors.averagingDouble(e -> e.getValue().value()) + )); + } + + /** + * Prints a summary to stdout. + */ + public void summary() { + System.out.println("=== AgentEval Results ==="); + System.out.printf("Cases: %d | Passed: %d | Failed: %d | Pass Rate: %.1f%%%n", + caseResults.size(), + caseResults.size() - failedCases().size(), + failedCases().size(), + passRate() * 100); + System.out.printf("Average Score: %.3f | Duration: %dms%n", averageScore(), durationMs); + + var byMetric = averageScoresByMetric(); + if (!byMetric.isEmpty()) { + System.out.println("--- Per-Metric Averages ---"); + byMetric.forEach((name, avg) -> + System.out.printf(" %-30s %.3f%n", name, avg)); + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/judge/JudgeModel.java b/agenteval-core/src/main/java/com/agenteval/core/judge/JudgeModel.java new file mode 100644 index 0000000..66f325b --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/judge/JudgeModel.java @@ -0,0 +1,24 @@ +package com.agenteval.core.judge; + +/** + * SPI interface for LLM-as-judge providers. + * + *

Implementations are provided by {@code agenteval-judge} module + * (OpenAI, Anthropic, Ollama, etc.). Users can implement this interface + * for custom HTTP-compatible LLM endpoints.

+ */ +public interface JudgeModel { + + /** + * Sends a prompt to the judge LLM and returns the evaluation response. + * + * @param prompt the evaluation prompt (metric-specific) + * @return the judge's response with score and reasoning + */ + JudgeResponse judge(String prompt); + + /** + * Returns the model identifier (e.g., "gpt-4o", "claude-sonnet-4-20250514"). + */ + String modelId(); +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/judge/JudgeResponse.java b/agenteval-core/src/main/java/com/agenteval/core/judge/JudgeResponse.java new file mode 100644 index 0000000..82edcbb --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/judge/JudgeResponse.java @@ -0,0 +1,21 @@ +package com.agenteval.core.judge; + +import com.agenteval.core.model.TokenUsage; + +/** + * Response from an LLM judge evaluation. + */ +public record JudgeResponse( + double score, + String reason, + TokenUsage tokenUsage +) { + public JudgeResponse { + if (score < 0.0 || score > 1.0) { + throw new IllegalArgumentException("score must be between 0.0 and 1.0, got: " + score); + } + if (reason == null) { + reason = ""; + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/metric/CompositeMetric.java b/agenteval-core/src/main/java/com/agenteval/core/metric/CompositeMetric.java new file mode 100644 index 0000000..b167dad --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/metric/CompositeMetric.java @@ -0,0 +1,139 @@ +package com.agenteval.core.metric; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.StringJoiner; + +/** + * Combines multiple metrics with configurable aggregation strategy. + * + * 
{@code
+ * var composite = CompositeMetric.builder()
+ *     .name("OverallQuality")
+ *     .add(new AnswerRelevancy(), 0.4)
+ *     .add(new Faithfulness(), 0.4)
+ *     .add(new Conciseness(), 0.2)
+ *     .strategy(CompositeStrategy.WEIGHTED_AVERAGE)
+ *     .threshold(0.7)
+ *     .build();
+ * }
+ */ +public final class CompositeMetric implements EvalMetric { + + private final String name; + private final List metrics; + private final CompositeStrategy strategy; + private final double threshold; + + private CompositeMetric(Builder builder) { + this.name = Objects.requireNonNull(builder.name, "name must not be null"); + if (builder.metrics.isEmpty()) { + throw new IllegalArgumentException("at least one metric must be added"); + } + this.metrics = List.copyOf(builder.metrics); + this.strategy = builder.strategy; + this.threshold = builder.threshold; + } + + @Override + public EvalScore evaluate(AgentTestCase testCase) { + List scores = metrics.stream() + .map(wm -> wm.metric().evaluate(testCase)) + .toList(); + + return switch (strategy) { + case WEIGHTED_AVERAGE -> evaluateWeightedAverage(scores); + case ALL_PASS -> evaluateAllPass(scores); + case ANY_PASS -> evaluateAnyPass(scores); + }; + } + + private EvalScore evaluateWeightedAverage(List scores) { + double totalWeight = metrics.stream().mapToDouble(WeightedMetric::weight).sum(); + double weightedSum = 0.0; + for (int i = 0; i < scores.size(); i++) { + weightedSum += scores.get(i).value() * metrics.get(i).weight(); + } + double value = totalWeight > 0 ? weightedSum / totalWeight : 0.0; + String reason = buildReason(scores); + return EvalScore.of(value, threshold, reason).withMetricName(name); + } + + private EvalScore evaluateAllPass(List scores) { + double minValue = scores.stream().mapToDouble(EvalScore::value).min().orElse(0.0); + boolean allPassed = scores.stream().allMatch(EvalScore::passed); + String reason = buildReason(scores); + return new EvalScore(minValue, threshold, allPassed && minValue >= threshold, reason, name); + } + + private EvalScore evaluateAnyPass(List scores) { + double maxValue = scores.stream().mapToDouble(EvalScore::value).max().orElse(0.0); + boolean anyPassed = scores.stream().anyMatch(EvalScore::passed); + String reason = buildReason(scores); + return new EvalScore(maxValue, threshold, anyPassed || maxValue >= threshold, reason, name); + } + + private String buildReason(List scores) { + var joiner = new StringJoiner("; ", "Composite [" + strategy + "]: ", ""); + for (int i = 0; i < scores.size(); i++) { + var wm = metrics.get(i); + var score = scores.get(i); + joiner.add(wm.metric().name() + "=" + String.format("%.2f", score.value())); + } + return joiner.toString(); + } + + @Override + public String name() { + return name; + } + + public List metrics() { return metrics; } + public CompositeStrategy strategy() { return strategy; } + public double threshold() { return threshold; } + + public static Builder builder() { + return new Builder(); + } + + public record WeightedMetric(EvalMetric metric, double weight) { + public WeightedMetric { + Objects.requireNonNull(metric, "metric must not be null"); + if (weight <= 0) throw new IllegalArgumentException("weight must be positive"); + } + } + + public static final class Builder { + private String name; + private final List metrics = new ArrayList<>(); + private CompositeStrategy strategy = CompositeStrategy.WEIGHTED_AVERAGE; + private double threshold = 0.5; + + private Builder() {} + + public Builder name(String name) { this.name = name; return this; } + + public Builder add(EvalMetric metric, double weight) { + metrics.add(new WeightedMetric(metric, weight)); + return this; + } + + public Builder strategy(CompositeStrategy strategy) { + this.strategy = Objects.requireNonNull(strategy); + return this; + } + + public Builder threshold(double threshold) { + this.threshold = threshold; + return this; + } + + public CompositeMetric build() { + return new CompositeMetric(this); + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/metric/CompositeStrategy.java b/agenteval-core/src/main/java/com/agenteval/core/metric/CompositeStrategy.java new file mode 100644 index 0000000..5c6bc10 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/metric/CompositeStrategy.java @@ -0,0 +1,25 @@ +package com.agenteval.core.metric; + +/** + * Strategy for combining multiple metric scores in a {@link CompositeMetric}. + */ +public enum CompositeStrategy { + + /** + * Final score is the weighted average of all metric scores. + * Passes if the weighted average meets the composite threshold. + */ + WEIGHTED_AVERAGE, + + /** + * Passes only if all individual metrics pass their own thresholds. + * Final score is the minimum score across all metrics. + */ + ALL_PASS, + + /** + * Passes if any individual metric passes its threshold. + * Final score is the maximum score across all metrics. + */ + ANY_PASS +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/metric/EvalMetric.java b/agenteval-core/src/main/java/com/agenteval/core/metric/EvalMetric.java new file mode 100644 index 0000000..9c828d3 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/metric/EvalMetric.java @@ -0,0 +1,26 @@ +package com.agenteval.core.metric; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; + +/** + * Interface for all evaluation metrics. + * + *

Implementations evaluate an {@link AgentTestCase} and return an {@link EvalScore} + * normalized to the 0.0–1.0 range. Metrics must be thread-safe.

+ */ +public interface EvalMetric { + + /** + * Evaluates the given test case and returns a score. + * + * @param testCase the test case to evaluate + * @return the evaluation score (0.0–1.0) + */ + EvalScore evaluate(AgentTestCase testCase); + + /** + * Returns the name of this metric. + */ + String name(); +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/model/AgentTestCase.java b/agenteval-core/src/main/java/com/agenteval/core/model/AgentTestCase.java new file mode 100644 index 0000000..7c98013 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/model/AgentTestCase.java @@ -0,0 +1,119 @@ +package com.agenteval.core.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Represents a single-turn agent evaluation test case. + * + *

Most fields are immutable and set at construction time via the builder. + * Capture-time fields ({@code actualOutput}, {@code latencyMs}, {@code tokenUsage}, + * {@code cost}) are mutable to support parameterized dataset tests where the agent + * response is captured after test case construction.

+ */ +@JsonDeserialize(builder = AgentTestCase.Builder.class) +public final class AgentTestCase { + + private final String input; + private volatile String actualOutput; + private final String expectedOutput; + private final List retrievalContext; + private final List context; + private final List toolCalls; + private final List expectedToolCalls; + private final List reasoningTrace; + private volatile long latencyMs; + private volatile TokenUsage tokenUsage; + private volatile BigDecimal cost; + private final Map metadata; + + private AgentTestCase(Builder builder) { + this.input = Objects.requireNonNull(builder.input, "input must not be null"); + this.actualOutput = builder.actualOutput; + this.expectedOutput = builder.expectedOutput; + this.retrievalContext = builder.retrievalContext == null ? List.of() : List.copyOf(builder.retrievalContext); + this.context = builder.context == null ? List.of() : List.copyOf(builder.context); + this.toolCalls = builder.toolCalls == null ? List.of() : List.copyOf(builder.toolCalls); + this.expectedToolCalls = builder.expectedToolCalls == null ? List.of() : List.copyOf(builder.expectedToolCalls); + this.reasoningTrace = builder.reasoningTrace == null ? List.of() : List.copyOf(builder.reasoningTrace); + this.latencyMs = builder.latencyMs; + this.tokenUsage = builder.tokenUsage; + this.cost = builder.cost; + this.metadata = builder.metadata == null ? Map.of() : Map.copyOf(builder.metadata); + } + + public static Builder builder() { + return new Builder(); + } + + // --- Getters --- + + public String getInput() { return input; } + public String getActualOutput() { return actualOutput; } + public String getExpectedOutput() { return expectedOutput; } + public List getRetrievalContext() { return retrievalContext; } + public List getContext() { return context; } + public List getToolCalls() { return toolCalls; } + public List getExpectedToolCalls() { return expectedToolCalls; } + public List getReasoningTrace() { return reasoningTrace; } + public long getLatencyMs() { return latencyMs; } + public TokenUsage getTokenUsage() { return tokenUsage; } + public BigDecimal getCost() { return cost; } + public Map getMetadata() { return metadata; } + + // --- Mutable capture-time setters --- + + public void setActualOutput(String actualOutput) { this.actualOutput = actualOutput; } + public void setLatencyMs(long latencyMs) { this.latencyMs = latencyMs; } + public void setTokenUsage(TokenUsage tokenUsage) { this.tokenUsage = tokenUsage; } + public void setCost(BigDecimal cost) { this.cost = cost; } + + @JsonPOJOBuilder(withPrefix = "") + public static final class Builder { + private String input; + private String actualOutput; + private String expectedOutput; + private List retrievalContext; + private List context; + private List toolCalls; + private List expectedToolCalls; + private List reasoningTrace; + private long latencyMs; + private TokenUsage tokenUsage; + private BigDecimal cost; + private Map metadata; + + private Builder() {} + + public Builder input(String input) { this.input = input; return this; } + public Builder actualOutput(String actualOutput) { this.actualOutput = actualOutput; return this; } + public Builder expectedOutput(String expectedOutput) { this.expectedOutput = expectedOutput; return this; } + public Builder retrievalContext(List retrievalContext) { + this.retrievalContext = retrievalContext; + return this; + } + public Builder context(List context) { this.context = context; return this; } + public Builder toolCalls(List toolCalls) { this.toolCalls = toolCalls; return this; } + public Builder expectedToolCalls(List expectedToolCalls) { + this.expectedToolCalls = expectedToolCalls; + return this; + } + public Builder reasoningTrace(List reasoningTrace) { + this.reasoningTrace = reasoningTrace; + return this; + } + public Builder latencyMs(long latencyMs) { this.latencyMs = latencyMs; return this; } + public Builder tokenUsage(TokenUsage tokenUsage) { this.tokenUsage = tokenUsage; return this; } + public Builder cost(BigDecimal cost) { this.cost = cost; return this; } + public Builder metadata(Map metadata) { this.metadata = metadata; return this; } + + public AgentTestCase build() { + return new AgentTestCase(this); + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/model/ConversationTestCase.java b/agenteval-core/src/main/java/com/agenteval/core/model/ConversationTestCase.java new file mode 100644 index 0000000..93f1616 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/model/ConversationTestCase.java @@ -0,0 +1,53 @@ +package com.agenteval.core.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; + +import java.util.List; +import java.util.Objects; + +/** + * Represents a multi-turn conversation evaluation test case. + */ +@JsonDeserialize(builder = ConversationTestCase.Builder.class) +public final class ConversationTestCase { + + private final List turns; + private final String conversationId; + private final String systemPrompt; + + private ConversationTestCase(Builder builder) { + Objects.requireNonNull(builder.turns, "turns must not be null"); + if (builder.turns.isEmpty()) { + throw new IllegalArgumentException("turns must not be empty"); + } + this.turns = List.copyOf(builder.turns); + this.conversationId = builder.conversationId; + this.systemPrompt = builder.systemPrompt; + } + + public static Builder builder() { + return new Builder(); + } + + public List getTurns() { return turns; } + public String getConversationId() { return conversationId; } + public String getSystemPrompt() { return systemPrompt; } + + @JsonPOJOBuilder(withPrefix = "") + public static final class Builder { + private List turns; + private String conversationId; + private String systemPrompt; + + private Builder() {} + + public Builder turns(List turns) { this.turns = turns; return this; } + public Builder conversationId(String conversationId) { this.conversationId = conversationId; return this; } + public Builder systemPrompt(String systemPrompt) { this.systemPrompt = systemPrompt; return this; } + + public ConversationTestCase build() { + return new ConversationTestCase(this); + } + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/model/EvalScore.java b/agenteval-core/src/main/java/com/agenteval/core/model/EvalScore.java new file mode 100644 index 0000000..a1815e2 --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/model/EvalScore.java @@ -0,0 +1,56 @@ +package com.agenteval.core.model; + +import java.util.Objects; + +/** + * The result of a metric evaluation. Score is normalized to 0.0–1.0. + */ +public record EvalScore( + double value, + double threshold, + boolean passed, + String reason, + String metricName +) { + public EvalScore { + if (value < 0.0 || value > 1.0) { + throw new IllegalArgumentException("value must be between 0.0 and 1.0, got: " + value); + } + if (threshold < 0.0 || threshold > 1.0) { + throw new IllegalArgumentException("threshold must be between 0.0 and 1.0, got: " + threshold); + } + Objects.requireNonNull(reason, "reason must not be null"); + if (metricName == null) { + metricName = ""; + } + } + + /** + * Creates an EvalScore with auto-derived passed flag (value >= threshold). + * metricName defaults to empty — populated by the evaluation engine. + */ + public static EvalScore of(double value, double threshold, String reason) { + return new EvalScore(value, threshold, value >= threshold, reason, ""); + } + + /** + * Creates a passing EvalScore with perfect score. + */ + public static EvalScore pass(String reason) { + return new EvalScore(1.0, 0.0, true, reason, ""); + } + + /** + * Creates a failing EvalScore with zero score. + */ + public static EvalScore fail(String reason) { + return new EvalScore(0.0, 1.0, false, reason, ""); + } + + /** + * Returns a new EvalScore with the specified metric name. + */ + public EvalScore withMetricName(String metricName) { + return new EvalScore(value, threshold, passed, reason, metricName); + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStep.java b/agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStep.java new file mode 100644 index 0000000..fc8050b --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStep.java @@ -0,0 +1,31 @@ +package com.agenteval.core.model; + +import java.util.Objects; + +/** + * A single step in an agent's reasoning trace. + */ +public record ReasoningStep( + ReasoningStepType type, + String content, + ToolCall toolCall +) { + public ReasoningStep { + Objects.requireNonNull(type, "type must not be null"); + Objects.requireNonNull(content, "content must not be null"); + } + + /** + * Creates a reasoning step without an associated tool call. + */ + public static ReasoningStep of(ReasoningStepType type, String content) { + return new ReasoningStep(type, content, null); + } + + /** + * Creates an ACTION reasoning step with an associated tool call. + */ + public static ReasoningStep action(String content, ToolCall toolCall) { + return new ReasoningStep(ReasoningStepType.ACTION, content, toolCall); + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStepType.java b/agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStepType.java new file mode 100644 index 0000000..a0d151e --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/model/ReasoningStepType.java @@ -0,0 +1,11 @@ +package com.agenteval.core.model; + +/** + * Types of reasoning steps an agent can take during execution. + */ +public enum ReasoningStepType { + PLAN, + THOUGHT, + OBSERVATION, + ACTION +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/model/TokenUsage.java b/agenteval-core/src/main/java/com/agenteval/core/model/TokenUsage.java new file mode 100644 index 0000000..213281d --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/model/TokenUsage.java @@ -0,0 +1,20 @@ +package com.agenteval.core.model; + +/** + * Token usage statistics for an LLM interaction. + */ +public record TokenUsage(int inputTokens, int outputTokens, int totalTokens) { + + public TokenUsage { + if (inputTokens < 0) throw new IllegalArgumentException("inputTokens must be non-negative"); + if (outputTokens < 0) throw new IllegalArgumentException("outputTokens must be non-negative"); + if (totalTokens < 0) throw new IllegalArgumentException("totalTokens must be non-negative"); + } + + /** + * Creates a TokenUsage where totalTokens is auto-calculated. + */ + public static TokenUsage of(int inputTokens, int outputTokens) { + return new TokenUsage(inputTokens, outputTokens, inputTokens + outputTokens); + } +} diff --git a/agenteval-core/src/main/java/com/agenteval/core/model/ToolCall.java b/agenteval-core/src/main/java/com/agenteval/core/model/ToolCall.java new file mode 100644 index 0000000..800d59d --- /dev/null +++ b/agenteval-core/src/main/java/com/agenteval/core/model/ToolCall.java @@ -0,0 +1,33 @@ +package com.agenteval.core.model; + +import java.util.Map; +import java.util.Objects; + +/** + * Represents a tool invocation made by an agent. + */ +public record ToolCall( + String name, + Map arguments, + String result, + long durationMs +) { + public ToolCall { + Objects.requireNonNull(name, "name must not be null"); + arguments = arguments == null ? Map.of() : Map.copyOf(arguments); + } + + /** + * Creates a ToolCall with just a name (minimal form). + */ + public static ToolCall of(String name) { + return new ToolCall(name, Map.of(), null, 0); + } + + /** + * Creates a ToolCall with name and arguments. + */ + public static ToolCall of(String name, Map arguments) { + return new ToolCall(name, arguments, null, 0); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/config/AgentEvalConfigTest.java b/agenteval-core/src/test/java/com/agenteval/core/config/AgentEvalConfigTest.java new file mode 100644 index 0000000..ba02eda --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/config/AgentEvalConfigTest.java @@ -0,0 +1,60 @@ +package com.agenteval.core.config; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AgentEvalConfigTest { + + @Test + void defaultsShouldHaveReasonableValues() { + var config = AgentEvalConfig.defaults(); + + assertThat(config.judgeModel()).isNull(); + assertThat(config.embeddingModel()).isNull(); + assertThat(config.maxConcurrentJudgeCalls()).isPositive(); + assertThat(config.retryOnRateLimit()).isTrue(); + assertThat(config.maxRetries()).isEqualTo(3); + assertThat(config.cacheJudgeResults()).isFalse(); + } + + @Test + void builderShouldSetAllFields() { + var config = AgentEvalConfig.builder() + .maxConcurrentJudgeCalls(8) + .retryOnRateLimit(false) + .maxRetries(5) + .cacheJudgeResults(true) + .build(); + + assertThat(config.maxConcurrentJudgeCalls()).isEqualTo(8); + assertThat(config.retryOnRateLimit()).isFalse(); + assertThat(config.maxRetries()).isEqualTo(5); + assertThat(config.cacheJudgeResults()).isTrue(); + } + + @Test + void shouldRejectZeroConcurrency() { + assertThatThrownBy(() -> AgentEvalConfig.builder() + .maxConcurrentJudgeCalls(0) + .build()) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectNegativeConcurrency() { + assertThatThrownBy(() -> AgentEvalConfig.builder() + .maxConcurrentJudgeCalls(-1) + .build()) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectNegativeMaxRetries() { + assertThatThrownBy(() -> AgentEvalConfig.builder() + .maxRetries(-1) + .build()) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/eval/AgentEvalTest.java b/agenteval-core/src/test/java/com/agenteval/core/eval/AgentEvalTest.java new file mode 100644 index 0000000..b167f5c --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/eval/AgentEvalTest.java @@ -0,0 +1,145 @@ +package com.agenteval.core.eval; + +import com.agenteval.core.metric.EvalMetric; +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AgentEvalTest { + + private static EvalMetric passingMetric() { + return new EvalMetric() { + @Override + public EvalScore evaluate(AgentTestCase testCase) { + return EvalScore.of(0.9, 0.7, "looks good"); + } + + @Override + public String name() { + return "PassingMetric"; + } + }; + } + + private static EvalMetric failingMetric() { + return new EvalMetric() { + @Override + public EvalScore evaluate(AgentTestCase testCase) { + return EvalScore.of(0.3, 0.7, "below threshold"); + } + + @Override + public String name() { + return "FailingMetric"; + } + }; + } + + private static EvalMetric throwingMetric() { + return new EvalMetric() { + @Override + public EvalScore evaluate(AgentTestCase testCase) { + throw new RuntimeException("LLM provider error"); + } + + @Override + public String name() { + return "ThrowingMetric"; + } + }; + } + + @Test + void shouldEvaluateAllTestCasesAgainstAllMetrics() { + var tc1 = AgentTestCase.builder().input("q1").actualOutput("a1").build(); + var tc2 = AgentTestCase.builder().input("q2").actualOutput("a2").build(); + + var result = AgentEval.evaluate( + List.of(tc1, tc2), + List.of(passingMetric()) + ); + + assertThat(result.caseResults()).hasSize(2); + assertThat(result.passRate()).isEqualTo(1.0); + assertThat(result.durationMs()).isNotNegative(); + } + + @Test + void shouldTrackPassAndFailSeparately() { + var tc = AgentTestCase.builder().input("q1").actualOutput("a1").build(); + + var result = AgentEval.evaluate( + List.of(tc), + List.of(passingMetric(), failingMetric()) + ); + + assertThat(result.caseResults()).hasSize(1); + var caseResult = result.caseResults().getFirst(); + assertThat(caseResult.passed()).isFalse(); // one metric failed + assertThat(caseResult.scores()).hasSize(2); + assertThat(caseResult.scores().get("PassingMetric").passed()).isTrue(); + assertThat(caseResult.scores().get("FailingMetric").passed()).isFalse(); + } + + @Test + void shouldHandleMetricExceptionsGracefully() { + var tc = AgentTestCase.builder().input("test").actualOutput("output").build(); + + var result = AgentEval.evaluate( + List.of(tc), + List.of(throwingMetric()) + ); + + assertThat(result.caseResults()).hasSize(1); + var caseResult = result.caseResults().getFirst(); + assertThat(caseResult.passed()).isFalse(); + assertThat(caseResult.scores().get("ThrowingMetric").passed()).isFalse(); + assertThat(caseResult.scores().get("ThrowingMetric").reason()).contains("error"); + } + + @Test + void shouldSetMetricNameOnScores() { + var tc = AgentTestCase.builder().input("q").actualOutput("a").build(); + + var result = AgentEval.evaluate(List.of(tc), List.of(passingMetric())); + + var score = result.caseResults().getFirst().scores().get("PassingMetric"); + assertThat(score.metricName()).isEqualTo("PassingMetric"); + } + + @Test + void shouldRejectNullTestCases() { + assertThatThrownBy(() -> AgentEval.evaluate(null, List.of(passingMetric()))) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldRejectNullMetrics() { + assertThatThrownBy(() -> AgentEval.evaluate(List.of(), null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldHandleEmptyTestCases() { + var result = AgentEval.evaluate(List.of(), List.of(passingMetric())); + + assertThat(result.caseResults()).isEmpty(); + assertThat(result.passRate()).isEqualTo(0.0); + } + + @Test + void configureShouldReturnConfigBuilder() { + var config = AgentEval.configure() + .maxConcurrentJudgeCalls(4) + .cacheJudgeResults(true) + .build(); + + assertThat(config.maxConcurrentJudgeCalls()).isEqualTo(4); + assertThat(config.cacheJudgeResults()).isTrue(); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/eval/EvalResultTest.java b/agenteval-core/src/test/java/com/agenteval/core/eval/EvalResultTest.java new file mode 100644 index 0000000..fb00eb7 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/eval/EvalResultTest.java @@ -0,0 +1,80 @@ +package com.agenteval.core.eval; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class EvalResultTest { + + private CaseResult passingCase() { + var tc = AgentTestCase.builder().input("test").actualOutput("good").build(); + return new CaseResult(tc, Map.of( + "Metric1", EvalScore.of(0.9, 0.7, "great"), + "Metric2", EvalScore.of(0.8, 0.7, "good") + ), true); + } + + private CaseResult failingCase() { + var tc = AgentTestCase.builder().input("test").actualOutput("bad").build(); + return new CaseResult(tc, Map.of( + "Metric1", EvalScore.of(0.5, 0.7, "below threshold"), + "Metric2", EvalScore.of(0.8, 0.7, "ok") + ), false); + } + + @Test + void shouldCalculatePassRate() { + var result = EvalResult.of(List.of(passingCase(), failingCase()), 100); + + assertThat(result.passRate()).isEqualTo(0.5); + } + + @Test + void shouldCalculateAverageScore() { + var result = EvalResult.of(List.of(passingCase(), failingCase()), 100); + + // passingCase avg: (0.9 + 0.8) / 2 = 0.85 + // failingCase avg: (0.5 + 0.8) / 2 = 0.65 + // overall: (0.85 + 0.65) / 2 = 0.75 + assertThat(result.averageScore()).isCloseTo(0.75, org.assertj.core.data.Offset.offset(0.001)); + } + + @Test + void shouldReturnFailedCases() { + var result = EvalResult.of(List.of(passingCase(), failingCase()), 100); + + assertThat(result.failedCases()).hasSize(1); + assertThat(result.failedCases().getFirst().passed()).isFalse(); + } + + @Test + void shouldCalculateAverageScoresByMetric() { + var result = EvalResult.of(List.of(passingCase(), failingCase()), 100); + var byMetric = result.averageScoresByMetric(); + + assertThat(byMetric).containsKey("Metric1"); + assertThat(byMetric).containsKey("Metric2"); + assertThat(byMetric.get("Metric1")).isCloseTo(0.7, org.assertj.core.data.Offset.offset(0.001)); + assertThat(byMetric.get("Metric2")).isCloseTo(0.8, org.assertj.core.data.Offset.offset(0.001)); + } + + @Test + void emptyResultsShouldReturnZeroes() { + var result = EvalResult.of(List.of(), 0); + + assertThat(result.passRate()).isEqualTo(0.0); + assertThat(result.averageScore()).isEqualTo(0.0); + assertThat(result.failedCases()).isEmpty(); + } + + @Test + void shouldTrackDuration() { + var result = EvalResult.of(List.of(passingCase()), 250); + assertThat(result.durationMs()).isEqualTo(250); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/metric/CompositeMetricTest.java b/agenteval-core/src/test/java/com/agenteval/core/metric/CompositeMetricTest.java new file mode 100644 index 0000000..7bf9530 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/metric/CompositeMetricTest.java @@ -0,0 +1,150 @@ +package com.agenteval.core.metric; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class CompositeMetricTest { + + private static EvalMetric stubMetric(String name, double score, double threshold) { + return new EvalMetric() { + @Override + public EvalScore evaluate(AgentTestCase testCase) { + return EvalScore.of(score, threshold, name + " result"); + } + + @Override + public String name() { + return name; + } + }; + } + + @Test + void weightedAverageShouldCalculateCorrectly() { + var composite = CompositeMetric.builder() + .name("Quality") + .add(stubMetric("A", 0.8, 0.5), 0.6) + .add(stubMetric("B", 0.6, 0.5), 0.4) + .strategy(CompositeStrategy.WEIGHTED_AVERAGE) + .threshold(0.7) + .build(); + + var tc = AgentTestCase.builder().input("test").actualOutput("output").build(); + var score = composite.evaluate(tc); + + // (0.8 * 0.6 + 0.6 * 0.4) / (0.6 + 0.4) = 0.72 + assertThat(score.value()).isCloseTo(0.72, org.assertj.core.data.Offset.offset(0.001)); + assertThat(score.passed()).isTrue(); + assertThat(score.metricName()).isEqualTo("Quality"); + } + + @Test + void weightedAverageShouldFailBelowThreshold() { + var composite = CompositeMetric.builder() + .name("Quality") + .add(stubMetric("A", 0.4, 0.5), 0.5) + .add(stubMetric("B", 0.6, 0.5), 0.5) + .strategy(CompositeStrategy.WEIGHTED_AVERAGE) + .threshold(0.7) + .build(); + + var tc = AgentTestCase.builder().input("test").actualOutput("output").build(); + var score = composite.evaluate(tc); + + // (0.4 * 0.5 + 0.6 * 0.5) / 1.0 = 0.5 + assertThat(score.value()).isCloseTo(0.5, org.assertj.core.data.Offset.offset(0.001)); + assertThat(score.passed()).isFalse(); + } + + @Test + void allPassShouldRequireAllMetricsToPass() { + var composite = CompositeMetric.builder() + .name("AllPass") + .add(stubMetric("A", 0.9, 0.7), 1.0) + .add(stubMetric("B", 0.5, 0.7), 1.0) // fails its own threshold + .strategy(CompositeStrategy.ALL_PASS) + .threshold(0.3) + .build(); + + var tc = AgentTestCase.builder().input("test").actualOutput("output").build(); + var score = composite.evaluate(tc); + + assertThat(score.value()).isEqualTo(0.5); // min value + assertThat(score.passed()).isFalse(); // B didn't pass its threshold + } + + @Test + void allPassShouldPassWhenAllMetricsPass() { + var composite = CompositeMetric.builder() + .name("AllPass") + .add(stubMetric("A", 0.9, 0.7), 1.0) + .add(stubMetric("B", 0.8, 0.7), 1.0) + .strategy(CompositeStrategy.ALL_PASS) + .threshold(0.5) + .build(); + + var tc = AgentTestCase.builder().input("test").actualOutput("output").build(); + var score = composite.evaluate(tc); + + assertThat(score.value()).isEqualTo(0.8); + assertThat(score.passed()).isTrue(); + } + + @Test + void anyPassShouldPassIfAnyMetricPasses() { + var composite = CompositeMetric.builder() + .name("AnyPass") + .add(stubMetric("A", 0.3, 0.7), 1.0) // fails + .add(stubMetric("B", 0.9, 0.7), 1.0) // passes + .strategy(CompositeStrategy.ANY_PASS) + .threshold(0.5) + .build(); + + var tc = AgentTestCase.builder().input("test").actualOutput("output").build(); + var score = composite.evaluate(tc); + + assertThat(score.value()).isEqualTo(0.9); // max value + assertThat(score.passed()).isTrue(); + } + + @Test + void shouldRejectNullName() { + assertThatThrownBy(() -> CompositeMetric.builder() + .add(stubMetric("A", 0.5, 0.5), 1.0) + .build()) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldRejectEmptyMetrics() { + assertThatThrownBy(() -> CompositeMetric.builder() + .name("Empty") + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("at least one"); + } + + @Test + void shouldRejectNonPositiveWeight() { + assertThatThrownBy(() -> CompositeMetric.builder() + .name("Bad") + .add(stubMetric("A", 0.5, 0.5), 0.0) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("positive"); + } + + @Test + void nameShouldReturnConfiguredName() { + var composite = CompositeMetric.builder() + .name("MyComposite") + .add(stubMetric("A", 0.5, 0.5), 1.0) + .build(); + + assertThat(composite.name()).isEqualTo("MyComposite"); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/model/AgentTestCaseTest.java b/agenteval-core/src/test/java/com/agenteval/core/model/AgentTestCaseTest.java new file mode 100644 index 0000000..7e227ad --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/model/AgentTestCaseTest.java @@ -0,0 +1,128 @@ +package com.agenteval.core.model; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AgentTestCaseTest { + + @Test + void shouldBuildMinimalTestCase() { + var tc = AgentTestCase.builder() + .input("What is the refund policy?") + .build(); + + assertThat(tc.getInput()).isEqualTo("What is the refund policy?"); + assertThat(tc.getActualOutput()).isNull(); + assertThat(tc.getExpectedOutput()).isNull(); + assertThat(tc.getRetrievalContext()).isEmpty(); + assertThat(tc.getContext()).isEmpty(); + assertThat(tc.getToolCalls()).isEmpty(); + assertThat(tc.getExpectedToolCalls()).isEmpty(); + assertThat(tc.getReasoningTrace()).isEmpty(); + assertThat(tc.getLatencyMs()).isZero(); + assertThat(tc.getTokenUsage()).isNull(); + assertThat(tc.getCost()).isNull(); + assertThat(tc.getMetadata()).isEmpty(); + } + + @Test + void shouldBuildFullTestCase() { + var tc = AgentTestCase.builder() + .input("How do I get a refund?") + .actualOutput("You can request a refund within 30 days.") + .expectedOutput("Full refund within 30 days of purchase.") + .retrievalContext(List.of("doc1", "doc2")) + .context(List.of("ground truth")) + .toolCalls(List.of(ToolCall.of("search"))) + .expectedToolCalls(List.of(ToolCall.of("search"))) + .reasoningTrace(List.of(ReasoningStep.of(ReasoningStepType.PLAN, "search docs"))) + .latencyMs(250) + .tokenUsage(TokenUsage.of(100, 50)) + .cost(new BigDecimal("0.005")) + .metadata(Map.of("category", "refund")) + .build(); + + assertThat(tc.getInput()).isEqualTo("How do I get a refund?"); + assertThat(tc.getActualOutput()).isEqualTo("You can request a refund within 30 days."); + assertThat(tc.getExpectedOutput()).isEqualTo("Full refund within 30 days of purchase."); + assertThat(tc.getRetrievalContext()).hasSize(2); + assertThat(tc.getContext()).hasSize(1); + assertThat(tc.getToolCalls()).hasSize(1); + assertThat(tc.getExpectedToolCalls()).hasSize(1); + assertThat(tc.getReasoningTrace()).hasSize(1); + assertThat(tc.getLatencyMs()).isEqualTo(250); + assertThat(tc.getTokenUsage().totalTokens()).isEqualTo(150); + assertThat(tc.getCost()).isEqualByComparingTo("0.005"); + assertThat(tc.getMetadata()).containsEntry("category", "refund"); + } + + @Test + void shouldRejectNullInput() { + assertThatThrownBy(() -> AgentTestCase.builder().build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("input"); + } + + @Test + void shouldAllowMutableActualOutput() { + var tc = AgentTestCase.builder() + .input("test") + .build(); + + assertThat(tc.getActualOutput()).isNull(); + tc.setActualOutput("response from agent"); + assertThat(tc.getActualOutput()).isEqualTo("response from agent"); + } + + @Test + void shouldAllowMutableLatency() { + var tc = AgentTestCase.builder().input("test").build(); + tc.setLatencyMs(500); + assertThat(tc.getLatencyMs()).isEqualTo(500); + } + + @Test + void shouldAllowMutableTokenUsage() { + var tc = AgentTestCase.builder().input("test").build(); + tc.setTokenUsage(TokenUsage.of(200, 100)); + assertThat(tc.getTokenUsage().totalTokens()).isEqualTo(300); + } + + @Test + void shouldAllowMutableCost() { + var tc = AgentTestCase.builder().input("test").build(); + tc.setCost(new BigDecimal("0.01")); + assertThat(tc.getCost()).isEqualByComparingTo("0.01"); + } + + @Test + void shouldDefensiveCopyLists() { + var context = new java.util.ArrayList<>(List.of("doc1")); + var tc = AgentTestCase.builder() + .input("test") + .retrievalContext(context) + .build(); + + context.add("doc2"); + assertThat(tc.getRetrievalContext()).hasSize(1); + } + + @Test + void shouldDefensiveCopyMetadata() { + var meta = new java.util.HashMap(); + meta.put("key", "value"); + var tc = AgentTestCase.builder() + .input("test") + .metadata(meta) + .build(); + + meta.put("other", "val"); + assertThat(tc.getMetadata()).hasSize(1); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/model/ConversationTestCaseTest.java b/agenteval-core/src/test/java/com/agenteval/core/model/ConversationTestCaseTest.java new file mode 100644 index 0000000..a47c32d --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/model/ConversationTestCaseTest.java @@ -0,0 +1,72 @@ +package com.agenteval.core.model; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ConversationTestCaseTest { + + @Test + void shouldBuildConversationTestCase() { + var turn1 = AgentTestCase.builder() + .input("Hello") + .actualOutput("Hi there!") + .build(); + var turn2 = AgentTestCase.builder() + .input("What is your name?") + .actualOutput("I'm an AI assistant.") + .build(); + + var conversation = ConversationTestCase.builder() + .turns(List.of(turn1, turn2)) + .conversationId("conv-1") + .systemPrompt("You are a helpful assistant.") + .build(); + + assertThat(conversation.getTurns()).hasSize(2); + assertThat(conversation.getConversationId()).isEqualTo("conv-1"); + assertThat(conversation.getSystemPrompt()).isEqualTo("You are a helpful assistant."); + } + + @Test + void shouldRejectNullTurns() { + assertThatThrownBy(() -> ConversationTestCase.builder().build()) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldRejectEmptyTurns() { + assertThatThrownBy(() -> ConversationTestCase.builder() + .turns(List.of()) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + } + + @Test + void shouldAllowNullOptionalFields() { + var turn = AgentTestCase.builder().input("test").build(); + var conversation = ConversationTestCase.builder() + .turns(List.of(turn)) + .build(); + + assertThat(conversation.getConversationId()).isNull(); + assertThat(conversation.getSystemPrompt()).isNull(); + } + + @Test + void shouldDefensiveCopyTurns() { + var turn = AgentTestCase.builder().input("test").build(); + var turns = new java.util.ArrayList<>(List.of(turn)); + + var conversation = ConversationTestCase.builder() + .turns(turns) + .build(); + + turns.add(AgentTestCase.builder().input("extra").build()); + assertThat(conversation.getTurns()).hasSize(1); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/model/EvalScoreTest.java b/agenteval-core/src/test/java/com/agenteval/core/model/EvalScoreTest.java new file mode 100644 index 0000000..9b8a4a9 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/model/EvalScoreTest.java @@ -0,0 +1,90 @@ +package com.agenteval.core.model; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class EvalScoreTest { + + @Test + void ofShouldAutoDerivePassed() { + var passing = EvalScore.of(0.8, 0.7, "good"); + assertThat(passing.passed()).isTrue(); + assertThat(passing.value()).isEqualTo(0.8); + assertThat(passing.threshold()).isEqualTo(0.7); + assertThat(passing.reason()).isEqualTo("good"); + assertThat(passing.metricName()).isEmpty(); + + var failing = EvalScore.of(0.5, 0.7, "below threshold"); + assertThat(failing.passed()).isFalse(); + } + + @Test + void ofShouldPassWhenValueEqualsThreshold() { + var score = EvalScore.of(0.7, 0.7, "exact match"); + assertThat(score.passed()).isTrue(); + } + + @Test + void passShouldCreatePerfectScore() { + var score = EvalScore.pass("all good"); + assertThat(score.value()).isEqualTo(1.0); + assertThat(score.passed()).isTrue(); + assertThat(score.reason()).isEqualTo("all good"); + } + + @Test + void failShouldCreateZeroScore() { + var score = EvalScore.fail("bad output"); + assertThat(score.value()).isEqualTo(0.0); + assertThat(score.passed()).isFalse(); + assertThat(score.reason()).isEqualTo("bad output"); + } + + @Test + void withMetricNameShouldReturnNewInstance() { + var score = EvalScore.of(0.8, 0.7, "test"); + var named = score.withMetricName("Faithfulness"); + + assertThat(named.metricName()).isEqualTo("Faithfulness"); + assertThat(named.value()).isEqualTo(score.value()); + assertThat(score.metricName()).isEmpty(); // original unchanged + } + + @Test + void shouldRejectValueBelowZero() { + assertThatThrownBy(() -> EvalScore.of(-0.1, 0.5, "bad")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectValueAboveOne() { + assertThatThrownBy(() -> EvalScore.of(1.1, 0.5, "bad")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectThresholdBelowZero() { + assertThatThrownBy(() -> EvalScore.of(0.5, -0.1, "bad")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectThresholdAboveOne() { + assertThatThrownBy(() -> EvalScore.of(0.5, 1.1, "bad")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectNullReason() { + assertThatThrownBy(() -> EvalScore.of(0.5, 0.5, null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldDefaultNullMetricNameToEmpty() { + var score = new EvalScore(0.5, 0.5, true, "test", null); + assertThat(score.metricName()).isEmpty(); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/model/ReasoningStepTest.java b/agenteval-core/src/test/java/com/agenteval/core/model/ReasoningStepTest.java new file mode 100644 index 0000000..3297400 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/model/ReasoningStepTest.java @@ -0,0 +1,63 @@ +package com.agenteval.core.model; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ReasoningStepTest { + + @Test + void shouldCreateReasoningStep() { + var step = new ReasoningStep(ReasoningStepType.THOUGHT, "Analyzing the query", null); + + assertThat(step.type()).isEqualTo(ReasoningStepType.THOUGHT); + assertThat(step.content()).isEqualTo("Analyzing the query"); + assertThat(step.toolCall()).isNull(); + } + + @Test + void ofShouldCreateStepWithoutToolCall() { + var step = ReasoningStep.of(ReasoningStepType.PLAN, "Step 1: search docs"); + + assertThat(step.type()).isEqualTo(ReasoningStepType.PLAN); + assertThat(step.content()).isEqualTo("Step 1: search docs"); + assertThat(step.toolCall()).isNull(); + } + + @Test + void actionShouldCreateActionStepWithToolCall() { + var tc = ToolCall.of("search", Map.of("query", "refund")); + var step = ReasoningStep.action("Calling search API", tc); + + assertThat(step.type()).isEqualTo(ReasoningStepType.ACTION); + assertThat(step.content()).isEqualTo("Calling search API"); + assertThat(step.toolCall()).isNotNull(); + assertThat(step.toolCall().name()).isEqualTo("search"); + } + + @Test + void shouldRejectNullType() { + assertThatThrownBy(() -> new ReasoningStep(null, "content", null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldRejectNullContent() { + assertThatThrownBy(() -> new ReasoningStep(ReasoningStepType.THOUGHT, null, null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void allEnumValuesShouldExist() { + assertThat(ReasoningStepType.values()) + .containsExactly( + ReasoningStepType.PLAN, + ReasoningStepType.THOUGHT, + ReasoningStepType.OBSERVATION, + ReasoningStepType.ACTION + ); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/model/TokenUsageTest.java b/agenteval-core/src/test/java/com/agenteval/core/model/TokenUsageTest.java new file mode 100644 index 0000000..c56e848 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/model/TokenUsageTest.java @@ -0,0 +1,51 @@ +package com.agenteval.core.model; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class TokenUsageTest { + + @Test + void shouldCreateTokenUsage() { + var usage = new TokenUsage(100, 50, 150); + + assertThat(usage.inputTokens()).isEqualTo(100); + assertThat(usage.outputTokens()).isEqualTo(50); + assertThat(usage.totalTokens()).isEqualTo(150); + } + + @Test + void ofShouldAutoCalculateTotal() { + var usage = TokenUsage.of(100, 50); + + assertThat(usage.inputTokens()).isEqualTo(100); + assertThat(usage.outputTokens()).isEqualTo(50); + assertThat(usage.totalTokens()).isEqualTo(150); + } + + @Test + void shouldRejectNegativeInputTokens() { + assertThatThrownBy(() -> new TokenUsage(-1, 50, 49)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectNegativeOutputTokens() { + assertThatThrownBy(() -> new TokenUsage(100, -1, 99)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldRejectNegativeTotalTokens() { + assertThatThrownBy(() -> new TokenUsage(100, 50, -1)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldSupportZeroValues() { + var usage = new TokenUsage(0, 0, 0); + assertThat(usage.totalTokens()).isZero(); + } +} diff --git a/agenteval-core/src/test/java/com/agenteval/core/model/ToolCallTest.java b/agenteval-core/src/test/java/com/agenteval/core/model/ToolCallTest.java new file mode 100644 index 0000000..f73ab85 --- /dev/null +++ b/agenteval-core/src/test/java/com/agenteval/core/model/ToolCallTest.java @@ -0,0 +1,63 @@ +package com.agenteval.core.model; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ToolCallTest { + + @Test + void shouldCreateToolCall() { + var tc = new ToolCall("search", Map.of("query", "refund"), "found 3 results", 150); + + assertThat(tc.name()).isEqualTo("search"); + assertThat(tc.arguments()).containsEntry("query", "refund"); + assertThat(tc.result()).isEqualTo("found 3 results"); + assertThat(tc.durationMs()).isEqualTo(150); + } + + @Test + void ofNameShouldCreateMinimalToolCall() { + var tc = ToolCall.of("search"); + + assertThat(tc.name()).isEqualTo("search"); + assertThat(tc.arguments()).isEmpty(); + assertThat(tc.result()).isNull(); + assertThat(tc.durationMs()).isZero(); + } + + @Test + void ofNameArgsShouldCreateToolCallWithArgs() { + var tc = ToolCall.of("search", Map.of("query", "test")); + + assertThat(tc.name()).isEqualTo("search"); + assertThat(tc.arguments()).containsEntry("query", "test"); + assertThat(tc.result()).isNull(); + } + + @Test + void shouldDefensiveCopyArguments() { + var args = new java.util.HashMap(); + args.put("key", "value"); + var tc = new ToolCall("test", args, null, 0); + + // Modifying the original map should not affect the record + args.put("other", "val"); + assertThat(tc.arguments()).hasSize(1); + } + + @Test + void shouldDefaultNullArgumentsToEmptyMap() { + var tc = new ToolCall("test", null, null, 0); + assertThat(tc.arguments()).isEmpty(); + } + + @Test + void shouldRejectNullName() { + assertThatThrownBy(() -> new ToolCall(null, Map.of(), null, 0)) + .isInstanceOf(NullPointerException.class); + } +} diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..58807c7 --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..7ea2a02 --- /dev/null +++ b/pom.xml @@ -0,0 +1,188 @@ + + + 4.0.0 + + com.agenteval + agenteval-parent + 0.1.0-SNAPSHOT + pom + + AgentEval + Java AI Agent Evaluation & Testing Library + https://github.com/agenteval/agenteval + + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0 + + + + + agenteval-core + + + + UTF-8 + 21 + + + 5.11.4 + 2.18.3 + 2.0.16 + 3.27.7 + 1.5.25 + + + 3.13.0 + 3.5.2 + 3.6.0 + 10.21.4 + 4.9.1.0 + + + + + + org.junit + junit-bom + ${junit-bom.version} + pom + import + + + com.fasterxml.jackson + jackson-bom + ${jackson-bom.version} + pom + import + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.assertj + assertj-core + ${assertj.version} + + + ch.qos.logback + logback-classic + ${logback.version} + + + + + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + ch.qos.logback + logback-classic + test + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${java.version} + true + + -Xlint:all + -Werror + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven-checkstyle-plugin.version} + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + checkstyle.xml + true + true + true + + + + validate + validate + + check + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + Max + Medium + true + spotbugs-exclude.xml + + + + spotbugs-check + verify + + check + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + + + + diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml new file mode 100644 index 0000000..0462363 --- /dev/null +++ b/spotbugs-exclude.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + +