Add P2 growth phase: parallel eval, 4 metrics, 3 new modules, reporting

agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfig.java

@@ -4,6 +4,8 @@
 import com.agenteval.core.embedding.EmbeddingModel;
 import com.agenteval.core.judge.JudgeModel;
 
+import com.agenteval.core.eval.ProgressCallback;
+
 import java.math.BigDecimal;
 
 /**
@@ -16,6 +18,8 @@
  *     .retryOnRateLimit(true)
  *     .maxRetries(3)
  *     .cacheJudgeResults(true)
+ *     .parallelEvaluation(true)
+ *     .parallelism(8)
  *     .build();
  * }</pre>
  */
@@ -29,6 +33,9 @@ public final class AgentEvalConfig {
     private final boolean cacheJudgeResults;
     private final BigDecimal costBudget;
     private final PricingModel pricingModel;
+    private final boolean parallelEvaluation;
+    private final int parallelism;
+    private final ProgressCallback progressCallback;
 
     private AgentEvalConfig(Builder builder) {
         this.judgeModel = builder.judgeModel;
@@ -39,6 +46,9 @@ private AgentEvalConfig(Builder builder) {
         this.cacheJudgeResults = builder.cacheJudgeResults;
         this.costBudget = builder.costBudget;
         this.pricingModel = builder.pricingModel;
+        this.parallelEvaluation = builder.parallelEvaluation;
+        this.parallelism = builder.parallelism;
+        this.progressCallback = builder.progressCallback;
     }
 
     public JudgeModel judgeModel() { return judgeModel; }
@@ -49,6 +59,9 @@ private AgentEvalConfig(Builder builder) {
     public boolean cacheJudgeResults() { return cacheJudgeResults; }
     public BigDecimal costBudget() { return costBudget; }
     public PricingModel pricingModel() { return pricingModel; }
+    public boolean parallelEvaluation() { return parallelEvaluation; }
+    public int parallelism() { return parallelism; }
+    public ProgressCallback progressCallback() { return progressCallback; }
 
     public static Builder builder() {
         return new Builder();
@@ -70,6 +83,9 @@ public static final class Builder {
         private boolean cacheJudgeResults = false;
         private BigDecimal costBudget;
         private PricingModel pricingModel;
+        private boolean parallelEvaluation = false;
+        private int parallelism = Runtime.getRuntime().availableProcessors();
+        private ProgressCallback progressCallback;
 
         private Builder() {}
 
@@ -92,6 +108,19 @@ public Builder maxRetries(int maxRetries) {
         public Builder cacheJudgeResults(boolean cache) { this.cacheJudgeResults = cache; return this; }
         public Builder costBudget(BigDecimal budget) { this.costBudget = budget; return this; }
         public Builder pricingModel(PricingModel pricing) { this.pricingModel = pricing; return this; }
+        public Builder parallelEvaluation(boolean parallel) {
+            this.parallelEvaluation = parallel;
+            return this;
+        }
+        public Builder parallelism(int parallelism) {
+            if (parallelism < 1) throw new IllegalArgumentException("parallelism must be >= 1");
+            this.parallelism = parallelism;
+            return this;
+        }
+        public Builder progressCallback(ProgressCallback callback) {
+            this.progressCallback = callback;
+            return this;
+        }
 
         public AgentEvalConfig build() {
             return new AgentEvalConfig(this);

agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfigLoader.java

@@ -84,6 +84,12 @@ private static AgentEvalConfig.Builder toBuilder(YamlConfigModel model) {
             if (defaults.getMaxConcurrentJudgeCalls() != null) {
                 builder.maxConcurrentJudgeCalls(defaults.getMaxConcurrentJudgeCalls());
             }
+            if (defaults.getParallelEvaluation() != null) {
+                builder.parallelEvaluation(defaults.getParallelEvaluation());
+            }
+            if (defaults.getParallelism() != null) {
+                builder.parallelism(defaults.getParallelism());
+            }
         }
 
         if (model.getCost() != null && model.getCost().getBudget() != null) {

agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java

@@ -61,6 +61,8 @@ public static final class DefaultsSection {
         private Integer maxRetries;
         private Boolean retryOnRateLimit;
         private Integer maxConcurrentJudgeCalls;
+        private Boolean parallelEvaluation;
+        private Integer parallelism;
 
         public Double getThreshold() { return threshold; }
         public void setThreshold(Double threshold) { this.threshold = threshold; }
@@ -72,6 +74,12 @@ public void setRetryOnRateLimit(Boolean retryOnRateLimit) {
         }
         public Integer getMaxConcurrentJudgeCalls() { return maxConcurrentJudgeCalls; }
         public void setMaxConcurrentJudgeCalls(Integer max) { this.maxConcurrentJudgeCalls = max; }
+        public Boolean getParallelEvaluation() { return parallelEvaluation; }
+        public void setParallelEvaluation(Boolean parallelEvaluation) {
+            this.parallelEvaluation = parallelEvaluation;
+        }
+        public Integer getParallelism() { return parallelism; }
+        public void setParallelism(Integer parallelism) { this.parallelism = parallelism; }
     }
 
     public static final class CostSection {

agenteval-core/src/main/java/com/agenteval/core/eval/AgentEval.java

@@ -7,10 +7,16 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * Static entry point for running evaluations.
@@ -48,16 +54,87 @@ public static EvalResult evaluate(List<AgentTestCase> testCases, List<EvalMetric
         LOG.info("Starting evaluation: {} test cases, {} metrics", testCases.size(), metrics.size());
         long startTime = System.currentTimeMillis();
 
-        List<CaseResult> caseResults = testCases.stream()
-                .map(tc -> evaluateCase(tc, metrics))
-                .toList();
+        List<CaseResult> caseResults;
+        if (config.parallelEvaluation() && testCases.size() > 1) {
+            caseResults = evaluateParallel(testCases, metrics, config, startTime);
+        } else {
+            caseResults = evaluateSequential(testCases, metrics, config, startTime);
+        }
 
         long durationMs = System.currentTimeMillis() - startTime;
         LOG.info("Evaluation complete in {}ms", durationMs);
 
         return EvalResult.of(caseResults, durationMs);
     }
 
+    private static List<CaseResult> evaluateSequential(
+            List<AgentTestCase> testCases, List<EvalMetric> metrics,
+            AgentEvalConfig config, long startTime) {
+        ProgressCallback callback = config.progressCallback();
+        int total = testCases.size();
+        List<CaseResult> results = new ArrayList<>(total);
+
+        for (int i = 0; i < total; i++) {
+            results.add(evaluateCase(testCases.get(i), metrics));
+            if (callback != null) {
+                callback.onProgress(buildProgressEvent(i + 1, total, startTime));
+            }
+        }
+        return results;
+    }
+
+    private static List<CaseResult> evaluateParallel(
+            List<AgentTestCase> testCases, List<EvalMetric> metrics,
+            AgentEvalConfig config, long startTime) {
+        int total = testCases.size();
+        ProgressCallback callback = config.progressCallback();
+        Semaphore semaphore = new Semaphore(config.parallelism());
+        AtomicInteger completed = new AtomicInteger(0);
+
+        try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
+            List<Future<CaseResult>> futures = new ArrayList<>(total);
+
+            for (AgentTestCase tc : testCases) {
+                futures.add(executor.submit(() -> {
+                    semaphore.acquire();
+                    try {
+                        CaseResult result = evaluateCase(tc, metrics);
+                        int done = completed.incrementAndGet();
+                        if (callback != null) {
+                            callback.onProgress(buildProgressEvent(done, total, startTime));
+                        }
+                        return result;
+                    } finally {
+                        semaphore.release();
+                    }
+                }));
+            }
+
+            List<CaseResult> results = new ArrayList<>(total);
+            for (Future<CaseResult> future : futures) {
+                try {
+                    results.add(future.get());
+                } catch (Exception e) {
+                    LOG.error("Parallel evaluation task failed", e);
+                    throw new EvaluationException("Parallel evaluation failed", e);
+                }
+            }
+            return results;
+        }
+    }
+
+    private static ProgressEvent buildProgressEvent(int done, int total, long startTime) {
+        long elapsed = System.currentTimeMillis() - startTime;
+        long estimatedRemaining = -1;
+        if (done > 0 && done < total) {
+            long msPerCase = elapsed / done;
+            estimatedRemaining = msPerCase * (total - done);
+        } else if (done == total) {
+            estimatedRemaining = 0;
+        }
+        return new ProgressEvent(done, total, elapsed, estimatedRemaining);
+    }
+
     private static CaseResult evaluateCase(AgentTestCase testCase, List<EvalMetric> metrics) {
         Map<String, EvalScore> scores = new LinkedHashMap<>();
         boolean allPassed = true;

agenteval-core/src/main/java/com/agenteval/core/eval/ConsoleProgressBar.java

@@ -0,0 +1,49 @@
+package com.agenteval.core.eval;
+
+import java.io.PrintStream;
+
+/**
+ * Default {@link ProgressCallback} implementation that prints a progress bar to stderr.
+ */
+public final class ConsoleProgressBar implements ProgressCallback {
+
+    private static final int BAR_WIDTH = 30;
+
+    private final PrintStream out;
+
+    public ConsoleProgressBar() {
+        this(System.err);
+    }
+
+    ConsoleProgressBar(PrintStream out) {
+        this.out = out;
+    }
+
+    @Override
+    public void onProgress(ProgressEvent event) {
+        int completed = event.completedCases();
+        int total = event.totalCases();
+        double ratio = event.completionRatio();
+
+        int filled = (int) (ratio * BAR_WIDTH);
+        var bar = new StringBuilder("[");
+        for (int i = 0; i < BAR_WIDTH; i++) {
+            bar.append(i < filled ? '#' : '.');
+        }
+        bar.append(']');
+
+        String eta;
+        if (event.estimatedRemainingMs() < 0) {
+            eta = "ETA: --";
+        } else {
+            long secs = event.estimatedRemainingMs() / 1000;
+            eta = String.format("ETA: %ds", secs);
+        }
+
+        out.printf("\r%s %d/%d (%.0f%%) %s", bar, completed, total, ratio * 100, eta);
+
+        if (completed == total) {
+            out.println();
+        }
+    }
+}

agenteval-core/src/main/java/com/agenteval/core/eval/EvaluationException.java

@@ -0,0 +1,17 @@
+package com.agenteval.core.eval;
+
+/**
+ * Thrown when an evaluation fails due to an unrecoverable error.
+ */
+public class EvaluationException extends RuntimeException {
+
+    private static final long serialVersionUID = 1L;
+
+    public EvaluationException(String message) {
+        super(message);
+    }
+
+    public EvaluationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

agenteval-core/src/main/java/com/agenteval/core/eval/ProgressCallback.java

@@ -0,0 +1,15 @@
+package com.agenteval.core.eval;
+
+/**
+ * Callback invoked during evaluation to report progress.
+ */
+@FunctionalInterface
+public interface ProgressCallback {
+
+    /**
+     * Called after each test case completes evaluation.
+     *
+     * @param event the progress event containing current status and ETA
+     */
+    void onProgress(ProgressEvent event);
+}

agenteval-core/src/main/java/com/agenteval/core/eval/ProgressEvent.java

@@ -0,0 +1,24 @@
+package com.agenteval.core.eval;
+
+/**
+ * Progress information emitted during evaluation.
+ *
+ * @param completedCases number of test cases completed so far
+ * @param totalCases total number of test cases to evaluate
+ * @param elapsedMs milliseconds elapsed since evaluation started
+ * @param estimatedRemainingMs estimated milliseconds remaining, or -1 if unknown
+ */
+public record ProgressEvent(
+        int completedCases,
+        int totalCases,
+        long elapsedMs,
+        long estimatedRemainingMs
+) {
+    /**
+     * Returns the completion ratio as a value between 0.0 and 1.0.
+     */
+    public double completionRatio() {
+        if (totalCases == 0) return 1.0;
+        return (double) completedCases / totalCases;
+    }
+}