diff --git a/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java b/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java index 9fa3fb8..98be693 100644 --- a/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java +++ b/agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java @@ -1,6 +1,7 @@ package com.agenteval.core.config; import java.math.BigDecimal; +import java.util.List; /** * POJO representing the {@code agenteval.yaml} configuration file structure. @@ -29,6 +30,8 @@ public static final class JudgeSection { private String model; private String apiKey; private String baseUrl; + private List models; + private String consensusStrategy; public String getProvider() { return provider; } public void setProvider(String provider) { this.provider = provider; } @@ -38,6 +41,12 @@ public static final class JudgeSection { public void setApiKey(String apiKey) { this.apiKey = apiKey; } public String getBaseUrl() { return baseUrl; } public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + public List getModels() { return models; } + public void setModels(List models) { this.models = models; } + public String getConsensusStrategy() { return consensusStrategy; } + public void setConsensusStrategy(String consensusStrategy) { + this.consensusStrategy = consensusStrategy; + } } public static final class EmbeddingSection { diff --git a/agenteval-datasets/pom.xml b/agenteval-datasets/pom.xml index cef6885..1fd2b9a 100644 --- a/agenteval-datasets/pom.xml +++ b/agenteval-datasets/pom.xml @@ -24,6 +24,14 @@ agenteval-judge true + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + org.slf4j slf4j-api diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java index 8562722..e6612ec 100644 --- a/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java @@ -4,6 +4,8 @@ import com.agenteval.datasets.csv.CsvDatasetWriter; import com.agenteval.datasets.json.JsonDatasetWriter; import com.agenteval.datasets.jsonl.JsonlDatasetWriter; +import com.agenteval.datasets.version.DatasetVersioner; +import com.agenteval.datasets.version.VersionedDataset; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; @@ -72,6 +74,17 @@ public void save(Path path, DatasetFormat format) { } } + /** + * Tags this dataset with a version label and saves it to the storage directory. + * + * @param label the version label (e.g., "v1.0") + * @param storageDir the directory where versioned datasets are stored + * @return the versioned dataset with git metadata + */ + public VersionedDataset tagVersion(String label, Path storageDir) { + return new DatasetVersioner(storageDir).tag(this, label); + } + @JsonPOJOBuilder(withPrefix = "") public static final class Builder { private String name; diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersion.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersion.java new file mode 100644 index 0000000..c31a5ef --- /dev/null +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersion.java @@ -0,0 +1,29 @@ +package com.agenteval.datasets.version; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; + +/** + * Version metadata for a golden dataset. + * + * @param versionLabel the human-readable version label (e.g., "v1.0", "2024-03-baseline") + * @param gitMetadata git repository state at version time (may be null) + * @param createdAt when this version was created + * @param extra arbitrary additional metadata + */ +public record DatasetVersion( + String versionLabel, + GitMetadata gitMetadata, + Instant createdAt, + Map extra +) { + + public DatasetVersion { + Objects.requireNonNull(versionLabel, "versionLabel must not be null"); + if (createdAt == null) { + createdAt = Instant.now(); + } + extra = extra == null ? Map.of() : Map.copyOf(extra); + } +} diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersioner.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersioner.java new file mode 100644 index 0000000..2266e3a --- /dev/null +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersioner.java @@ -0,0 +1,164 @@ +package com.agenteval.datasets.version; + +import com.agenteval.datasets.DatasetException; +import com.agenteval.datasets.EvalDataset; +import com.agenteval.datasets.json.JsonDatasetLoader; +import com.agenteval.datasets.json.JsonDatasetWriter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +/** + * Manages versioned golden datasets with git metadata. + * + *

Stores versioned datasets in a directory structure: + * {@code ///dataset.json} alongside + * a {@code version.json} metadata file.

+ */ +public final class DatasetVersioner { + + private static final Logger LOG = LoggerFactory.getLogger(DatasetVersioner.class); + private static final String DATASET_FILE = "dataset.json"; + private static final String VERSION_FILE = "version.json"; + private static final ObjectMapper MAPPER = new ObjectMapper() + .registerModule(new JavaTimeModule()) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .enable(SerializationFeature.INDENT_OUTPUT); + + private final Path storageDir; + private final GitResolver gitResolver; + + /** + * Creates a versioner that stores datasets under the given directory + * and resolves git metadata from the given working directory. + */ + public DatasetVersioner(Path storageDir, Path gitWorkingDir) { + this.storageDir = Objects.requireNonNull(storageDir, "storageDir must not be null"); + this.gitResolver = new GitResolver( + Objects.requireNonNull(gitWorkingDir, "gitWorkingDir must not be null")); + } + + /** + * Creates a versioner that stores datasets under the given directory. + * Git metadata is resolved from the storage directory itself. + */ + public DatasetVersioner(Path storageDir) { + this(storageDir, storageDir); + } + + /** + * Tags a dataset with a version label, resolves git metadata, and saves it. + * + * @param dataset the dataset to version + * @param label the version label (e.g., "v1.0") + * @return the versioned dataset + */ + public VersionedDataset tag(EvalDataset dataset, String label) { + Objects.requireNonNull(dataset, "dataset must not be null"); + Objects.requireNonNull(label, "label must not be null"); + + String name = dataset.getName(); + if (name == null || name.isBlank()) { + throw new DatasetException("Dataset must have a name for versioning"); + } + + GitMetadata git = gitResolver.resolve(); + DatasetVersion version = new DatasetVersion(label, git, Instant.now(), null); + + Path versionDir = storageDir.resolve(name).resolve(label); + try { + Files.createDirectories(versionDir); + new JsonDatasetWriter().write(dataset, versionDir.resolve(DATASET_FILE)); + MAPPER.writeValue(versionDir.resolve(VERSION_FILE).toFile(), version); + } catch (IOException e) { + throw new DatasetException("Failed to save versioned dataset: " + e.getMessage(), e); + } + + LOG.info("Tagged dataset '{}' as version '{}'", name, label); + return new VersionedDataset(dataset, version); + } + + /** + * Loads a specific version of a dataset. + * + * @param name the dataset name + * @param label the version label + * @return the versioned dataset + */ + public VersionedDataset load(String name, String label) { + Path versionDir = storageDir.resolve(name).resolve(label); + Path datasetFile = versionDir.resolve(DATASET_FILE); + Path versionFile = versionDir.resolve(VERSION_FILE); + + if (!Files.exists(datasetFile)) { + throw new DatasetException( + "Version '" + label + "' not found for dataset '" + name + "'"); + } + + EvalDataset dataset = new JsonDatasetLoader().load(datasetFile); + DatasetVersion version; + try { + version = MAPPER.readValue(versionFile.toFile(), DatasetVersion.class); + } catch (IOException e) { + throw new DatasetException("Failed to read version metadata: " + e.getMessage(), e); + } + + return new VersionedDataset(dataset, version); + } + + /** + * Lists all version labels for a dataset, sorted by creation time (newest first). + * + * @param name the dataset name + * @return the list of version labels + */ + public List listVersions(String name) { + Path datasetDir = storageDir.resolve(name); + if (!Files.isDirectory(datasetDir)) { + return List.of(); + } + + try (Stream dirs = Files.list(datasetDir)) { + return dirs + .filter(Files::isDirectory) + .filter(d -> Files.exists(d.resolve(VERSION_FILE))) + .sorted(Comparator.comparing(d -> { + try { + return Files.getLastModifiedTime(d.resolve(VERSION_FILE)) + .toMillis(); + } catch (IOException e) { + return 0L; + } + }).reversed()) + .map(d -> d.getFileName().toString()) + .toList(); + } catch (IOException e) { + throw new DatasetException("Failed to list versions: " + e.getMessage(), e); + } + } + + /** + * Loads the latest (most recently created) version of a dataset. + * + * @param name the dataset name + * @return the latest versioned dataset + */ + public VersionedDataset latest(String name) { + List versions = listVersions(name); + if (versions.isEmpty()) { + throw new DatasetException("No versions found for dataset '" + name + "'"); + } + return load(name, versions.getFirst()); + } +} diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/version/GitMetadata.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/GitMetadata.java new file mode 100644 index 0000000..a29de54 --- /dev/null +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/GitMetadata.java @@ -0,0 +1,24 @@ +package com.agenteval.datasets.version; + +import java.time.Instant; + +/** + * Git repository metadata captured at the time of dataset versioning. + * + * @param commitHash full commit SHA + * @param shortHash abbreviated commit SHA + * @param branch current branch name (may be null for detached HEAD) + * @param tag most recent tag reachable from HEAD (may be null) + * @param timestamp commit timestamp + * @param authorName commit author name + * @param authorEmail commit author email + */ +public record GitMetadata( + String commitHash, + String shortHash, + String branch, + String tag, + Instant timestamp, + String authorName, + String authorEmail +) {} diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/version/GitResolver.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/GitResolver.java new file mode 100644 index 0000000..c6c1859 --- /dev/null +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/GitResolver.java @@ -0,0 +1,113 @@ +package com.agenteval.datasets.version; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +/** + * Resolves Git metadata from the working directory using {@code git} CLI commands. + * + *

All operations gracefully return null when git is unavailable or the directory + * is not a git repository.

+ */ +public final class GitResolver { + + private static final Logger LOG = LoggerFactory.getLogger(GitResolver.class); + private static final int TIMEOUT_SECONDS = 5; + + private final Path workingDir; + + public GitResolver(Path workingDir) { + this.workingDir = workingDir; + } + + /** + * Checks whether the working directory is inside a git repository. + */ + public boolean isGitRepository() { + String result = runGit("rev-parse", "--is-inside-work-tree"); + return "true".equals(result); + } + + /** + * Resolves full git metadata from the current HEAD. + * + * @return the metadata, or null if git is unavailable or this is not a repository + */ + public GitMetadata resolve() { + if (!isGitRepository()) { + LOG.debug("Not a git repository: {}", workingDir); + return null; + } + + String commitHash = runGit("rev-parse", "HEAD"); + if (commitHash == null) return null; + + String shortHash = runGit("rev-parse", "--short", "HEAD"); + String branch = runGit("rev-parse", "--abbrev-ref", "HEAD"); + String tag = runGit("describe", "--tags", "--abbrev=0"); + String timestampStr = runGit("log", "-1", "--format=%ct"); + String authorName = runGit("log", "-1", "--format=%an"); + String authorEmail = runGit("log", "-1", "--format=%ae"); + + Instant timestamp = null; + if (timestampStr != null) { + try { + timestamp = Instant.ofEpochSecond(Long.parseLong(timestampStr)); + } catch (NumberFormatException e) { + LOG.debug("Could not parse git timestamp: {}", timestampStr); + } + } + + // "HEAD" means detached HEAD + if ("HEAD".equals(branch)) { + branch = null; + } + + return new GitMetadata(commitHash, shortHash, branch, tag, + timestamp, authorName, authorEmail); + } + + @SuppressWarnings("IllegalCatch") + private String runGit(String... args) { + try { + String[] command = new String[args.length + 1]; + command[0] = "git"; + System.arraycopy(args, 0, command, 1, args.length); + + ProcessBuilder pb = new ProcessBuilder(command) + .directory(workingDir.toFile()) + .redirectErrorStream(true); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes(), + StandardCharsets.UTF_8).trim(); + + boolean finished = process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + LOG.debug("Git command timed out: git {}", String.join(" ", args)); + return null; + } + + if (process.exitValue() != 0) { + LOG.debug("Git command failed (exit {}): git {}", + process.exitValue(), String.join(" ", args)); + return null; + } + + return output.isEmpty() ? null : output; + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + LOG.debug("Git command error: {}", e.getMessage()); + return null; + } + } +} diff --git a/agenteval-datasets/src/main/java/com/agenteval/datasets/version/VersionedDataset.java b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/VersionedDataset.java new file mode 100644 index 0000000..2c44b4f --- /dev/null +++ b/agenteval-datasets/src/main/java/com/agenteval/datasets/version/VersionedDataset.java @@ -0,0 +1,39 @@ +package com.agenteval.datasets.version; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.datasets.EvalDataset; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * An {@link EvalDataset} paired with version metadata. + * + *

Provides delegate methods for convenient access to the underlying dataset.

+ * + * @param dataset the evaluation dataset + * @param version the version metadata + */ +public record VersionedDataset(EvalDataset dataset, DatasetVersion version) { + + public VersionedDataset { + Objects.requireNonNull(dataset, "dataset must not be null"); + Objects.requireNonNull(version, "version must not be null"); + } + + /** Delegate: returns the dataset name. */ + public String getName() { return dataset.getName(); } + + /** Delegate: returns the dataset version string. */ + public String getVersion() { return dataset.getVersion(); } + + /** Delegate: returns the test cases. */ + public List getTestCases() { return dataset.getTestCases(); } + + /** Delegate: returns the dataset metadata. */ + public Map getMetadata() { return dataset.getMetadata(); } + + /** Delegate: returns the number of test cases. */ + public int size() { return dataset.size(); } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/version/DatasetVersionerTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/version/DatasetVersionerTest.java new file mode 100644 index 0000000..14323e4 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/version/DatasetVersionerTest.java @@ -0,0 +1,118 @@ +package com.agenteval.datasets.version; + +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.datasets.DatasetException; +import com.agenteval.datasets.EvalDataset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class DatasetVersionerTest { + + @Test + void shouldTagAndLoadDataset(@TempDir Path tempDir) { + var dataset = testDataset("my-golden-set"); + var versioner = new DatasetVersioner(tempDir); + + VersionedDataset versioned = versioner.tag(dataset, "v1.0"); + + assertThat(versioned.dataset()).isNotNull(); + assertThat(versioned.version().versionLabel()).isEqualTo("v1.0"); + assertThat(versioned.version().createdAt()).isNotNull(); + assertThat(versioned.size()).isEqualTo(2); + + // Load it back + VersionedDataset loaded = versioner.load("my-golden-set", "v1.0"); + assertThat(loaded.getName()).isEqualTo("my-golden-set"); + assertThat(loaded.getTestCases()).hasSize(2); + assertThat(loaded.version().versionLabel()).isEqualTo("v1.0"); + } + + @Test + void shouldListVersionsSortedByNewest(@TempDir Path tempDir) throws Exception { + var dataset = testDataset("ds"); + var versioner = new DatasetVersioner(tempDir); + + versioner.tag(dataset, "v1.0"); + // Small delay to ensure different timestamps + Thread.sleep(50); + versioner.tag(dataset, "v2.0"); + + List versions = versioner.listVersions("ds"); + assertThat(versions).containsExactly("v2.0", "v1.0"); + } + + @Test + void shouldLoadLatestVersion(@TempDir Path tempDir) throws Exception { + var dataset = testDataset("ds"); + var versioner = new DatasetVersioner(tempDir); + + versioner.tag(dataset, "v1.0"); + Thread.sleep(50); + versioner.tag(dataset, "v2.0"); + + VersionedDataset latest = versioner.latest("ds"); + assertThat(latest.version().versionLabel()).isEqualTo("v2.0"); + } + + @Test + void shouldReturnEmptyListForUnknownDataset(@TempDir Path tempDir) { + var versioner = new DatasetVersioner(tempDir); + + assertThat(versioner.listVersions("nonexistent")).isEmpty(); + } + + @Test + void shouldThrowWhenLoadingNonexistentVersion(@TempDir Path tempDir) { + var versioner = new DatasetVersioner(tempDir); + + assertThatThrownBy(() -> versioner.load("ds", "v1.0")) + .isInstanceOf(DatasetException.class) + .hasMessageContaining("not found"); + } + + @Test + void shouldThrowWhenLatestCalledWithNoVersions(@TempDir Path tempDir) { + var versioner = new DatasetVersioner(tempDir); + + assertThatThrownBy(() -> versioner.latest("ds")) + .isInstanceOf(DatasetException.class) + .hasMessageContaining("No versions found"); + } + + @Test + void shouldThrowWhenDatasetHasNoName(@TempDir Path tempDir) { + var dataset = EvalDataset.builder() + .testCases(List.of(AgentTestCase.builder().input("q").build())) + .build(); + var versioner = new DatasetVersioner(tempDir); + + assertThatThrownBy(() -> versioner.tag(dataset, "v1.0")) + .isInstanceOf(DatasetException.class) + .hasMessageContaining("name"); + } + + @Test + void tagVersionConvenienceMethodShouldWork(@TempDir Path tempDir) { + var dataset = testDataset("convenience-ds"); + + VersionedDataset versioned = dataset.tagVersion("v1.0", tempDir); + + assertThat(versioned.version().versionLabel()).isEqualTo("v1.0"); + assertThat(versioned.getName()).isEqualTo("convenience-ds"); + } + + private static EvalDataset testDataset(String name) { + return EvalDataset.builder() + .name(name) + .testCases(List.of( + AgentTestCase.builder().input("What is AI?").build(), + AgentTestCase.builder().input("Explain ML").build())) + .build(); + } +} diff --git a/agenteval-datasets/src/test/java/com/agenteval/datasets/version/GitResolverTest.java b/agenteval-datasets/src/test/java/com/agenteval/datasets/version/GitResolverTest.java new file mode 100644 index 0000000..f525729 --- /dev/null +++ b/agenteval-datasets/src/test/java/com/agenteval/datasets/version/GitResolverTest.java @@ -0,0 +1,78 @@ +package com.agenteval.datasets.version; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class GitResolverTest { + + @Test + void shouldDetectNonGitDirectory(@TempDir Path tempDir) { + var resolver = new GitResolver(tempDir); + + assertThat(resolver.isGitRepository()).isFalse(); + assertThat(resolver.resolve()).isNull(); + } + + @Test + void shouldResolveGitMetadataInGitRepo(@TempDir Path tempDir) throws Exception { + // Initialize a git repo with a commit + runCmd(tempDir, "git", "init"); + runCmd(tempDir, "git", "config", "user.name", "Test User"); + runCmd(tempDir, "git", "config", "user.email", "test@example.com"); + java.nio.file.Files.writeString(tempDir.resolve("file.txt"), "content"); + runCmd(tempDir, "git", "add", "."); + runCmd(tempDir, "git", "commit", "-m", "initial commit"); + + var resolver = new GitResolver(tempDir); + + assertThat(resolver.isGitRepository()).isTrue(); + + GitMetadata metadata = resolver.resolve(); + assertThat(metadata).isNotNull(); + assertThat(metadata.commitHash()).isNotBlank(); + assertThat(metadata.commitHash()).hasSize(40); + assertThat(metadata.shortHash()).isNotBlank(); + assertThat(metadata.branch()).isNotNull(); + assertThat(metadata.timestamp()).isNotNull(); + assertThat(metadata.authorName()).isEqualTo("Test User"); + assertThat(metadata.authorEmail()).isEqualTo("test@example.com"); + } + + @Test + void shouldReturnNullTagWhenNoTagsExist(@TempDir Path tempDir) throws Exception { + runCmd(tempDir, "git", "init"); + runCmd(tempDir, "git", "config", "user.name", "Test"); + runCmd(tempDir, "git", "config", "user.email", "test@example.com"); + java.nio.file.Files.writeString(tempDir.resolve("file.txt"), "content"); + runCmd(tempDir, "git", "add", "."); + runCmd(tempDir, "git", "commit", "-m", "commit"); + + var resolver = new GitResolver(tempDir); + GitMetadata metadata = resolver.resolve(); + + assertThat(metadata).isNotNull(); + assertThat(metadata.tag()).isNull(); + } + + private static void runCmd(Path dir, String... cmd) throws Exception { + ProcessBuilder pb = new ProcessBuilder(cmd) + .directory(dir.toFile()) + .redirectErrorStream(true); + // Clear git env vars that hooks set (GIT_AUTHOR_NAME etc.) + // so local git config takes precedence in test repos + pb.environment().remove("GIT_AUTHOR_NAME"); + pb.environment().remove("GIT_AUTHOR_EMAIL"); + pb.environment().remove("GIT_COMMITTER_NAME"); + pb.environment().remove("GIT_COMMITTER_EMAIL"); + pb.environment().remove("GIT_AUTHOR_DATE"); + pb.environment().remove("GIT_COMMITTER_DATE"); + int exit = pb.start().waitFor(); + if (exit != 0) { + throw new RuntimeException("Command failed: " + String.join(" ", cmd)); + } + } +} diff --git a/agenteval-github-actions/pom.xml b/agenteval-github-actions/pom.xml new file mode 100644 index 0000000..152349d --- /dev/null +++ b/agenteval-github-actions/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + + + com.agenteval + agenteval-parent + 0.1.0-SNAPSHOT + + + agenteval-github-actions + AgentEval GitHub Actions + GitHub Actions integration with Markdown reporter and PR commenting + + + + com.agenteval + agenteval-core + + + com.agenteval + agenteval-reporting + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + org.mockito + mockito-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + + + com.agenteval.github.GitHubActionRunner + + + false + + + + + + + diff --git a/agenteval-github-actions/src/main/java/com/agenteval/github/GitHubActionRunner.java b/agenteval-github-actions/src/main/java/com/agenteval/github/GitHubActionRunner.java new file mode 100644 index 0000000..23d5943 --- /dev/null +++ b/agenteval-github-actions/src/main/java/com/agenteval/github/GitHubActionRunner.java @@ -0,0 +1,178 @@ +package com.agenteval.github; + +import com.agenteval.core.eval.EvalResult; +import com.agenteval.reporting.EvalReporter; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Entry point for the GitHub Actions integration. + * + *

Reads environment variables, loads configuration and dataset, runs the evaluation, + * generates a Markdown report, optionally posts a PR comment, and exits non-zero on failure.

+ * + *

Environment variables:

+ *
    + *
  • {@code INPUT_DATASET_PATH} — path to the dataset file (required)
    • + *
  • {@code INPUT_CONFIG_FILE} — path to agenteval.yaml (default: agenteval.yaml)
    • + *
  • {@code INPUT_FAIL_ON_REGRESSION} — exit non-zero if any test fails (default: false)
    • + *
  • {@code INPUT_POST_PR_COMMENT} — post results as PR comment (default: true)
    • + *
  • {@code GITHUB_TOKEN} — GitHub token for PR commenting
    • + *
  • {@code GITHUB_REPOSITORY} — "owner/repo"
    • + *
  • {@code GITHUB_EVENT_PATH} — path to event JSON (to extract PR number)
    • + *
  • {@code GITHUB_API_URL} — GitHub API base URL
    • + *
  • {@code GITHUB_OUTPUT} — path to set action outputs
    • + *
+ */ +public final class GitHubActionRunner { + + private static final Logger LOG = LoggerFactory.getLogger(GitHubActionRunner.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private GitHubActionRunner() {} + + @SuppressWarnings({"IllegalCatch", "SystemExitOutsideMain"}) + public static void main(String[] args) { + try { + run(); + } catch (Exception e) { + LOG.error("AgentEval GitHub Action failed: {}", e.getMessage(), e); + System.exit(1); + } + } + + static void run() throws Exception { + String datasetPath = requireEnv("INPUT_DATASET_PATH"); + String configFile = envOrDefault("INPUT_CONFIG_FILE", "agenteval.yaml"); + boolean failOnRegression = Boolean.parseBoolean( + envOrDefault("INPUT_FAIL_ON_REGRESSION", "false")); + boolean postPrComment = Boolean.parseBoolean( + envOrDefault("INPUT_POST_PR_COMMENT", "true")); + + LOG.info("AgentEval GitHub Action starting"); + LOG.info(" Dataset: {}", datasetPath); + LOG.info(" Config: {}", configFile); + + // Load and run evaluation result from report JSON if available, + // or run directly. For now, we load from a pre-generated JSON report. + Path reportJsonPath = Path.of( + envOrDefault("INPUT_REPORT_JSON", "target/agenteval/agenteval-report.json")); + + if (!Files.exists(reportJsonPath)) { + throw new IllegalStateException( + "Report JSON not found at " + reportJsonPath + + ". Run 'mvn verify' with agenteval-maven-plugin first."); + } + + EvalResult result = loadResult(reportJsonPath); + LOG.info("Loaded evaluation results: {} cases, {:.1f}% pass rate", + result.caseResults().size(), result.passRate() * 100); + + // Generate Markdown report + Path mdPath = Path.of("agenteval-report.md"); + MarkdownConfig mdConfig = MarkdownConfig.builder() + .outputPath(mdPath) + .includeFailedDetails(true) + .includeMetricBreakdown(true) + .build(); + EvalReporter mdReporter = new MarkdownReporter(mdConfig); + mdReporter.report(result); + LOG.info("Markdown report written to {}", mdPath); + + // Post PR comment + if (postPrComment) { + postComment(result, mdConfig); + } + + // Set GitHub Action outputs + setOutput("pass-rate", String.format("%.1f", result.passRate() * 100)); + setOutput("average-score", String.format("%.3f", result.averageScore())); + setOutput("total-cases", String.valueOf(result.caseResults().size())); + setOutput("failed-cases", String.valueOf(result.failedCases().size())); + + // Fail if regression + if (failOnRegression && !result.failedCases().isEmpty()) { + throw new RuntimeException(String.format( + "Evaluation regression: %d/%d cases failed", + result.failedCases().size(), result.caseResults().size())); + } + + LOG.info("AgentEval GitHub Action completed successfully"); + } + + private static void postComment(EvalResult result, MarkdownConfig mdConfig) { + String token = System.getenv("GITHUB_TOKEN"); + String repo = System.getenv("GITHUB_REPOSITORY"); + String apiUrl = envOrDefault("GITHUB_API_URL", "https://api.github.com"); + + if (token == null || repo == null) { + LOG.warn("GITHUB_TOKEN or GITHUB_REPOSITORY not set, skipping PR comment"); + return; + } + + int prNumber = extractPrNumber(); + if (prNumber <= 0) { + LOG.warn("Could not determine PR number, skipping PR comment"); + return; + } + + try { + MarkdownReporter renderer = new MarkdownReporter(mdConfig); + String body = renderer.render(result); + new GitHubPrCommenter(token, apiUrl).postOrUpdate(repo, prNumber, body); + } catch (IOException e) { + LOG.error("Failed to post PR comment: {}", e.getMessage()); + } + } + + @SuppressWarnings("IllegalCatch") + private static int extractPrNumber() { + String eventPath = System.getenv("GITHUB_EVENT_PATH"); + if (eventPath == null) return -1; + + try { + var eventJson = MAPPER.readTree(Path.of(eventPath).toFile()); + return eventJson.path("pull_request").path("number").asInt(-1); + } catch (Exception e) { + LOG.debug("Could not extract PR number from event: {}", e.getMessage()); + return -1; + } + } + + private static EvalResult loadResult(Path jsonPath) throws IOException { + return MAPPER.readValue(jsonPath.toFile(), EvalResult.class); + } + + private static void setOutput(String name, String value) { + String outputFile = System.getenv("GITHUB_OUTPUT"); + if (outputFile != null) { + try { + Files.writeString(Path.of(outputFile), + name + "=" + value + "\n", + java.nio.charset.StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND, + java.nio.file.StandardOpenOption.CREATE); + } catch (IOException e) { + LOG.debug("Could not write GitHub output: {}", e.getMessage()); + } + } + } + + private static String requireEnv(String name) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException("Required environment variable not set: " + name); + } + return value; + } + + private static String envOrDefault(String name, String defaultValue) { + String value = System.getenv(name); + return (value != null && !value.isBlank()) ? value : defaultValue; + } +} diff --git a/agenteval-github-actions/src/main/java/com/agenteval/github/GitHubPrCommenter.java b/agenteval-github-actions/src/main/java/com/agenteval/github/GitHubPrCommenter.java new file mode 100644 index 0000000..cdfef48 --- /dev/null +++ b/agenteval-github-actions/src/main/java/com/agenteval/github/GitHubPrCommenter.java @@ -0,0 +1,166 @@ +package com.agenteval.github; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Objects; + +/** + * Posts or updates evaluation results as a PR comment via the GitHub REST API. + * + *

Uses a hidden HTML marker ({@code }) to identify + * and update existing comments instead of creating duplicates.

+ */ +public final class GitHubPrCommenter { + + private static final Logger LOG = LoggerFactory.getLogger(GitHubPrCommenter.class); + private static final String MARKER = ""; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final String token; + private final String apiUrl; + private final HttpClient httpClient; + + /** + * Creates a commenter for the given repository and PR. + * + * @param token GitHub token (from GITHUB_TOKEN env var) + * @param apiUrl base GitHub API URL (e.g., "https://api.github.com") + */ + public GitHubPrCommenter(String token, String apiUrl) { + this(token, apiUrl, HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build()); + } + + GitHubPrCommenter(String token, String apiUrl, HttpClient httpClient) { + this.token = Objects.requireNonNull(token, "token must not be null"); + this.apiUrl = Objects.requireNonNull(apiUrl, "apiUrl must not be null") + .replaceAll("/$", ""); + this.httpClient = Objects.requireNonNull(httpClient, "httpClient must not be null"); + } + + /** + * Posts or updates a comment on the specified PR. + * + * @param repo repository in "owner/repo" format + * @param prNumber the pull request number + * @param body the Markdown comment body + */ + public void postOrUpdate(String repo, int prNumber, String body) throws IOException { + String markedBody = MARKER + "\n" + body; + String commentsUrl = apiUrl + "/repos/" + repo + "/issues/" + prNumber + "/comments"; + + // Check for existing comment with our marker + Long existingCommentId = findExistingComment(commentsUrl); + + if (existingCommentId != null) { + updateComment(commentsUrl, existingCommentId, markedBody); + LOG.info("Updated existing AgentEval comment on PR #{}", prNumber); + } else { + createComment(commentsUrl, markedBody); + LOG.info("Created new AgentEval comment on PR #{}", prNumber); + } + } + + private Long findExistingComment(String commentsUrl) throws IOException { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(commentsUrl)) + .header("Authorization", "Bearer " + token) + .header("Accept", "application/vnd.github+json") + .GET() + .build(); + + try { + HttpResponse response = httpClient.send(request, + HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + + if (response.statusCode() != 200) { + LOG.warn("Failed to list comments (status {})", response.statusCode()); + return null; + } + + JsonNode comments = MAPPER.readTree(response.body()); + for (JsonNode comment : comments) { + String commentBody = comment.path("body").asText(""); + if (commentBody.contains(MARKER)) { + return comment.path("id").asLong(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while listing comments", e); + } + + return null; + } + + private void createComment(String commentsUrl, String body) throws IOException { + ObjectNode payload = MAPPER.createObjectNode(); + payload.put("body", body); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(commentsUrl)) + .header("Authorization", "Bearer " + token) + .header("Accept", "application/vnd.github+json") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + MAPPER.writeValueAsString(payload), StandardCharsets.UTF_8)) + .build(); + + sendRequest(request, "create comment"); + } + + private void updateComment(String commentsUrl, long commentId, String body) throws IOException { + // PATCH /repos/{owner}/{repo}/issues/comments/{comment_id} + String patchUrl = apiUrl + "/repos/" + + extractRepo(commentsUrl) + "/issues/comments/" + commentId; + + ObjectNode payload = MAPPER.createObjectNode(); + payload.put("body", body); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(patchUrl)) + .header("Authorization", "Bearer " + token) + .header("Accept", "application/vnd.github+json") + .header("Content-Type", "application/json") + .method("PATCH", HttpRequest.BodyPublishers.ofString( + MAPPER.writeValueAsString(payload), StandardCharsets.UTF_8)) + .build(); + + sendRequest(request, "update comment"); + } + + private void sendRequest(HttpRequest request, String action) throws IOException { + try { + HttpResponse response = httpClient.send(request, + HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + + if (response.statusCode() >= 300) { + LOG.error("Failed to {} (status {}): {}", action, + response.statusCode(), response.body()); + throw new IOException("GitHub API " + action + " failed with status " + + response.statusCode()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during " + action, e); + } + } + + private static String extractRepo(String commentsUrl) { + // commentsUrl = https://api.github.com/repos/owner/repo/issues/123/comments + String[] parts = commentsUrl.split("/repos/")[1].split("/issues/"); + return parts[0]; + } +} diff --git a/agenteval-github-actions/src/main/java/com/agenteval/github/MarkdownConfig.java b/agenteval-github-actions/src/main/java/com/agenteval/github/MarkdownConfig.java new file mode 100644 index 0000000..78d06f8 --- /dev/null +++ b/agenteval-github-actions/src/main/java/com/agenteval/github/MarkdownConfig.java @@ -0,0 +1,56 @@ +package com.agenteval.github; + +import java.nio.file.Path; +import java.util.Objects; + +/** + * Configuration for the Markdown report generator. + */ +public final class MarkdownConfig { + + private final Path outputPath; + private final boolean includeFailedDetails; + private final boolean includeMetricBreakdown; + + private MarkdownConfig(Builder builder) { + this.outputPath = Objects.requireNonNull(builder.outputPath, + "outputPath must not be null"); + this.includeFailedDetails = builder.includeFailedDetails; + this.includeMetricBreakdown = builder.includeMetricBreakdown; + } + + public Path outputPath() { return outputPath; } + public boolean includeFailedDetails() { return includeFailedDetails; } + public boolean includeMetricBreakdown() { return includeMetricBreakdown; } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private Path outputPath; + private boolean includeFailedDetails = true; + private boolean includeMetricBreakdown = true; + + private Builder() {} + + public Builder outputPath(Path outputPath) { + this.outputPath = outputPath; + return this; + } + + public Builder includeFailedDetails(boolean includeFailedDetails) { + this.includeFailedDetails = includeFailedDetails; + return this; + } + + public Builder includeMetricBreakdown(boolean includeMetricBreakdown) { + this.includeMetricBreakdown = includeMetricBreakdown; + return this; + } + + public MarkdownConfig build() { + return new MarkdownConfig(this); + } + } +} diff --git a/agenteval-github-actions/src/main/java/com/agenteval/github/MarkdownReporter.java b/agenteval-github-actions/src/main/java/com/agenteval/github/MarkdownReporter.java new file mode 100644 index 0000000..7642f7f --- /dev/null +++ b/agenteval-github-actions/src/main/java/com/agenteval/github/MarkdownReporter.java @@ -0,0 +1,116 @@ +package com.agenteval.github; + +import com.agenteval.core.eval.CaseResult; +import com.agenteval.core.eval.EvalResult; +import com.agenteval.core.model.EvalScore; +import com.agenteval.reporting.EvalReporter; +import com.agenteval.reporting.ReportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Map; +import java.util.Objects; + +/** + * Generates GitHub-Flavored Markdown evaluation reports. + * + *

Produces GFM tables with metric averages, pass rates, and optional + * failed case details in collapsible {@code

} tags.

+ */ +public final class MarkdownReporter implements EvalReporter { + + private static final Logger LOG = LoggerFactory.getLogger(MarkdownReporter.class); + + private final MarkdownConfig config; + + public MarkdownReporter(MarkdownConfig config) { + this.config = Objects.requireNonNull(config, "config must not be null"); + } + + @Override + public void report(EvalResult result) { + LOG.debug("Generating Markdown report at {}", config.outputPath()); + String markdown = render(result); + try { + Files.writeString(config.outputPath(), markdown, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new ReportException( + "Failed to write Markdown report to " + config.outputPath(), e); + } + } + + /** + * Renders the evaluation result as a Markdown string. + */ + public String render(EvalResult result) { + var sb = new StringBuilder(); + + sb.append("## AgentEval Results\n\n"); + + // Summary table + int total = result.caseResults().size(); + int failed = result.failedCases().size(); + int passed = total - failed; + String passRateStr = String.format("%.1f%%", result.passRate() * 100); + String avgScoreStr = String.format("%.3f", result.averageScore()); + String statusIcon = failed == 0 ? "✅" : "❌"; + + sb.append("| Status | Cases | Passed | Failed | Pass Rate | Avg Score | Duration |\n"); + sb.append("|--------|-------|--------|--------|-----------|-----------|----------|\n"); + sb.append(String.format("| %s | %d | %d | %d | %s | %s | %dms |\n\n", + statusIcon, total, passed, failed, passRateStr, avgScoreStr, + result.durationMs())); + + // Per-metric breakdown + if (config.includeMetricBreakdown()) { + Map byMetric = result.averageScoresByMetric(); + if (!byMetric.isEmpty()) { + sb.append("### Per-Metric Averages\n\n"); + sb.append("| Metric | Average Score | Status |\n"); + sb.append("|--------|---------------|--------|\n"); + byMetric.forEach((name, avg) -> { + String icon = avg >= 0.5 ? "✅" : "❌"; + sb.append(String.format("| %s | %.3f | %s |\n", name, avg, icon)); + }); + sb.append("\n"); + } + } + + // Failed case details + if (config.includeFailedDetails() && !result.failedCases().isEmpty()) { + sb.append("
\n"); + sb.append("Failed Cases (") + .append(failed).append(")\n\n"); + + for (CaseResult cr : result.failedCases()) { + String input = truncate(cr.testCase().getInput(), 80); + sb.append("#### ").append(escapeMarkdown(input)).append("\n\n"); + sb.append("| Metric | Score | Threshold | Reason |\n"); + sb.append("|--------|-------|-----------|--------|\n"); + for (EvalScore score : cr.failedScores()) { + sb.append(String.format("| %s | %.2f | %.2f | %s |\n", + score.metricName(), score.value(), + score.threshold(), escapeMarkdown(score.reason()))); + } + sb.append("\n"); + } + + sb.append("
\n"); + } + + return sb.toString(); + } + + private static String truncate(String s, int max) { + if (s == null) return ""; + return s.length() <= max ? s : s.substring(0, max - 3) + "..."; + } + + private static String escapeMarkdown(String s) { + if (s == null) return ""; + return s.replace("|", "\\|").replace("\n", " "); + } +} diff --git a/agenteval-github-actions/src/main/resources/action.yml b/agenteval-github-actions/src/main/resources/action.yml new file mode 100644 index 0000000..48a4b68 --- /dev/null +++ b/agenteval-github-actions/src/main/resources/action.yml @@ -0,0 +1,58 @@ +name: 'AgentEval' +description: 'Run AgentEval AI agent evaluations and post results to PRs' +author: 'AgentEval' + +inputs: + dataset-path: + description: 'Path to the evaluation dataset file' + required: true + config-file: + description: 'Path to agenteval.yaml configuration file' + required: false + default: 'agenteval.yaml' + fail-on-regression: + description: 'Exit with failure if any test case fails' + required: false + default: 'false' + post-pr-comment: + description: 'Post evaluation results as a PR comment' + required: false + default: 'true' + report-json: + description: 'Path to a pre-generated JSON evaluation report' + required: false + default: 'target/agenteval/agenteval-report.json' + java-version: + description: 'Java version to use' + required: false + default: '21' + +outputs: + pass-rate: + description: 'Overall pass rate as a percentage' + average-score: + description: 'Overall average score' + total-cases: + description: 'Total number of test cases evaluated' + failed-cases: + description: 'Number of failed test cases' + +runs: + using: 'composite' + steps: + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + + - name: Run AgentEval + shell: bash + env: + INPUT_DATASET_PATH: ${{ inputs.dataset-path }} + INPUT_CONFIG_FILE: ${{ inputs.config-file }} + INPUT_FAIL_ON_REGRESSION: ${{ inputs.fail-on-regression }} + INPUT_POST_PR_COMMENT: ${{ inputs.post-pr-comment }} + INPUT_REPORT_JSON: ${{ inputs.report-json }} + run: | + java -jar ${{ github.action_path }}/agenteval-github-actions.jar diff --git a/agenteval-github-actions/src/test/java/com/agenteval/github/GitHubPrCommenterTest.java b/agenteval-github-actions/src/test/java/com/agenteval/github/GitHubPrCommenterTest.java new file mode 100644 index 0000000..6e8a51b --- /dev/null +++ b/agenteval-github-actions/src/test/java/com/agenteval/github/GitHubPrCommenterTest.java @@ -0,0 +1,141 @@ +package com.agenteval.github; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class GitHubPrCommenterTest { + + @SuppressWarnings("unchecked") + @Test + void shouldCreateNewCommentWhenNoneExists() throws Exception { + HttpClient mockClient = mock(HttpClient.class); + HttpResponse listResponse = mock(HttpResponse.class); + when(listResponse.statusCode()).thenReturn(200); + when(listResponse.body()).thenReturn("[]"); + + HttpResponse createResponse = mock(HttpResponse.class); + when(createResponse.statusCode()).thenReturn(201); + when(createResponse.body()).thenReturn("{\"id\":1}"); + + List requests = new ArrayList<>(); + when(mockClient.send(any(HttpRequest.class), any())) + .thenAnswer(invocation -> { + HttpRequest req = invocation.getArgument(0); + requests.add(req); + // First call = list, second call = create + if (requests.size() == 1) return listResponse; + return createResponse; + }); + + var commenter = new GitHubPrCommenter("test-token", + "https://api.github.com", mockClient); + commenter.postOrUpdate("owner/repo", 42, "Test results"); + + assertThat(requests).hasSize(2); + assertThat(requests.get(0).method()).isEqualTo("GET"); + assertThat(requests.get(1).method()).isEqualTo("POST"); + assertThat(requests.get(0).uri().toString()) + .contains("/repos/owner/repo/issues/42/comments"); + } + + @SuppressWarnings("unchecked") + @Test + void shouldUpdateExistingCommentWithMarker() throws Exception { + HttpClient mockClient = mock(HttpClient.class); + HttpResponse listResponse = mock(HttpResponse.class); + when(listResponse.statusCode()).thenReturn(200); + when(listResponse.body()).thenReturn( + "[{\"id\":99,\"body\":\"\\nOld results\"}]"); + + HttpResponse updateResponse = mock(HttpResponse.class); + when(updateResponse.statusCode()).thenReturn(200); + when(updateResponse.body()).thenReturn("{\"id\":99}"); + + List requests = new ArrayList<>(); + when(mockClient.send(any(HttpRequest.class), any())) + .thenAnswer(invocation -> { + HttpRequest req = invocation.getArgument(0); + requests.add(req); + if (requests.size() == 1) return listResponse; + return updateResponse; + }); + + var commenter = new GitHubPrCommenter("test-token", + "https://api.github.com", mockClient); + commenter.postOrUpdate("owner/repo", 42, "New results"); + + assertThat(requests).hasSize(2); + assertThat(requests.get(0).method()).isEqualTo("GET"); + assertThat(requests.get(1).method()).isEqualTo("PATCH"); + assertThat(requests.get(1).uri().toString()).contains("/issues/comments/99"); + } + + @SuppressWarnings("unchecked") + @Test + void shouldThrowOnCreateFailure() throws Exception { + HttpClient mockClient = mock(HttpClient.class); + HttpResponse listResponse = mock(HttpResponse.class); + when(listResponse.statusCode()).thenReturn(200); + when(listResponse.body()).thenReturn("[]"); + + HttpResponse errorResponse = mock(HttpResponse.class); + when(errorResponse.statusCode()).thenReturn(403); + when(errorResponse.body()).thenReturn("{\"message\":\"Forbidden\"}"); + + List> responses = List.of(listResponse, errorResponse); + var callCount = new int[]{0}; + when(mockClient.send(any(HttpRequest.class), any())) + .thenAnswer(invocation -> responses.get(callCount[0]++)); + + var commenter = new GitHubPrCommenter("bad-token", + "https://api.github.com", mockClient); + + assertThatThrownBy(() -> commenter.postOrUpdate("owner/repo", 42, "Results")) + .isInstanceOf(IOException.class) + .hasMessageContaining("failed with status 403"); + } + + @Test + void shouldIncludeAuthorizationHeader() throws Exception { + HttpClient mockClient = mock(HttpClient.class); + + @SuppressWarnings("unchecked") + HttpResponse listResponse = mock(HttpResponse.class); + when(listResponse.statusCode()).thenReturn(200); + when(listResponse.body()).thenReturn("[]"); + + @SuppressWarnings("unchecked") + HttpResponse createResponse = mock(HttpResponse.class); + when(createResponse.statusCode()).thenReturn(201); + when(createResponse.body()).thenReturn("{\"id\":1}"); + + List requests = new ArrayList<>(); + when(mockClient.send(any(HttpRequest.class), any())) + .thenAnswer(invocation -> { + requests.add(invocation.getArgument(0)); + if (requests.size() == 1) return listResponse; + return createResponse; + }); + + var commenter = new GitHubPrCommenter("my-secret-token", + "https://api.github.com", mockClient); + commenter.postOrUpdate("owner/repo", 1, "Results"); + + for (HttpRequest req : requests) { + assertThat(req.headers().firstValue("Authorization")) + .hasValue("Bearer my-secret-token"); + } + } +} diff --git a/agenteval-github-actions/src/test/java/com/agenteval/github/MarkdownReporterTest.java b/agenteval-github-actions/src/test/java/com/agenteval/github/MarkdownReporterTest.java new file mode 100644 index 0000000..8071489 --- /dev/null +++ b/agenteval-github-actions/src/test/java/com/agenteval/github/MarkdownReporterTest.java @@ -0,0 +1,137 @@ +package com.agenteval.github; + +import com.agenteval.core.eval.CaseResult; +import com.agenteval.core.eval.EvalResult; +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class MarkdownReporterTest { + + @Test + void shouldGenerateMarkdownWithSummaryTable(@TempDir Path tempDir) throws Exception { + Path outPath = tempDir.resolve("report.md"); + var config = MarkdownConfig.builder() + .outputPath(outPath) + .includeMetricBreakdown(true) + .includeFailedDetails(true) + .build(); + + var reporter = new MarkdownReporter(config); + reporter.report(sampleResult()); + + assertThat(outPath).exists(); + String content = Files.readString(outPath); + assertThat(content).contains("## AgentEval Results"); + assertThat(content).contains("| Status |"); + assertThat(content).contains("Cases"); + assertThat(content).contains("Pass Rate"); + } + + @Test + void shouldIncludeMetricBreakdown() { + var config = MarkdownConfig.builder() + .outputPath(Path.of("test.md")) + .includeMetricBreakdown(true) + .build(); + var reporter = new MarkdownReporter(config); + + String markdown = reporter.render(sampleResult()); + + assertThat(markdown).contains("### Per-Metric Averages"); + assertThat(markdown).contains("Relevancy"); + } + + @Test + void shouldExcludeMetricBreakdownWhenDisabled() { + var config = MarkdownConfig.builder() + .outputPath(Path.of("test.md")) + .includeMetricBreakdown(false) + .build(); + var reporter = new MarkdownReporter(config); + + String markdown = reporter.render(sampleResult()); + + assertThat(markdown).doesNotContain("### Per-Metric Averages"); + } + + @Test + void shouldIncludeFailedDetailsInCollapsible() { + var config = MarkdownConfig.builder() + .outputPath(Path.of("test.md")) + .includeFailedDetails(true) + .build(); + var reporter = new MarkdownReporter(config); + + String markdown = reporter.render(resultWithFailures()); + + assertThat(markdown).contains("
"); + assertThat(markdown).contains("Failed Cases"); + assertThat(markdown).contains("
"); + } + + @Test + void shouldExcludeFailedDetailsWhenDisabled() { + var config = MarkdownConfig.builder() + .outputPath(Path.of("test.md")) + .includeFailedDetails(false) + .build(); + var reporter = new MarkdownReporter(config); + + String markdown = reporter.render(resultWithFailures()); + + assertThat(markdown).doesNotContain("
"); + } + + @Test + void shouldShowCheckmarkForAllPassing() { + var config = MarkdownConfig.builder() + .outputPath(Path.of("test.md")) + .build(); + var reporter = new MarkdownReporter(config); + + String markdown = reporter.render(sampleResult()); + + assertThat(markdown).contains("✅"); + } + + @Test + void shouldShowCrossForFailures() { + var config = MarkdownConfig.builder() + .outputPath(Path.of("test.md")) + .build(); + var reporter = new MarkdownReporter(config); + + String markdown = reporter.render(resultWithFailures()); + + assertThat(markdown).contains("❌"); + } + + private static EvalResult sampleResult() { + var tc = AgentTestCase.builder().input("What is AI?").build(); + var score = new EvalScore(0.9, 0.7, true, "Good answer", "Relevancy"); + var caseResult = new CaseResult(tc, Map.of("Relevancy", score), true); + return EvalResult.of(List.of(caseResult), 150); + } + + private static EvalResult resultWithFailures() { + var tc1 = AgentTestCase.builder().input("What is AI?").build(); + var score1 = new EvalScore(0.9, 0.7, true, "Good answer", "Relevancy"); + + var tc2 = AgentTestCase.builder().input("Explain quantum computing").build(); + var score2 = new EvalScore(0.3, 0.7, false, "Off topic", "Relevancy"); + + return EvalResult.of(List.of( + new CaseResult(tc1, Map.of("Relevancy", score1), true), + new CaseResult(tc2, Map.of("Relevancy", score2), false) + ), 200); + } +} diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/JudgeModels.java b/agenteval-judge/src/main/java/com/agenteval/judge/JudgeModels.java index 69da369..d15d9d2 100644 --- a/agenteval-judge/src/main/java/com/agenteval/judge/JudgeModels.java +++ b/agenteval-judge/src/main/java/com/agenteval/judge/JudgeModels.java @@ -2,6 +2,7 @@ import com.agenteval.core.judge.JudgeModel; import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.multi.MultiModelJudge; import com.agenteval.judge.provider.AnthropicJudgeModel; import com.agenteval.judge.provider.OllamaJudgeModel; import com.agenteval.judge.provider.OpenAiJudgeModel; @@ -88,6 +89,21 @@ public static JudgeModel ollama(JudgeConfig config) { return new OllamaJudgeModel(config); } + /** + * Creates a multi-model judge builder for combining multiple judge models. + * + * 
{@code
+     * var judge = JudgeModels.multi()
+     *     .add(JudgeModels.openai("gpt-4o"))
+     *     .add(JudgeModels.anthropic("claude-sonnet-4-20250514"), 2.0)
+     *     .strategy(ConsensusStrategy.WEIGHTED_AVERAGE)
+     *     .build();
+     * }
+ */ + public static MultiModelJudge.Builder multi() { + return MultiModelJudge.builder(); + } + private static String resolveApiKey(String envVar, String providerName) { String key = System.getenv(envVar); if (key == null || key.isBlank()) { diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/multi/ConsensusStrategy.java b/agenteval-judge/src/main/java/com/agenteval/judge/multi/ConsensusStrategy.java new file mode 100644 index 0000000..45016be --- /dev/null +++ b/agenteval-judge/src/main/java/com/agenteval/judge/multi/ConsensusStrategy.java @@ -0,0 +1,19 @@ +package com.agenteval.judge.multi; + +/** + * Strategy for aggregating scores from multiple judge models. + */ +public enum ConsensusStrategy { + + /** Score passes if more than half of judges agree (score >= threshold). */ + MAJORITY, + + /** Final score is the arithmetic mean of all judge scores. */ + AVERAGE, + + /** Final score is the weighted mean of all judge scores. */ + WEIGHTED_AVERAGE, + + /** Score passes only if all judges agree (score >= threshold). */ + UNANIMOUS +} diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/multi/IndividualJudgeResult.java b/agenteval-judge/src/main/java/com/agenteval/judge/multi/IndividualJudgeResult.java new file mode 100644 index 0000000..6e02a04 --- /dev/null +++ b/agenteval-judge/src/main/java/com/agenteval/judge/multi/IndividualJudgeResult.java @@ -0,0 +1,35 @@ +package com.agenteval.judge.multi; + +import com.agenteval.core.judge.JudgeResponse; + +/** + * The result from a single judge within a multi-model evaluation. + * + * @param modelId the judge model identifier + * @param response the judge's response (null if the judge failed) + * @param weight the weight assigned to this judge + * @param error the error message if the judge failed (null otherwise) + */ +public record IndividualJudgeResult( + String modelId, + JudgeResponse response, + double weight, + String error +) { + + /** Creates a successful result. */ + public static IndividualJudgeResult success(String modelId, JudgeResponse response, + double weight) { + return new IndividualJudgeResult(modelId, response, weight, null); + } + + /** Creates a failed result. */ + public static IndividualJudgeResult failure(String modelId, double weight, String error) { + return new IndividualJudgeResult(modelId, null, weight, error); + } + + /** Returns true if this judge produced a successful response. */ + public boolean succeeded() { + return response != null; + } +} diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/multi/MultiJudgeResponse.java b/agenteval-judge/src/main/java/com/agenteval/judge/multi/MultiJudgeResponse.java new file mode 100644 index 0000000..cbe0333 --- /dev/null +++ b/agenteval-judge/src/main/java/com/agenteval/judge/multi/MultiJudgeResponse.java @@ -0,0 +1,38 @@ +package com.agenteval.judge.multi; + +import com.agenteval.core.judge.JudgeResponse; + +import java.util.List; +import java.util.Objects; + +/** + * Aggregated response from a multi-model judge evaluation. + * + * @param consensusResponse the aggregated consensus response + * @param individualResults results from each individual judge + */ +public record MultiJudgeResponse( + JudgeResponse consensusResponse, + List individualResults +) { + + public MultiJudgeResponse { + Objects.requireNonNull(consensusResponse, "consensusResponse must not be null"); + individualResults = individualResults == null + ? List.of() : List.copyOf(individualResults); + } + + /** Returns only the successful individual results. */ + public List successfulResults() { + return individualResults.stream() + .filter(IndividualJudgeResult::succeeded) + .toList(); + } + + /** Returns only the failed individual results. */ + public List failedResults() { + return individualResults.stream() + .filter(r -> !r.succeeded()) + .toList(); + } +} diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/multi/MultiModelJudge.java b/agenteval-judge/src/main/java/com/agenteval/judge/multi/MultiModelJudge.java new file mode 100644 index 0000000..c23ef75 --- /dev/null +++ b/agenteval-judge/src/main/java/com/agenteval/judge/multi/MultiModelJudge.java @@ -0,0 +1,247 @@ +package com.agenteval.judge.multi; + +import com.agenteval.core.judge.JudgeModel; +import com.agenteval.core.judge.JudgeResponse; +import com.agenteval.core.model.TokenUsage; +import com.agenteval.judge.JudgeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +/** + * A composite judge that fans out evaluation to multiple {@link JudgeModel} instances + * and aggregates their scores using a configurable {@link ConsensusStrategy}. + * + *

Judges are invoked concurrently using virtual threads. The last aggregated + * {@link MultiJudgeResponse} is available via {@link #lastMultiJudgeResponse()}.

+ * + * 
{@code
+ * var judge = MultiModelJudge.builder()
+ *     .add(JudgeModels.openai("gpt-4o"))
+ *     .add(JudgeModels.anthropic("claude-sonnet-4-20250514"), 2.0)
+ *     .strategy(ConsensusStrategy.WEIGHTED_AVERAGE)
+ *     .build();
+ * }
+ */ +public final class MultiModelJudge implements JudgeModel { + + private static final Logger LOG = LoggerFactory.getLogger(MultiModelJudge.class); + + private final List judges; + private final ConsensusStrategy strategy; + private final boolean failOnAnyError; + private final ThreadLocal lastResponse = new ThreadLocal<>(); + + private MultiModelJudge(Builder builder) { + this.judges = List.copyOf(builder.judges); + this.strategy = builder.strategy; + this.failOnAnyError = builder.failOnAnyError; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public JudgeResponse judge(String prompt) { + LOG.debug("Multi-model judge invoked with {} judges, strategy={}", judges.size(), strategy); + + List results = fanOut(prompt); + List successful = results.stream() + .filter(IndividualJudgeResult::succeeded) + .toList(); + + if (successful.isEmpty()) { + String errors = results.stream() + .map(r -> r.modelId() + ": " + r.error()) + .collect(Collectors.joining("; ")); + throw new JudgeException("All judges failed: " + errors); + } + + JudgeResponse consensus = aggregate(successful); + var multiResponse = new MultiJudgeResponse(consensus, results); + lastResponse.set(multiResponse); + + return consensus; + } + + @Override + public String modelId() { + return "multi[" + judges.stream() + .map(wj -> wj.model().modelId()) + .collect(Collectors.joining(",")) + "]"; + } + + /** + * Returns the full multi-judge response from the last invocation on this thread. + */ + public MultiJudgeResponse lastMultiJudgeResponse() { + return lastResponse.get(); + } + + private List fanOut(String prompt) { + List results = new ArrayList<>(); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(); + + for (WeightedJudge wj : judges) { + futures.add(executor.submit(() -> invokeJudge(wj, prompt))); + } + + for (Future future : futures) { + results.add(future.get()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new JudgeException("Multi-model judge interrupted", e); + } catch (ExecutionException e) { + throw new JudgeException("Multi-model judge execution failed", e.getCause()); + } + + if (failOnAnyError) { + List failures = results.stream() + .filter(r -> !r.succeeded()) + .toList(); + if (!failures.isEmpty()) { + String errors = failures.stream() + .map(r -> r.modelId() + ": " + r.error()) + .collect(Collectors.joining("; ")); + throw new JudgeException("Judge(s) failed with failOnAnyError=true: " + errors); + } + } + + return results; + } + + @SuppressWarnings("IllegalCatch") + private IndividualJudgeResult invokeJudge(WeightedJudge wj, String prompt) { + try { + JudgeResponse response = wj.model().judge(prompt); + LOG.debug("Judge {} returned score={}", wj.model().modelId(), response.score()); + return IndividualJudgeResult.success(wj.model().modelId(), response, wj.weight()); + } catch (Exception e) { + LOG.warn("Judge {} failed: {}", wj.model().modelId(), e.getMessage()); + return IndividualJudgeResult.failure(wj.model().modelId(), wj.weight(), e.getMessage()); + } + } + + private JudgeResponse aggregate(List successful) { + return switch (strategy) { + case AVERAGE -> aggregateAverage(successful); + case WEIGHTED_AVERAGE -> aggregateWeightedAverage(successful); + case MAJORITY -> aggregateMajority(successful); + case UNANIMOUS -> aggregateUnanimous(successful); + }; + } + + private JudgeResponse aggregateAverage(List results) { + double avgScore = results.stream() + .mapToDouble(r -> r.response().score()) + .average() + .orElse(0.0); + return buildConsensusResponse(avgScore, results, "Average of " + results.size() + " judges"); + } + + private JudgeResponse aggregateWeightedAverage(List results) { + double totalWeight = results.stream().mapToDouble(IndividualJudgeResult::weight).sum(); + double weightedSum = results.stream() + .mapToDouble(r -> r.response().score() * r.weight()) + .sum(); + double score = totalWeight > 0 ? weightedSum / totalWeight : 0.0; + return buildConsensusResponse(score, results, + "Weighted average of " + results.size() + " judges"); + } + + private JudgeResponse aggregateMajority(List results) { + // Majority: use average score, but reason reflects majority vote + double avgScore = results.stream() + .mapToDouble(r -> r.response().score()) + .average() + .orElse(0.0); + long highScoreCount = results.stream() + .filter(r -> r.response().score() >= 0.5) + .count(); + boolean majorityPass = highScoreCount > results.size() / 2.0; + String reason = String.format("Majority vote: %d/%d judges scored >= 0.5", + highScoreCount, results.size()); + double finalScore = majorityPass ? avgScore : Math.min(avgScore, 0.49); + return buildConsensusResponse(finalScore, results, reason); + } + + private JudgeResponse aggregateUnanimous(List results) { + double avgScore = results.stream() + .mapToDouble(r -> r.response().score()) + .average() + .orElse(0.0); + boolean allPass = results.stream() + .allMatch(r -> r.response().score() >= 0.5); + String reason = allPass + ? "Unanimous: all " + results.size() + " judges scored >= 0.5" + : "Not unanimous: not all judges scored >= 0.5"; + double finalScore = allPass ? avgScore : 0.0; + return buildConsensusResponse(finalScore, results, reason); + } + + private JudgeResponse buildConsensusResponse(double score, + List results, + String reason) { + int totalInput = results.stream() + .mapToInt(r -> r.response().tokenUsage() != null + ? r.response().tokenUsage().inputTokens() : 0) + .sum(); + int totalOutput = results.stream() + .mapToInt(r -> r.response().tokenUsage() != null + ? r.response().tokenUsage().outputTokens() : 0) + .sum(); + TokenUsage totalUsage = TokenUsage.of(totalInput, totalOutput); + + return new JudgeResponse(score, reason, totalUsage); + } + + public static final class Builder { + private final List judges = new ArrayList<>(); + private ConsensusStrategy strategy = ConsensusStrategy.AVERAGE; + private boolean failOnAnyError; + + private Builder() {} + + /** Adds a judge with default weight of 1.0. */ + public Builder add(JudgeModel model) { + return add(model, 1.0); + } + + /** Adds a judge with the specified weight. */ + public Builder add(JudgeModel model, double weight) { + judges.add(new WeightedJudge(model, weight)); + return this; + } + + /** Sets the consensus strategy. Default is {@link ConsensusStrategy#AVERAGE}. */ + public Builder strategy(ConsensusStrategy strategy) { + this.strategy = Objects.requireNonNull(strategy, "strategy must not be null"); + return this; + } + + /** If true, throws when any single judge fails. Default is false. */ + public Builder failOnAnyError(boolean failOnAnyError) { + this.failOnAnyError = failOnAnyError; + return this; + } + + public MultiModelJudge build() { + if (judges.isEmpty()) { + throw new IllegalStateException("At least one judge must be added"); + } + return new MultiModelJudge(this); + } + } +} diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/multi/WeightedJudge.java b/agenteval-judge/src/main/java/com/agenteval/judge/multi/WeightedJudge.java new file mode 100644 index 0000000..780af63 --- /dev/null +++ b/agenteval-judge/src/main/java/com/agenteval/judge/multi/WeightedJudge.java @@ -0,0 +1,21 @@ +package com.agenteval.judge.multi; + +import com.agenteval.core.judge.JudgeModel; + +/** + * A judge model paired with a weight for weighted consensus strategies. + * + * @param model the judge model instance + * @param weight the relative weight (must be positive) + */ +public record WeightedJudge(JudgeModel model, double weight) { + + public WeightedJudge { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + if (weight <= 0.0) { + throw new IllegalArgumentException("weight must be positive, got: " + weight); + } + } +} diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/multi/MultiModelJudgeTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/multi/MultiModelJudgeTest.java new file mode 100644 index 0000000..b47fcf0 --- /dev/null +++ b/agenteval-judge/src/test/java/com/agenteval/judge/multi/MultiModelJudgeTest.java @@ -0,0 +1,239 @@ +package com.agenteval.judge.multi; + +import com.agenteval.core.judge.JudgeModel; +import com.agenteval.core.judge.JudgeResponse; +import com.agenteval.core.model.TokenUsage; +import com.agenteval.judge.JudgeException; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +class MultiModelJudgeTest { + + @Test + void averageStrategyShouldComputeMean() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.8)) + .add(stubJudge("b", 0.6)) + .strategy(ConsensusStrategy.AVERAGE) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + assertThat(response.score()).isCloseTo(0.7, within(0.001)); + assertThat(response.reason()).contains("Average"); + } + + @Test + void weightedAverageStrategyShouldRespectWeights() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.8), 3.0) + .add(stubJudge("b", 0.2), 1.0) + .strategy(ConsensusStrategy.WEIGHTED_AVERAGE) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + // (0.8*3 + 0.2*1) / (3+1) = 2.6/4 = 0.65 + assertThat(response.score()).isCloseTo(0.65, within(0.001)); + assertThat(response.reason()).contains("Weighted average"); + } + + @Test + void majorityStrategyShouldPassWhenMajorityScoresHigh() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.9)) + .add(stubJudge("b", 0.7)) + .add(stubJudge("c", 0.3)) + .strategy(ConsensusStrategy.MAJORITY) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + // 2/3 judges scored >= 0.5, majority passes + assertThat(response.score()).isGreaterThanOrEqualTo(0.5); + assertThat(response.reason()).contains("Majority vote"); + } + + @Test + void majorityStrategyShouldCapScoreWhenMajorityFails() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.1)) + .add(stubJudge("b", 0.2)) + .add(stubJudge("c", 0.9)) + .strategy(ConsensusStrategy.MAJORITY) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + // Only 1/3 judges scored >= 0.5, majority fails + assertThat(response.score()).isLessThan(0.5); + } + + @Test + void unanimousStrategyShouldPassWhenAllAgree() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.9)) + .add(stubJudge("b", 0.7)) + .strategy(ConsensusStrategy.UNANIMOUS) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + assertThat(response.score()).isCloseTo(0.8, within(0.001)); + assertThat(response.reason()).contains("Unanimous"); + } + + @Test + void unanimousStrategyShouldReturnZeroWhenNotAllAgree() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.9)) + .add(stubJudge("b", 0.3)) + .strategy(ConsensusStrategy.UNANIMOUS) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + assertThat(response.score()).isEqualTo(0.0); + assertThat(response.reason()).contains("Not unanimous"); + } + + @Test + void shouldSumTokenUsageAcrossJudges() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.8, TokenUsage.of(100, 50))) + .add(stubJudge("b", 0.6, TokenUsage.of(200, 80))) + .strategy(ConsensusStrategy.AVERAGE) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + assertThat(response.tokenUsage().inputTokens()).isEqualTo(300); + assertThat(response.tokenUsage().outputTokens()).isEqualTo(130); + assertThat(response.tokenUsage().totalTokens()).isEqualTo(430); + } + + @Test + void shouldHandlePartialJudgeFailure() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.8)) + .add(failingJudge("b")) + .strategy(ConsensusStrategy.AVERAGE) + .build(); + + JudgeResponse response = judge.judge("test prompt"); + + // Only the successful judge contributes + assertThat(response.score()).isCloseTo(0.8, within(0.001)); + } + + @Test + void shouldThrowWhenAllJudgesFail() { + var judge = MultiModelJudge.builder() + .add(failingJudge("a")) + .add(failingJudge("b")) + .strategy(ConsensusStrategy.AVERAGE) + .build(); + + assertThatThrownBy(() -> judge.judge("test prompt")) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("All judges failed"); + } + + @Test + void failOnAnyErrorShouldThrowOnSingleFailure() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.8)) + .add(failingJudge("b")) + .strategy(ConsensusStrategy.AVERAGE) + .failOnAnyError(true) + .build(); + + assertThatThrownBy(() -> judge.judge("test prompt")) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("failOnAnyError"); + } + + @Test + void shouldStoreLastMultiJudgeResponseInThreadLocal() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.8)) + .add(stubJudge("b", 0.6)) + .strategy(ConsensusStrategy.AVERAGE) + .build(); + + judge.judge("test prompt"); + MultiJudgeResponse multiResponse = judge.lastMultiJudgeResponse(); + + assertThat(multiResponse).isNotNull(); + assertThat(multiResponse.consensusResponse().score()).isCloseTo(0.7, within(0.001)); + assertThat(multiResponse.individualResults()).hasSize(2); + assertThat(multiResponse.successfulResults()).hasSize(2); + assertThat(multiResponse.failedResults()).isEmpty(); + } + + @Test + void modelIdShouldListAllModels() { + var judge = MultiModelJudge.builder() + .add(stubJudge("gpt-4o", 0.8)) + .add(stubJudge("claude", 0.6)) + .strategy(ConsensusStrategy.AVERAGE) + .build(); + + assertThat(judge.modelId()).isEqualTo("multi[gpt-4o,claude]"); + } + + @Test + void builderShouldRejectEmptyJudges() { + assertThatThrownBy(() -> MultiModelJudge.builder().build()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("At least one judge"); + } + + @Test + void defaultWeightShouldBeOne() { + var judge = MultiModelJudge.builder() + .add(stubJudge("a", 0.8)) + .strategy(ConsensusStrategy.WEIGHTED_AVERAGE) + .build(); + + judge.judge("test"); + MultiJudgeResponse response = judge.lastMultiJudgeResponse(); + + assertThat(response.individualResults().getFirst().weight()).isEqualTo(1.0); + } + + private static JudgeModel stubJudge(String id, double score) { + return stubJudge(id, score, TokenUsage.of(10, 5)); + } + + private static JudgeModel stubJudge(String id, double score, TokenUsage usage) { + return new JudgeModel() { + @Override + public JudgeResponse judge(String prompt) { + return new JudgeResponse(score, "Stub reason from " + id, usage); + } + + @Override + public String modelId() { + return id; + } + }; + } + + private static JudgeModel failingJudge(String id) { + return new JudgeModel() { + @Override + public JudgeResponse judge(String prompt) { + throw new RuntimeException("Simulated failure for " + id); + } + + @Override + public String modelId() { + return id; + } + }; + } +} diff --git a/agenteval-maven-plugin/pom.xml b/agenteval-maven-plugin/pom.xml new file mode 100644 index 0000000..2e92edb --- /dev/null +++ b/agenteval-maven-plugin/pom.xml @@ -0,0 +1,80 @@ + + + 4.0.0 + + + com.agenteval + agenteval-parent + 0.1.0-SNAPSHOT + + + agenteval-maven-plugin + maven-plugin + AgentEval Maven Plugin + Maven plugin for running AgentEval evaluations + + + 3.9.9 + 3.15.1 + 3.15.1 + + + + + org.apache.maven + maven-plugin-api + ${maven-plugin-api.version} + provided + + + org.apache.maven.plugin-tools + maven-plugin-annotations + ${maven-plugin-annotations.version} + provided + + + com.agenteval + agenteval-core + + + com.agenteval + agenteval-judge + + + com.agenteval + agenteval-metrics + + + com.agenteval + agenteval-datasets + + + com.agenteval + agenteval-reporting + + + org.slf4j + slf4j-api + + + + + + + org.apache.maven.plugins + maven-plugin-plugin + ${maven-plugin-plugin.version} + + + + com.github.spotbugs + spotbugs-maven-plugin + + true + + + + + diff --git a/agenteval-maven-plugin/src/main/java/com/agenteval/maven/EvaluateMojo.java b/agenteval-maven-plugin/src/main/java/com/agenteval/maven/EvaluateMojo.java new file mode 100644 index 0000000..e7325d6 --- /dev/null +++ b/agenteval-maven-plugin/src/main/java/com/agenteval/maven/EvaluateMojo.java @@ -0,0 +1,177 @@ +package com.agenteval.maven; + +import com.agenteval.core.config.AgentEvalConfigLoader; +import com.agenteval.core.config.YamlConfigModel; +import com.agenteval.core.eval.CaseResult; +import com.agenteval.core.eval.EvalResult; +import com.agenteval.core.judge.JudgeModel; +import com.agenteval.core.metric.EvalMetric; +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.core.model.EvalScore; +import com.agenteval.datasets.DatasetLoaders; +import com.agenteval.datasets.EvalDataset; +import com.agenteval.judge.JudgeModels; +import com.agenteval.reporting.EvalReporter; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Runs AgentEval evaluations against a dataset and reports results. + * + * 
{@code
+ * 
+ *   com.agenteval
+ *   agenteval-maven-plugin
+ *   
+ *     src/test/resources/golden-set.json
+ *     AnswerRelevancy,Faithfulness,Correctness
+ *     console,json
+ *   
+ * 
+ * }
+ */ +@Mojo(name = "evaluate", defaultPhase = LifecyclePhase.VERIFY) +public class EvaluateMojo extends AbstractMojo { + + @Parameter(property = "agenteval.datasetPath", required = true) + private String datasetPath; + + @Parameter(property = "agenteval.configFile", defaultValue = "agenteval.yaml") + private String configFile; + + @Parameter(property = "agenteval.reportFormats", defaultValue = "console,json") + private String reportFormats; + + @Parameter(property = "agenteval.outputDirectory", + defaultValue = "${project.build.directory}/agenteval") + private String outputDirectory; + + @Parameter(property = "agenteval.failOnRegression", defaultValue = "false") + private boolean failOnRegression; + + @Parameter(property = "agenteval.threshold", defaultValue = "0.7") + private double threshold; + + @Parameter(property = "agenteval.metrics", defaultValue = "AnswerRelevancy") + private String metrics; + + @Override + public void execute() throws MojoExecutionException, MojoFailureException { + try { + getLog().info("AgentEval: Loading dataset from " + datasetPath); + + Path dsPath = Path.of(datasetPath); + if (!Files.exists(dsPath)) { + throw new MojoExecutionException("Dataset not found: " + datasetPath); + } + + EvalDataset dataset = DatasetLoaders.forPath(dsPath); + getLog().info("AgentEval: Loaded " + dataset.size() + " test cases"); + + // Resolve judge from config file or environment + JudgeModel judge = resolveJudge(); + + // Resolve metrics + List evalMetrics = resolveMetrics(judge); + getLog().info("AgentEval: Running " + evalMetrics.size() + " metrics"); + + // Run evaluation + long start = System.currentTimeMillis(); + List caseResults = new ArrayList<>(); + for (AgentTestCase testCase : dataset.getTestCases()) { + Map scores = new HashMap<>(); + boolean allPassed = true; + for (EvalMetric metric : evalMetrics) { + EvalScore score = metric.evaluate(testCase); + score = score.withMetricName(metric.name()); + scores.put(metric.name(), score); + if (!score.passed()) { + allPassed = false; + } + } + caseResults.add(new CaseResult(testCase, scores, allPassed)); + } + long duration = System.currentTimeMillis() - start; + EvalResult result = EvalResult.of(caseResults, duration); + + // Resolve and run reporters + Path outDir = Path.of(outputDirectory); + Files.createDirectories(outDir); + List reporters = ReportFormatResolver.resolve(reportFormats, outDir); + for (EvalReporter reporter : reporters) { + reporter.report(result); + } + + // Check pass rate + if (failOnRegression && result.passRate() < 1.0) { + throw new MojoFailureException(String.format( + "AgentEval: Evaluation failed — pass rate %.1f%% (threshold: 100%%)", + result.passRate() * 100)); + } + + getLog().info(String.format("AgentEval: Completed — %.1f%% pass rate, avg score %.3f", + result.passRate() * 100, result.averageScore())); + + } catch (MojoFailureException e) { + throw e; + } catch (Exception e) { + throw new MojoExecutionException("AgentEval evaluation failed", e); + } + } + + @SuppressWarnings("IllegalCatch") + private JudgeModel resolveJudge() { + Path cfgPath = Path.of(configFile); + if (Files.exists(cfgPath)) { + try { + YamlConfigModel model = AgentEvalConfigLoader.loadModel(cfgPath); + if (model.getJudge() != null) { + String provider = model.getJudge().getProvider(); + String modelName = model.getJudge().getModel(); + if (provider != null && modelName != null) { + return createJudge(provider, modelName); + } + } + } catch (Exception e) { + getLog().debug("Could not load config: " + e.getMessage()); + } + } + + // Fall back to environment variables + String provider = System.getenv("AGENTEVAL_JUDGE_PROVIDER"); + String model = System.getenv("AGENTEVAL_JUDGE_MODEL"); + if (provider != null && model != null) { + return createJudge(provider, model); + } + + return null; + } + + private JudgeModel createJudge(String provider, String model) { + return switch (provider.toLowerCase(java.util.Locale.ROOT)) { + case "openai" -> JudgeModels.openai(model); + case "anthropic" -> JudgeModels.anthropic(model); + case "ollama" -> JudgeModels.ollama(model); + default -> throw new IllegalArgumentException("Unknown judge provider: " + provider); + }; + } + + private List resolveMetrics(JudgeModel judge) { + List resolved = new ArrayList<>(); + for (String name : metrics.split(",")) { + resolved.add(MetricResolver.resolve(name.trim(), judge)); + } + return resolved; + } +} diff --git a/agenteval-maven-plugin/src/main/java/com/agenteval/maven/MetricResolver.java b/agenteval-maven-plugin/src/main/java/com/agenteval/maven/MetricResolver.java new file mode 100644 index 0000000..bca2129 --- /dev/null +++ b/agenteval-maven-plugin/src/main/java/com/agenteval/maven/MetricResolver.java @@ -0,0 +1,97 @@ +package com.agenteval.maven; + +import com.agenteval.core.judge.JudgeModel; +import com.agenteval.core.metric.EvalMetric; +import com.agenteval.metrics.agent.ToolSelectionAccuracyMetric; +import com.agenteval.metrics.response.AnswerRelevancyMetric; +import com.agenteval.metrics.response.BiasMetric; +import com.agenteval.metrics.response.CoherenceMetric; +import com.agenteval.metrics.response.ConcisenessMetric; +import com.agenteval.metrics.response.CorrectnessMetric; +import com.agenteval.metrics.response.FaithfulnessMetric; +import com.agenteval.metrics.response.HallucinationMetric; +import com.agenteval.metrics.response.ToxicityMetric; +import com.agenteval.metrics.rag.ContextualPrecisionMetric; +import com.agenteval.metrics.rag.ContextualRecallMetric; +import com.agenteval.metrics.rag.ContextualRelevancyMetric; +import com.agenteval.metrics.agent.TaskCompletionMetric; + +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; + +/** + * Resolves metric name strings to {@link EvalMetric} instances. + * + *

Metric names are case-insensitive. All LLM-based metrics require a {@link JudgeModel}.

+ */ +public final class MetricResolver { + + private static final Map> LLM_METRICS = Map.ofEntries( + entry("answerrelevancy", AnswerRelevancyMetric::new), + entry("faithfulness", FaithfulnessMetric::new), + entry("correctness", CorrectnessMetric::new), + entry("hallucination", HallucinationMetric::new), + entry("toxicity", ToxicityMetric::new), + entry("coherence", CoherenceMetric::new), + entry("conciseness", ConcisenessMetric::new), + entry("bias", BiasMetric::new), + entry("contextualrelevancy", ContextualRelevancyMetric::new), + entry("contextualprecision", ContextualPrecisionMetric::new), + entry("contextualrecall", ContextualRecallMetric::new), + entry("taskcompletion", TaskCompletionMetric::new) + ); + + private static final Map STANDALONE_METRICS = Map.of( + "toolselectionaccuracy", new ToolSelectionAccuracyMetric() + ); + + private MetricResolver() {} + + /** + * Resolves a metric by name. LLM-based metrics use the provided judge. + * + * @param name the metric name (case-insensitive) + * @param judge the judge model for LLM-based metrics (may be null for standalone metrics) + * @return the resolved metric + * @throws IllegalArgumentException if the metric name is unknown + */ + public static EvalMetric resolve(String name, JudgeModel judge) { + String key = normalize(name); + + EvalMetric standalone = STANDALONE_METRICS.get(key); + if (standalone != null) { + return standalone; + } + + Function factory = LLM_METRICS.get(key); + if (factory != null) { + if (judge == null) { + throw new IllegalArgumentException( + "Metric '" + name + "' requires a judge model"); + } + return factory.apply(judge); + } + + throw new IllegalArgumentException("Unknown metric: " + name + + ". Available: " + availableMetrics()); + } + + /** + * Returns comma-separated list of all known metric names. + */ + public static String availableMetrics() { + var all = new java.util.TreeSet(); + all.addAll(LLM_METRICS.keySet()); + all.addAll(STANDALONE_METRICS.keySet()); + return String.join(", ", all); + } + + private static String normalize(String name) { + return name.toLowerCase(Locale.ROOT).replace("_", "").replace("-", ""); + } + + private static Map.Entry entry(String key, T value) { + return Map.entry(key, value); + } +} diff --git a/agenteval-maven-plugin/src/main/java/com/agenteval/maven/ReportFormatResolver.java b/agenteval-maven-plugin/src/main/java/com/agenteval/maven/ReportFormatResolver.java new file mode 100644 index 0000000..03c6dc9 --- /dev/null +++ b/agenteval-maven-plugin/src/main/java/com/agenteval/maven/ReportFormatResolver.java @@ -0,0 +1,53 @@ +package com.agenteval.maven; + +import com.agenteval.reporting.ConsoleReporter; +import com.agenteval.reporting.EvalReporter; +import com.agenteval.reporting.HtmlReportConfig; +import com.agenteval.reporting.HtmlReporter; +import com.agenteval.reporting.JsonReporter; +import com.agenteval.reporting.JunitXmlReporter; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * Resolves report format strings to {@link EvalReporter} instances. + * + *

Supported formats: console, json, xml, html.

+ */ +public final class ReportFormatResolver { + + private ReportFormatResolver() {} + + /** + * Resolves a list of format names to reporter instances. + * + * @param formats comma-separated format names + * @param outputDirectory the directory for file-based reports + * @return the list of reporters + * @throws IllegalArgumentException if a format name is unknown + */ + public static List resolve(String formats, Path outputDirectory) { + List reporters = new ArrayList<>(); + for (String format : formats.split(",")) { + reporters.add(resolveOne(format.trim(), outputDirectory)); + } + return reporters; + } + + private static EvalReporter resolveOne(String format, Path outputDirectory) { + return switch (format.toLowerCase(Locale.ROOT)) { + case "console" -> new ConsoleReporter(); + case "json" -> new JsonReporter(outputDirectory.resolve("agenteval-report.json")); + case "xml" -> new JunitXmlReporter(outputDirectory.resolve("agenteval-report.xml")); + case "html" -> new HtmlReporter(HtmlReportConfig.builder() + .outputPath(outputDirectory.resolve("agenteval-report.html")) + .build()); + default -> throw new IllegalArgumentException( + "Unknown report format: " + format + + ". Supported: console, json, xml, html"); + }; + } +} diff --git a/agenteval-maven-plugin/src/test/java/com/agenteval/maven/MetricResolverTest.java b/agenteval-maven-plugin/src/test/java/com/agenteval/maven/MetricResolverTest.java new file mode 100644 index 0000000..bf2dec6 --- /dev/null +++ b/agenteval-maven-plugin/src/test/java/com/agenteval/maven/MetricResolverTest.java @@ -0,0 +1,92 @@ +package com.agenteval.maven; + +import com.agenteval.core.judge.JudgeModel; +import com.agenteval.core.judge.JudgeResponse; +import com.agenteval.core.metric.EvalMetric; +import com.agenteval.core.model.TokenUsage; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class MetricResolverTest { + + private final JudgeModel stubJudge = new JudgeModel() { + @Override + public JudgeResponse judge(String prompt) { + return new JudgeResponse(0.8, "good", TokenUsage.of(10, 5)); + } + + @Override + public String modelId() { return "stub"; } + }; + + @Test + void shouldResolveLlmMetricByName() { + EvalMetric metric = MetricResolver.resolve("AnswerRelevancy", stubJudge); + + assertThat(metric).isNotNull(); + assertThat(metric.name()).isEqualTo("AnswerRelevancy"); + } + + @Test + void shouldResolveCaseInsensitively() { + EvalMetric metric = MetricResolver.resolve("answerrelevancy", stubJudge); + + assertThat(metric).isNotNull(); + assertThat(metric.name()).isEqualTo("AnswerRelevancy"); + } + + @Test + void shouldResolveWithHyphensAndUnderscores() { + EvalMetric metric = MetricResolver.resolve("answer-relevancy", stubJudge); + assertThat(metric.name()).isEqualTo("AnswerRelevancy"); + + EvalMetric metric2 = MetricResolver.resolve("answer_relevancy", stubJudge); + assertThat(metric2.name()).isEqualTo("AnswerRelevancy"); + } + + @Test + void shouldResolveStandaloneMetricWithoutJudge() { + EvalMetric metric = MetricResolver.resolve("ToolSelectionAccuracy", null); + + assertThat(metric).isNotNull(); + assertThat(metric.name()).isEqualTo("ToolSelectionAccuracy"); + } + + @Test + void shouldThrowForUnknownMetric() { + assertThatThrownBy(() -> MetricResolver.resolve("NonexistentMetric", stubJudge)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown metric"); + } + + @Test + void shouldThrowWhenLlmMetricHasNoJudge() { + assertThatThrownBy(() -> MetricResolver.resolve("Faithfulness", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("requires a judge model"); + } + + @Test + void shouldResolveAllKnownLlmMetrics() { + String[] metricNames = { + "Faithfulness", "Correctness", "Hallucination", + "Toxicity", "Coherence", "Conciseness", "Bias", + "ContextualRelevancy", "ContextualPrecision", "ContextualRecall", + "TaskCompletion" + }; + for (String name : metricNames) { + EvalMetric metric = MetricResolver.resolve(name, stubJudge); + assertThat(metric).as("Metric: " + name).isNotNull(); + } + } + + @Test + void availableMetricsShouldReturnNonEmptyString() { + String available = MetricResolver.availableMetrics(); + + assertThat(available).contains("answerrelevancy"); + assertThat(available).contains("faithfulness"); + } +} diff --git a/agenteval-maven-plugin/src/test/java/com/agenteval/maven/ReportFormatResolverTest.java b/agenteval-maven-plugin/src/test/java/com/agenteval/maven/ReportFormatResolverTest.java new file mode 100644 index 0000000..8d7445e --- /dev/null +++ b/agenteval-maven-plugin/src/test/java/com/agenteval/maven/ReportFormatResolverTest.java @@ -0,0 +1,76 @@ +package com.agenteval.maven; + +import com.agenteval.reporting.ConsoleReporter; +import com.agenteval.reporting.EvalReporter; +import com.agenteval.reporting.HtmlReporter; +import com.agenteval.reporting.JsonReporter; +import com.agenteval.reporting.JunitXmlReporter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ReportFormatResolverTest { + + @Test + void shouldResolveConsoleFormat(@TempDir Path tempDir) { + List reporters = ReportFormatResolver.resolve("console", tempDir); + + assertThat(reporters).hasSize(1); + assertThat(reporters.getFirst()).isInstanceOf(ConsoleReporter.class); + } + + @Test + void shouldResolveJsonFormat(@TempDir Path tempDir) { + List reporters = ReportFormatResolver.resolve("json", tempDir); + + assertThat(reporters).hasSize(1); + assertThat(reporters.getFirst()).isInstanceOf(JsonReporter.class); + } + + @Test + void shouldResolveXmlFormat(@TempDir Path tempDir) { + List reporters = ReportFormatResolver.resolve("xml", tempDir); + + assertThat(reporters).hasSize(1); + assertThat(reporters.getFirst()).isInstanceOf(JunitXmlReporter.class); + } + + @Test + void shouldResolveHtmlFormat(@TempDir Path tempDir) { + List reporters = ReportFormatResolver.resolve("html", tempDir); + + assertThat(reporters).hasSize(1); + assertThat(reporters.getFirst()).isInstanceOf(HtmlReporter.class); + } + + @Test + void shouldResolveMultipleFormats(@TempDir Path tempDir) { + List reporters = ReportFormatResolver.resolve("console,json,xml", tempDir); + + assertThat(reporters).hasSize(3); + assertThat(reporters.get(0)).isInstanceOf(ConsoleReporter.class); + assertThat(reporters.get(1)).isInstanceOf(JsonReporter.class); + assertThat(reporters.get(2)).isInstanceOf(JunitXmlReporter.class); + } + + @Test + void shouldBeCaseInsensitive(@TempDir Path tempDir) { + List reporters = ReportFormatResolver.resolve("CONSOLE,JSON", tempDir); + + assertThat(reporters).hasSize(2); + assertThat(reporters.get(0)).isInstanceOf(ConsoleReporter.class); + assertThat(reporters.get(1)).isInstanceOf(JsonReporter.class); + } + + @Test + void shouldThrowForUnknownFormat(@TempDir Path tempDir) { + assertThatThrownBy(() -> ReportFormatResolver.resolve("pdf", tempDir)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown report format"); + } +} diff --git a/pom.xml b/pom.xml index 0e745f0..b8fc50b 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,8 @@ agenteval-langgraph4j agenteval-mcp agenteval-redteam + agenteval-maven-plugin + agenteval-github-actions diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml index 152a318..8fe1927 100644 --- a/spotbugs-exclude.xml +++ b/spotbugs-exclude.xml @@ -89,4 +89,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +