{@code
* var judge = JudgeModels.openai("gpt-4o");
* var judge = JudgeModels.anthropic("claude-sonnet-4-20250514");
+ * var judge = JudgeModels.google("gemini-1.5-pro");
+ * var judge = JudgeModels.azure(JudgeConfig.builder()
+ * .apiKey("...").model("my-deployment").baseUrl("https://myresource.openai.azure.com").build());
+ * var judge = JudgeModels.bedrock("anthropic.claude-3-sonnet-20240229-v1:0");
+ * var judge = JudgeModels.custom(JudgeConfig.builder()
+ * .model("my-model").baseUrl("http://localhost:8000").build());
* var judge = JudgeModels.ollama("llama3");
- * var judge = JudgeModels.openai(JudgeConfig.builder()
- * .apiKey("sk-...")
- * .model("gpt-4o")
- * .baseUrl("https://api.openai.com")
- * .build());
* }
*/
public final class JudgeModels {
private static final String OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
private static final String ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY";
+ private static final String GOOGLE_API_KEY_ENV = "GOOGLE_API_KEY";
private static final String OPENAI_BASE_URL = "https://api.openai.com";
private static final String ANTHROPIC_BASE_URL = "https://api.anthropic.com";
+ private static final String GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com";
+ private static final String BEDROCK_BASE_URL = "https://bedrock-runtime.us-east-1.amazonaws.com";
private static final String OLLAMA_BASE_URL = "http://localhost:11434";
private JudgeModels() {}
@@ -71,6 +79,76 @@ public static JudgeModel anthropic(JudgeConfig config) {
return new AnthropicJudgeModel(config);
}
+ /**
+ * Creates a Google Gemini judge model using the given model ID.
+ * API key is resolved from the {@code GOOGLE_API_KEY} environment variable.
+ */
+ public static JudgeModel google(String model) {
+ return google(JudgeConfig.builder()
+ .apiKey(resolveApiKey(GOOGLE_API_KEY_ENV, "Google"))
+ .model(model)
+ .baseUrl(GOOGLE_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates a Google Gemini judge model with full configuration.
+ */
+ public static JudgeModel google(JudgeConfig config) {
+ return new GoogleJudgeModel(config);
+ }
+
+ /**
+ * Creates an Azure OpenAI judge model using the given deployment name.
+ * API key is resolved from the {@code AZURE_OPENAI_API_KEY} environment variable.
+ * The base URL must point to your Azure OpenAI resource
+ * (e.g., {@code https://myresource.openai.azure.com}).
+ */
+ public static JudgeModel azure(JudgeConfig config) {
+ return new AzureOpenAiJudgeModel(config);
+ }
+
+ /**
+ * Creates an Azure OpenAI judge model with a specific API version.
+ */
+ public static JudgeModel azure(JudgeConfig config, String apiVersion) {
+ return new AzureOpenAiJudgeModel(config, apiVersion);
+ }
+
+ /**
+ * Creates an Amazon Bedrock judge model using the given model ID.
+ * AWS credentials are resolved from {@code AWS_ACCESS_KEY_ID} and
+ * {@code AWS_SECRET_ACCESS_KEY} environment variables.
+ *
+ * @param model the Bedrock model ID (e.g., {@code anthropic.claude-3-sonnet-20240229-v1:0})
+ */
+ public static JudgeModel bedrock(String model) {
+ return bedrock(JudgeConfig.builder()
+ .model(model)
+ .baseUrl(BEDROCK_BASE_URL)
+ .build());
+ }
+
+ /**
+ * Creates an Amazon Bedrock judge model with full configuration.
+ * The base URL determines the AWS region
+ * (e.g., {@code https://bedrock-runtime.eu-west-1.amazonaws.com}).
+ */
+ public static JudgeModel bedrock(JudgeConfig config) {
+ return new BedrockJudgeModel(config);
+ }
+
+ /**
+ * Creates a custom HTTP judge model for any OpenAI-compatible endpoint.
+ * No API key required — if one is set in the config, it will be sent
+ * as a {@code Bearer} token.
+ *
+ * Use this for vLLM, LiteLLM, LocalAI, or any OpenAI-compatible server.
+ */ + public static JudgeModel custom(JudgeConfig config) { + return new CustomHttpJudgeModel(config); + } + /** * Creates an Ollama judge model using the given model ID. * Defaults to {@code localhost:11434}. No API key required. diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/AzureOpenAiJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/AzureOpenAiJudgeModel.java new file mode 100644 index 0000000..ebdf8ce --- /dev/null +++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/AzureOpenAiJudgeModel.java @@ -0,0 +1,111 @@ +package com.agenteval.judge.provider; + +import com.agenteval.core.model.TokenUsage; +import com.agenteval.judge.JudgeException; +import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.http.HttpJudgeClient; +import com.agenteval.judge.http.HttpJudgeRequest; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.Map; + +/** + * Azure OpenAI judge model provider. + * + *Sends requests to Azure's OpenAI-compatible endpoint at + * {@code {baseUrl}/openai/deployments/{model}/chat/completions?api-version=...} + * with {@code api-key} header authentication.
+ * + *The {@link JudgeConfig#getModel()} value is used as the deployment name. + * The API version defaults to {@code 2024-02-01} but can be customized + * by appending it to the base URL.
+ */ +public final class AzureOpenAiJudgeModel extends AbstractHttpJudgeModel { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String DEFAULT_API_VERSION = "2024-02-01"; + private static final String COMPLETIONS_PATH = + "/openai/deployments/%s/chat/completions?api-version=%s"; + private static final String SYSTEM_PROMPT = + "You are an evaluation judge. Respond ONLY with a JSON object " + + "containing \"score\" (a number between 0.0 and 1.0) " + + "and \"reason\" (a brief explanation)."; + + private final String apiVersion; + + public AzureOpenAiJudgeModel(JudgeConfig config) { + this(config, DEFAULT_API_VERSION); + } + + public AzureOpenAiJudgeModel(JudgeConfig config, String apiVersion) { + super(config); + if (config.getApiKey() == null || config.getApiKey().isBlank()) { + throw new JudgeException("Azure OpenAI requires a non-null API key"); + } + this.apiVersion = apiVersion != null ? apiVersion : DEFAULT_API_VERSION; + } + + AzureOpenAiJudgeModel(JudgeConfig config, String apiVersion, HttpJudgeClient client) { + super(config, client); + this.apiVersion = apiVersion != null ? apiVersion : DEFAULT_API_VERSION; + } + + String getApiVersion() { + return apiVersion; + } + + @Override + protected HttpJudgeRequest buildRequest(String prompt) { + try { + var body = MAPPER.createObjectNode(); + body.put("temperature", config.getTemperature()); + + var responseFormat = MAPPER.createObjectNode(); + responseFormat.put("type", "json_object"); + body.set("response_format", responseFormat); + + var messages = body.putArray("messages"); + + var systemMsg = messages.addObject(); + systemMsg.put("role", "system"); + systemMsg.put("content", SYSTEM_PROMPT); + + var userMsg = messages.addObject(); + userMsg.put("role", "user"); + userMsg.put("content", prompt); + + String url = config.getBaseUrl() + + String.format(COMPLETIONS_PATH, config.getModel(), apiVersion); + return new HttpJudgeRequest( + url, + Map.of("api-key", config.getApiKey()), + MAPPER.writeValueAsString(body)); + } catch (Exception e) { + throw new JudgeException("Failed to build Azure OpenAI request", e); + } + } + + @Override + protected String extractContent(String responseBody) { + JsonNode root = parseJson(responseBody); + JsonNode choices = root.path("choices"); + if (choices.isEmpty()) { + throw new JudgeException("No choices in Azure OpenAI response"); + } + return choices.get(0).path("message").path("content").asText(""); + } + + @Override + protected TokenUsage extractTokenUsage(String responseBody) { + JsonNode root = parseJson(responseBody); + JsonNode usage = root.path("usage"); + if (usage.isMissingNode()) { + return null; + } + return new TokenUsage( + usage.path("prompt_tokens").asInt(0), + usage.path("completion_tokens").asInt(0), + usage.path("total_tokens").asInt(0)); + } +} diff --git a/agenteval-judge/src/main/java/com/agenteval/judge/provider/BedrockJudgeModel.java b/agenteval-judge/src/main/java/com/agenteval/judge/provider/BedrockJudgeModel.java new file mode 100644 index 0000000..d86c660 --- /dev/null +++ b/agenteval-judge/src/main/java/com/agenteval/judge/provider/BedrockJudgeModel.java @@ -0,0 +1,262 @@ +package com.agenteval.judge.provider; + +import com.agenteval.core.model.TokenUsage; +import com.agenteval.judge.JudgeException; +import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.http.HttpJudgeClient; +import com.agenteval.judge.http.HttpJudgeRequest; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.TreeMap; + +/** + * Amazon Bedrock judge model provider. + * + *Sends requests to the Bedrock Runtime {@code InvokeModel} API using + * the Anthropic Messages API format (for Claude models on Bedrock). + * Authentication uses AWS Signature Version 4.
+ * + *Required environment variables: {@code AWS_ACCESS_KEY_ID}, + * {@code AWS_SECRET_ACCESS_KEY}, and optionally {@code AWS_SESSION_TOKEN}. + * The region is extracted from the base URL + * ({@code https://bedrock-runtime.{region}.amazonaws.com}).
+ */ +public final class BedrockJudgeModel extends AbstractHttpJudgeModel { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String DEFAULT_BASE_URL = "https://bedrock-runtime.us-east-1.amazonaws.com"; + private static final String INVOKE_PATH = "/model/%s/invoke"; + private static final String SERVICE = "bedrock"; + private static final String ALGORITHM = "AWS4-HMAC-SHA256"; + private static final String SYSTEM_PROMPT = + "You are an evaluation judge. Respond ONLY with a JSON object " + + "containing \"score\" (a number between 0.0 and 1.0) " + + "and \"reason\" (a brief explanation)."; + + private static final DateTimeFormatter DATE_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneOffset.UTC); + private static final DateTimeFormatter DATETIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC); + + private final String accessKeyId; + private final String secretAccessKey; + private final String sessionToken; + private final String region; + + public BedrockJudgeModel(JudgeConfig config) { + this(config, + resolveEnv("AWS_ACCESS_KEY_ID", "Bedrock"), + resolveEnv("AWS_SECRET_ACCESS_KEY", "Bedrock"), + System.getenv("AWS_SESSION_TOKEN")); + } + + public BedrockJudgeModel(JudgeConfig config, + String accessKeyId, String secretAccessKey, + String sessionToken) { + super(config); + if (accessKeyId == null || accessKeyId.isBlank()) { + throw new JudgeException("Bedrock requires AWS_ACCESS_KEY_ID"); + } + if (secretAccessKey == null || secretAccessKey.isBlank()) { + throw new JudgeException("Bedrock requires AWS_SECRET_ACCESS_KEY"); + } + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + this.sessionToken = sessionToken; + this.region = extractRegion(config.getBaseUrl()); + } + + BedrockJudgeModel(JudgeConfig config, + String accessKeyId, String secretAccessKey, + String sessionToken, + HttpJudgeClient client) { + super(config, client); + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + this.sessionToken = sessionToken; + this.region = extractRegion(config.getBaseUrl()); + } + + static String defaultBaseUrl() { + return DEFAULT_BASE_URL; + } + + @Override + protected HttpJudgeRequest buildRequest(String prompt) { + try { + var body = MAPPER.createObjectNode(); + body.put("anthropic_version", "bedrock-2023-05-31"); + body.put("max_tokens", 1024); + body.put("temperature", config.getTemperature()); + body.put("system", SYSTEM_PROMPT); + + var messages = body.putArray("messages"); + var userMsg = messages.addObject(); + userMsg.put("role", "user"); + userMsg.put("content", prompt); + + String url = config.getBaseUrl() + + String.format(INVOKE_PATH, config.getModel()); + String payload = MAPPER.writeValueAsString(body); + + MapPoints to any self-hosted or third-party API that exposes + * the OpenAI chat completions format. Supports optional API key + * authentication via {@code Authorization: Bearer} header.
+ * + *Use this provider for vLLM, LiteLLM, LocalAI, or any other + * OpenAI-compatible inference server.
+ * + *{@code
+ * var judge = JudgeModels.custom(JudgeConfig.builder()
+ * .model("my-model")
+ * .baseUrl("http://localhost:8000")
+ * .build());
+ * }
+ */
+public final class CustomHttpJudgeModel extends AbstractHttpJudgeModel {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final String COMPLETIONS_PATH = "/v1/chat/completions";
+ private static final String SYSTEM_PROMPT =
+ "You are an evaluation judge. Respond ONLY with a JSON object "
+ + "containing \"score\" (a number between 0.0 and 1.0) "
+ + "and \"reason\" (a brief explanation).";
+
+ public CustomHttpJudgeModel(JudgeConfig config) {
+ super(config);
+ }
+
+ CustomHttpJudgeModel(JudgeConfig config, HttpJudgeClient client) {
+ super(config, client);
+ }
+
+ @Override
+ protected HttpJudgeRequest buildRequest(String prompt) {
+ try {
+ var body = MAPPER.createObjectNode();
+ body.put("model", config.getModel());
+ body.put("temperature", config.getTemperature());
+
+ var responseFormat = MAPPER.createObjectNode();
+ responseFormat.put("type", "json_object");
+ body.set("response_format", responseFormat);
+
+ var messages = body.putArray("messages");
+
+ var systemMsg = messages.addObject();
+ systemMsg.put("role", "system");
+ systemMsg.put("content", SYSTEM_PROMPT);
+
+ var userMsg = messages.addObject();
+ userMsg.put("role", "user");
+ userMsg.put("content", prompt);
+
+ String url = config.getBaseUrl() + COMPLETIONS_PATH;
+ MapSends requests to the Gemini {@code generateContent} API with + * JSON response format via {@code responseMimeType}.
+ */ +public final class GoogleJudgeModel extends AbstractHttpJudgeModel { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com"; + private static final String GENERATE_PATH = "/v1beta/models/%s:generateContent"; + private static final String SYSTEM_PROMPT = + "You are an evaluation judge. Respond ONLY with a JSON object " + + "containing \"score\" (a number between 0.0 and 1.0) " + + "and \"reason\" (a brief explanation)."; + + public GoogleJudgeModel(JudgeConfig config) { + super(config); + if (config.getApiKey() == null || config.getApiKey().isBlank()) { + throw new JudgeException("Google requires a non-null API key"); + } + } + + GoogleJudgeModel(JudgeConfig config, HttpJudgeClient client) { + super(config, client); + } + + static String defaultBaseUrl() { + return DEFAULT_BASE_URL; + } + + @Override + protected HttpJudgeRequest buildRequest(String prompt) { + try { + var body = MAPPER.createObjectNode(); + + var systemInstruction = MAPPER.createObjectNode(); + var systemParts = systemInstruction.putArray("parts"); + systemParts.addObject().put("text", SYSTEM_PROMPT); + body.set("systemInstruction", systemInstruction); + + var contents = body.putArray("contents"); + var userContent = contents.addObject(); + userContent.put("role", "user"); + var parts = userContent.putArray("parts"); + parts.addObject().put("text", prompt); + + var generationConfig = MAPPER.createObjectNode(); + generationConfig.put("temperature", config.getTemperature()); + generationConfig.put("responseMimeType", "application/json"); + body.set("generationConfig", generationConfig); + + String url = config.getBaseUrl() + + String.format(GENERATE_PATH, config.getModel()); + return new HttpJudgeRequest( + url, + Map.of("x-goog-api-key", config.getApiKey()), + MAPPER.writeValueAsString(body)); + } catch (Exception e) { + throw new JudgeException("Failed to build Google request", e); + } + } + + @Override + protected String extractContent(String responseBody) { + JsonNode root = parseJson(responseBody); + JsonNode candidates = root.path("candidates"); + if (candidates.isEmpty()) { + throw new JudgeException("No candidates in Google response"); + } + return candidates.get(0) + .path("content").path("parts").path(0) + .path("text").asText(""); + } + + @Override + protected TokenUsage extractTokenUsage(String responseBody) { + JsonNode root = parseJson(responseBody); + JsonNode usage = root.path("usageMetadata"); + if (usage.isMissingNode()) { + return null; + } + int promptTokens = usage.path("promptTokenCount").asInt(0); + int candidateTokens = usage.path("candidatesTokenCount").asInt(0); + int totalTokens = usage.path("totalTokenCount").asInt(0); + if (totalTokens == 0) { + totalTokens = promptTokens + candidateTokens; + } + return new TokenUsage(promptTokens, candidateTokens, totalTokens); + } +} diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/JudgeModelsTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/JudgeModelsTest.java index 0e8240e..e3121df 100644 --- a/agenteval-judge/src/test/java/com/agenteval/judge/JudgeModelsTest.java +++ b/agenteval-judge/src/test/java/com/agenteval/judge/JudgeModelsTest.java @@ -1,6 +1,9 @@ package com.agenteval.judge; import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.provider.AzureOpenAiJudgeModel; +import com.agenteval.judge.provider.CustomHttpJudgeModel; +import com.agenteval.judge.provider.GoogleJudgeModel; import com.agenteval.judge.provider.OpenAiJudgeModel; import com.agenteval.judge.provider.AnthropicJudgeModel; import org.junit.jupiter.api.Test; @@ -38,6 +41,47 @@ void anthropicWithConfigShouldCreateAnthropicModel() { assertThat(model.modelId()).isEqualTo("claude-sonnet-4-20250514"); } + @Test + void googleWithConfigShouldCreateGoogleModel() { + var config = JudgeConfig.builder() + .apiKey("test-key") + .model("gemini-1.5-pro") + .baseUrl("https://generativelanguage.googleapis.com") + .build(); + + var model = JudgeModels.google(config); + + assertThat(model).isInstanceOf(GoogleJudgeModel.class); + assertThat(model.modelId()).isEqualTo("gemini-1.5-pro"); + } + + @Test + void azureWithConfigShouldCreateAzureModel() { + var config = JudgeConfig.builder() + .apiKey("azure-key") + .model("my-deployment") + .baseUrl("https://myresource.openai.azure.com") + .build(); + + var model = JudgeModels.azure(config); + + assertThat(model).isInstanceOf(AzureOpenAiJudgeModel.class); + assertThat(model.modelId()).isEqualTo("my-deployment"); + } + + @Test + void customWithConfigShouldCreateCustomModel() { + var config = JudgeConfig.builder() + .model("local-model") + .baseUrl("http://localhost:8000") + .build(); + + var model = JudgeModels.custom(config); + + assertThat(model).isInstanceOf(CustomHttpJudgeModel.class); + assertThat(model.modelId()).isEqualTo("local-model"); + } + @Test void openaiWithoutEnvKeyShouldThrow() { assertThatThrownBy(() -> JudgeModels.openai("gpt-4o")) @@ -51,4 +95,17 @@ void anthropicWithoutEnvKeyShouldThrow() { .isInstanceOf(JudgeException.class) .hasMessageContaining("ANTHROPIC_API_KEY"); } + + @Test + void googleWithoutEnvKeyShouldThrow() { + assertThatThrownBy(() -> JudgeModels.google("gemini-1.5-pro")) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("GOOGLE_API_KEY"); + } + + @Test + void bedrockWithoutEnvKeyShouldThrow() { + assertThatThrownBy(() -> JudgeModels.bedrock("anthropic.claude-3-sonnet")) + .isInstanceOf(JudgeException.class); + } } diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/provider/AzureOpenAiJudgeModelTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/provider/AzureOpenAiJudgeModelTest.java new file mode 100644 index 0000000..d9a3b00 --- /dev/null +++ b/agenteval-judge/src/test/java/com/agenteval/judge/provider/AzureOpenAiJudgeModelTest.java @@ -0,0 +1,139 @@ +package com.agenteval.judge.provider; + +import com.agenteval.core.judge.JudgeResponse; +import com.agenteval.judge.JudgeException; +import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.http.HttpJudgeClient; +import com.agenteval.judge.http.HttpJudgeRequest; +import com.agenteval.judge.http.HttpJudgeResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class AzureOpenAiJudgeModelTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private JudgeConfig config; + private HttpJudgeClient mockClient; + + @BeforeEach + void setUp() { + config = JudgeConfig.builder() + .apiKey("azure-test-key") + .model("my-gpt4-deployment") + .baseUrl("https://myresource.openai.azure.com") + .build(); + mockClient = mock(HttpJudgeClient.class); + } + + @Test + void shouldReturnModelId() { + var model = new AzureOpenAiJudgeModel(config, "2024-02-01", mockClient); + assertThat(model.modelId()).isEqualTo("my-gpt4-deployment"); + } + + @Test + void shouldBuildValidRequest() { + var model = new AzureOpenAiJudgeModel(config, "2024-02-01", mockClient); + + when(mockClient.send(any())).thenAnswer(invocation -> { + HttpJudgeRequest req = invocation.getArgument(0); + assertThat(req.url()).isEqualTo( + "https://myresource.openai.azure.com/openai/deployments/" + + "my-gpt4-deployment/chat/completions?api-version=2024-02-01"); + assertThat(req.headers()).containsEntry("api-key", "azure-test-key"); + assertThat(req.headers()).doesNotContainKey("Authorization"); + + JsonNode body = MAPPER.readTree(req.body()); + assertThat(body.has("model")).isFalse(); + assertThat(body.get("response_format").get("type").asText()).isEqualTo("json_object"); + assertThat(body.get("messages")).hasSize(2); + + return makeResponse(0.85, "Relevant", 100, 50, 150); + }); + + model.judge("Evaluate this"); + } + + @Test + void shouldParseSuccessResponse() { + var model = new AzureOpenAiJudgeModel(config, "2024-02-01", mockClient); + when(mockClient.send(any())).thenReturn( + makeResponse(0.85, "Good answer", 100, 50, 150)); + + JudgeResponse response = model.judge("test prompt"); + + assertThat(response.score()).isCloseTo(0.85, within(0.001)); + assertThat(response.reason()).isEqualTo("Good answer"); + assertThat(response.tokenUsage()).isNotNull(); + assertThat(response.tokenUsage().inputTokens()).isEqualTo(100); + assertThat(response.tokenUsage().outputTokens()).isEqualTo(50); + } + + @Test + void shouldUseDefaultApiVersion() { + var model = new AzureOpenAiJudgeModel(config, null, mockClient); + assertThat(model.getApiVersion()).isEqualTo("2024-02-01"); + } + + @Test + void shouldThrowOnEmptyChoices() { + var model = new AzureOpenAiJudgeModel(config, "2024-02-01", mockClient); + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, "{\"choices\": []}", null)); + + assertThatThrownBy(() -> model.judge("test")) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("No choices"); + } + + @Test + void shouldThrowOnMissingApiKey() { + var noKeyConfig = JudgeConfig.builder() + .model("my-deployment") + .baseUrl("https://myresource.openai.azure.com") + .build(); + + assertThatThrownBy(() -> new AzureOpenAiJudgeModel(noKeyConfig)) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("Azure OpenAI requires"); + } + + @Test + void shouldHandleResponseWithoutUsage() { + var model = new AzureOpenAiJudgeModel(config, "2024-02-01", mockClient); + String responseBody = """ + {"choices": [{"message": {"content": "{\\"score\\": 0.7, \\"reason\\": \\"OK\\"}"}}]}"""; + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, responseBody, null)); + + JudgeResponse response = model.judge("test"); + assertThat(response.score()).isCloseTo(0.7, within(0.001)); + assertThat(response.tokenUsage()).isNull(); + } + + private HttpJudgeResponse makeResponse( + double score, String reason, int promptTokens, int completionTokens, int totalTokens) { + String content = String.format( + "{\"score\": %s, \"reason\": \"%s\"}", score, reason); + String body = String.format(""" + { + "choices": [{"message": {"content": %s}}], + "usage": { + "prompt_tokens": %d, + "completion_tokens": %d, + "total_tokens": %d + } + }""", + MAPPER.valueToTree(content), promptTokens, completionTokens, totalTokens); + return new HttpJudgeResponse(200, body, null); + } +} diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/provider/BedrockJudgeModelTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/provider/BedrockJudgeModelTest.java new file mode 100644 index 0000000..bd517de --- /dev/null +++ b/agenteval-judge/src/test/java/com/agenteval/judge/provider/BedrockJudgeModelTest.java @@ -0,0 +1,161 @@ +package com.agenteval.judge.provider; + +import com.agenteval.core.judge.JudgeResponse; +import com.agenteval.judge.JudgeException; +import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.http.HttpJudgeClient; +import com.agenteval.judge.http.HttpJudgeRequest; +import com.agenteval.judge.http.HttpJudgeResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class BedrockJudgeModelTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private JudgeConfig config; + private HttpJudgeClient mockClient; + + @BeforeEach + void setUp() { + config = JudgeConfig.builder() + .model("anthropic.claude-3-sonnet-20240229-v1:0") + .baseUrl("https://bedrock-runtime.us-east-1.amazonaws.com") + .build(); + mockClient = mock(HttpJudgeClient.class); + } + + @Test + void shouldReturnModelId() { + var model = new BedrockJudgeModel(config, "AKID", "secret", null, mockClient); + assertThat(model.modelId()).isEqualTo("anthropic.claude-3-sonnet-20240229-v1:0"); + } + + @Test + void shouldBuildValidRequest() { + var model = new BedrockJudgeModel(config, "AKID", "secret", null, mockClient); + + when(mockClient.send(any())).thenAnswer(invocation -> { + HttpJudgeRequest req = invocation.getArgument(0); + assertThat(req.url()).isEqualTo( + "https://bedrock-runtime.us-east-1.amazonaws.com" + + "/model/anthropic.claude-3-sonnet-20240229-v1:0/invoke"); + assertThat(req.headers()).containsKey("Authorization"); + assertThat(req.headers().get("Authorization")).startsWith("AWS4-HMAC-SHA256"); + assertThat(req.headers()).containsKey("x-amz-date"); + assertThat(req.headers()).containsKey("x-amz-content-sha256"); + + JsonNode body = MAPPER.readTree(req.body()); + assertThat(body.get("anthropic_version").asText()).isEqualTo("bedrock-2023-05-31"); + assertThat(body.get("max_tokens").asInt()).isEqualTo(1024); + assertThat(body.has("system")).isTrue(); + assertThat(body.get("messages")).hasSize(1); + + return makeResponse(0.9, "Excellent"); + }); + + model.judge("Evaluate this"); + } + + @Test + void shouldParseSuccessResponse() { + var model = new BedrockJudgeModel(config, "AKID", "secret", null, mockClient); + when(mockClient.send(any())).thenReturn(makeResponse(0.9, "Excellent")); + + JudgeResponse response = model.judge("test prompt"); + + assertThat(response.score()).isCloseTo(0.9, within(0.001)); + assertThat(response.reason()).isEqualTo("Excellent"); + assertThat(response.tokenUsage()).isNotNull(); + assertThat(response.tokenUsage().inputTokens()).isEqualTo(150); + assertThat(response.tokenUsage().outputTokens()).isEqualTo(60); + } + + @Test + void shouldIncludeSessionToken() { + var model = new BedrockJudgeModel(config, "AKID", "secret", "token123", mockClient); + + when(mockClient.send(any())).thenAnswer(invocation -> { + HttpJudgeRequest req = invocation.getArgument(0); + assertThat(req.headers()).containsEntry("x-amz-security-token", "token123"); + return makeResponse(0.8, "OK"); + }); + + model.judge("test"); + } + + @Test + void shouldExtractRegionFromBaseUrl() { + assertThat(BedrockJudgeModel.extractRegion( + "https://bedrock-runtime.eu-west-1.amazonaws.com")) + .isEqualTo("eu-west-1"); + assertThat(BedrockJudgeModel.extractRegion( + "https://bedrock-runtime.ap-southeast-1.amazonaws.com")) + .isEqualTo("ap-southeast-1"); + } + + @Test + void shouldDefaultRegionOnInvalidUrl() { + assertThat(BedrockJudgeModel.extractRegion("http://localhost")) + .isEqualTo("us-east-1"); + } + + @Test + void shouldThrowOnEmptyContent() { + var model = new BedrockJudgeModel(config, "AKID", "secret", null, mockClient); + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, "{\"content\": []}", null)); + + assertThatThrownBy(() -> model.judge("test")) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("No content"); + } + + @Test + void shouldThrowOnMissingAccessKey() { + assertThatThrownBy(() -> + new BedrockJudgeModel(config, null, "secret", null)) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("AWS_ACCESS_KEY_ID"); + } + + @Test + void shouldThrowOnMissingSecretKey() { + assertThatThrownBy(() -> + new BedrockJudgeModel(config, "AKID", null, null)) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("AWS_SECRET_ACCESS_KEY"); + } + + @Test + void shouldHandleResponseWithoutUsage() { + var model = new BedrockJudgeModel(config, "AKID", "secret", null, mockClient); + String responseBody = """ + {"content": [{"type": "text", "text": "{\\"score\\": 0.7, \\"reason\\": \\"OK\\"}"}]}"""; + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, responseBody, null)); + + JudgeResponse response = model.judge("test"); + assertThat(response.score()).isCloseTo(0.7, within(0.001)); + assertThat(response.tokenUsage()).isNull(); + } + + private HttpJudgeResponse makeResponse(double score, String reason) { + String content = String.format( + "{\"score\": %s, \"reason\": \"%s\"}", score, reason); + String body = String.format(""" + { + "content": [{"type": "text", "text": %s}], + "usage": {"input_tokens": 150, "output_tokens": 60} + }""", MAPPER.valueToTree(content)); + return new HttpJudgeResponse(200, body, null); + } +} diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/provider/CustomHttpJudgeModelTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/provider/CustomHttpJudgeModelTest.java new file mode 100644 index 0000000..660ec4f --- /dev/null +++ b/agenteval-judge/src/test/java/com/agenteval/judge/provider/CustomHttpJudgeModelTest.java @@ -0,0 +1,148 @@ +package com.agenteval.judge.provider; + +import com.agenteval.core.judge.JudgeResponse; +import com.agenteval.judge.JudgeException; +import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.http.HttpJudgeClient; +import com.agenteval.judge.http.HttpJudgeRequest; +import com.agenteval.judge.http.HttpJudgeResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class CustomHttpJudgeModelTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private HttpJudgeClient mockClient; + + @BeforeEach + void setUp() { + mockClient = mock(HttpJudgeClient.class); + } + + @Test + void shouldReturnModelId() { + var config = JudgeConfig.builder() + .model("my-custom-model") + .baseUrl("http://localhost:8000") + .build(); + var model = new CustomHttpJudgeModel(config, mockClient); + assertThat(model.modelId()).isEqualTo("my-custom-model"); + } + + @Test + void shouldBuildRequestWithApiKey() { + var config = JudgeConfig.builder() + .apiKey("custom-key") + .model("my-model") + .baseUrl("http://localhost:8000") + .build(); + var model = new CustomHttpJudgeModel(config, mockClient); + + when(mockClient.send(any())).thenAnswer(invocation -> { + HttpJudgeRequest req = invocation.getArgument(0); + assertThat(req.url()).isEqualTo("http://localhost:8000/v1/chat/completions"); + assertThat(req.headers()).containsEntry("Authorization", "Bearer custom-key"); + + JsonNode body = MAPPER.readTree(req.body()); + assertThat(body.get("model").asText()).isEqualTo("my-model"); + assertThat(body.get("response_format").get("type").asText()).isEqualTo("json_object"); + assertThat(body.get("messages")).hasSize(2); + + return makeResponse(0.85, "Relevant", 100, 50, 150); + }); + + model.judge("Evaluate this"); + } + + @Test + void shouldBuildRequestWithoutApiKey() { + var config = JudgeConfig.builder() + .model("local-model") + .baseUrl("http://localhost:8000") + .build(); + var model = new CustomHttpJudgeModel(config, mockClient); + + when(mockClient.send(any())).thenAnswer(invocation -> { + HttpJudgeRequest req = invocation.getArgument(0); + assertThat(req.headers()).doesNotContainKey("Authorization"); + return makeResponse(0.8, "OK", 50, 30, 80); + }); + + model.judge("test"); + } + + @Test + void shouldParseSuccessResponse() { + var config = JudgeConfig.builder() + .model("my-model") + .baseUrl("http://localhost:8000") + .build(); + var model = new CustomHttpJudgeModel(config, mockClient); + when(mockClient.send(any())).thenReturn( + makeResponse(0.85, "Good answer", 100, 50, 150)); + + JudgeResponse response = model.judge("test prompt"); + + assertThat(response.score()).isCloseTo(0.85, within(0.001)); + assertThat(response.reason()).isEqualTo("Good answer"); + assertThat(response.tokenUsage()).isNotNull(); + } + + @Test + void shouldThrowOnEmptyChoices() { + var config = JudgeConfig.builder() + .model("my-model") + .baseUrl("http://localhost:8000") + .build(); + var model = new CustomHttpJudgeModel(config, mockClient); + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, "{\"choices\": []}", null)); + + assertThatThrownBy(() -> model.judge("test")) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("No choices"); + } + + @Test + void shouldHandleResponseWithoutUsage() { + var config = JudgeConfig.builder() + .model("my-model") + .baseUrl("http://localhost:8000") + .build(); + var model = new CustomHttpJudgeModel(config, mockClient); + String responseBody = """ + {"choices": [{"message": {"content": "{\\"score\\": 0.7, \\"reason\\": \\"OK\\"}"}}]}"""; + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, responseBody, null)); + + JudgeResponse response = model.judge("test"); + assertThat(response.score()).isCloseTo(0.7, within(0.001)); + assertThat(response.tokenUsage()).isNull(); + } + + private HttpJudgeResponse makeResponse( + double score, String reason, int promptTokens, int completionTokens, int totalTokens) { + String content = String.format( + "{\"score\": %s, \"reason\": \"%s\"}", score, reason); + String body = String.format(""" + { + "choices": [{"message": {"content": %s}}], + "usage": { + "prompt_tokens": %d, + "completion_tokens": %d, + "total_tokens": %d + } + }""", + MAPPER.valueToTree(content), promptTokens, completionTokens, totalTokens); + return new HttpJudgeResponse(200, body, null); + } +} diff --git a/agenteval-judge/src/test/java/com/agenteval/judge/provider/GoogleJudgeModelTest.java b/agenteval-judge/src/test/java/com/agenteval/judge/provider/GoogleJudgeModelTest.java new file mode 100644 index 0000000..65239a7 --- /dev/null +++ b/agenteval-judge/src/test/java/com/agenteval/judge/provider/GoogleJudgeModelTest.java @@ -0,0 +1,131 @@ +package com.agenteval.judge.provider; + +import com.agenteval.core.judge.JudgeResponse; +import com.agenteval.judge.JudgeException; +import com.agenteval.judge.config.JudgeConfig; +import com.agenteval.judge.http.HttpJudgeClient; +import com.agenteval.judge.http.HttpJudgeRequest; +import com.agenteval.judge.http.HttpJudgeResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class GoogleJudgeModelTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private JudgeConfig config; + private HttpJudgeClient mockClient; + + @BeforeEach + void setUp() { + config = JudgeConfig.builder() + .apiKey("test-google-key") + .model("gemini-1.5-pro") + .baseUrl("https://generativelanguage.googleapis.com") + .build(); + mockClient = mock(HttpJudgeClient.class); + } + + @Test + void shouldReturnModelId() { + var model = new GoogleJudgeModel(config, mockClient); + assertThat(model.modelId()).isEqualTo("gemini-1.5-pro"); + } + + @Test + void shouldBuildValidRequest() { + var model = new GoogleJudgeModel(config, mockClient); + + when(mockClient.send(any())).thenAnswer(invocation -> { + HttpJudgeRequest req = invocation.getArgument(0); + assertThat(req.url()).isEqualTo( + "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-pro:generateContent"); + assertThat(req.headers()).containsEntry("x-goog-api-key", "test-google-key"); + + JsonNode body = MAPPER.readTree(req.body()); + assertThat(body.has("systemInstruction")).isTrue(); + assertThat(body.path("systemInstruction").path("parts").get(0) + .path("text").asText()).contains("evaluation judge"); + assertThat(body.path("contents").get(0).path("role").asText()).isEqualTo("user"); + assertThat(body.path("generationConfig").path("responseMimeType").asText()) + .isEqualTo("application/json"); + + return makeResponse(0.85, "Relevant"); + }); + + model.judge("Evaluate this"); + } + + @Test + void shouldParseSuccessResponse() { + var model = new GoogleJudgeModel(config, mockClient); + when(mockClient.send(any())).thenReturn(makeResponse(0.85, "Good answer")); + + JudgeResponse response = model.judge("test prompt"); + + assertThat(response.score()).isCloseTo(0.85, within(0.001)); + assertThat(response.reason()).isEqualTo("Good answer"); + assertThat(response.tokenUsage()).isNotNull(); + assertThat(response.tokenUsage().inputTokens()).isEqualTo(120); + assertThat(response.tokenUsage().outputTokens()).isEqualTo(45); + } + + @Test + void shouldHandleResponseWithoutUsage() { + var model = new GoogleJudgeModel(config, mockClient); + String responseBody = """ + {"candidates": [{"content": {"parts": [{"text": "{\\"score\\": 0.7, \\"reason\\": \\"OK\\"}"}]}}]}"""; + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, responseBody, null)); + + JudgeResponse response = model.judge("test"); + assertThat(response.score()).isCloseTo(0.7, within(0.001)); + assertThat(response.tokenUsage()).isNull(); + } + + @Test + void shouldThrowOnEmptyCandidates() { + var model = new GoogleJudgeModel(config, mockClient); + when(mockClient.send(any())).thenReturn( + new HttpJudgeResponse(200, "{\"candidates\": []}", null)); + + assertThatThrownBy(() -> model.judge("test")) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("No candidates"); + } + + @Test + void shouldThrowOnMissingApiKey() { + var noKeyConfig = JudgeConfig.builder() + .model("gemini-1.5-pro") + .baseUrl("https://generativelanguage.googleapis.com") + .build(); + + assertThatThrownBy(() -> new GoogleJudgeModel(noKeyConfig)) + .isInstanceOf(JudgeException.class) + .hasMessageContaining("Google requires"); + } + + private HttpJudgeResponse makeResponse(double score, String reason) { + String content = String.format( + "{\"score\": %s, \"reason\": \"%s\"}", score, reason); + String body = String.format(""" + { + "candidates": [{"content": {"parts": [{"text": %s}]}}], + "usageMetadata": { + "promptTokenCount": 120, + "candidatesTokenCount": 45, + "totalTokenCount": 165 + } + }""", MAPPER.valueToTree(content)); + return new HttpJudgeResponse(200, body, null); + } +} diff --git a/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/TrajectoryOptimalityMetric.java b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/TrajectoryOptimalityMetric.java new file mode 100644 index 0000000..e694ac3 --- /dev/null +++ b/agenteval-metrics/src/main/java/com/agenteval/metrics/agent/TrajectoryOptimalityMetric.java @@ -0,0 +1,88 @@ +package com.agenteval.metrics.agent; + +import com.agenteval.core.judge.JudgeModel; +import com.agenteval.core.model.AgentTestCase; +import com.agenteval.metrics.llm.LLMJudgeMetric; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Evaluates whether the agent took the most efficient path to complete its task. + * + *Penalizes unnecessary tool calls, redundant LLM invocations, and + * circular reasoning. When {@code maxSteps} is configured, step counts + * exceeding the limit are penalized by the judge.
+ */ +public final class TrajectoryOptimalityMetric extends LLMJudgeMetric { + + private static final String NAME = "TrajectoryOptimality"; + private static final String PROMPT_PATH = + "com/agenteval/metrics/prompts/trajectory-optimality.txt"; + private static final double DEFAULT_THRESHOLD = 0.7; + private static final int DEFAULT_MAX_STEPS = -1; + + private final int maxSteps; + + public TrajectoryOptimalityMetric(JudgeModel judge) { + this(judge, DEFAULT_THRESHOLD, DEFAULT_MAX_STEPS); + } + + public TrajectoryOptimalityMetric(JudgeModel judge, double threshold) { + this(judge, threshold, DEFAULT_MAX_STEPS); + } + + public TrajectoryOptimalityMetric(JudgeModel judge, double threshold, int maxSteps) { + super(judge, threshold, PROMPT_PATH); + this.maxSteps = maxSteps; + } + + @Override + public String name() { + return NAME; + } + + @Override + protected void validate(AgentTestCase testCase) { + if (testCase.getInput() == null || testCase.getInput().isBlank()) { + throw new IllegalArgumentException(NAME + " requires non-empty input"); + } + if (testCase.getReasoningTrace().isEmpty() && testCase.getToolCalls().isEmpty()) { + throw new IllegalArgumentException( + NAME + " requires non-empty reasoningTrace or toolCalls"); + } + } + + @Override + protected Map