From 85253ba8967af592a6465af4981b1e177bca6dbc Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 18 Jun 2026 19:23:45 +0000 Subject: [PATCH 1/2] feat(evaluation): expose evaluationReferenceInputs on EvaluatorInput Code-based evaluators now receive ground-truth reference inputs in the Lambda event's evaluationReferenceInputs field, but EvaluatorInput dropped it. Add a typed ReferenceInput model and a reference_inputs list on EvaluatorInput, populated by the @custom_code_based_evaluator decorator. Backward compatible: existing evaluators ignore the new field. Closes #539 --- src/bedrock_agentcore/evaluation/__init__.py | 2 + .../custom_code_based_evaluators/__init__.py | 7 ++- .../custom_code_based_evaluators/decorator.py | 1 + .../custom_code_based_evaluators/models.py | 34 ++++++++++- .../test_decorator.py | 33 +++++++++++ .../test_models.py | 56 ++++++++++++++++++- 6 files changed, 129 insertions(+), 4 deletions(-) diff --git a/src/bedrock_agentcore/evaluation/__init__.py b/src/bedrock_agentcore/evaluation/__init__.py index a70c267f..ffa88401 100644 --- a/src/bedrock_agentcore/evaluation/__init__.py +++ b/src/bedrock_agentcore/evaluation/__init__.py @@ -4,6 +4,7 @@ from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( EvaluatorInput, EvaluatorOutput, + ReferenceInput, custom_code_based_evaluator, ) from bedrock_agentcore.evaluation.dataset_client import DatasetClient @@ -103,6 +104,7 @@ "Turn", "PredefinedScenario", "PredefinedScenarioExecutor", + "ReferenceInput", "SimulatedScenario", "SimulatedScenarioExecutor", "custom_code_based_evaluator", diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/__init__.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/__init__.py index c98b4a25..76d0239c 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/__init__.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/__init__.py @@ -1,10 +1,15 @@ """Code-based evaluator support for AgentCore Evaluation.""" from bedrock_agentcore.evaluation.custom_code_based_evaluators.decorator import custom_code_based_evaluator -from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput +from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import ( + EvaluatorInput, + EvaluatorOutput, + ReferenceInput, +) __all__ = [ "custom_code_based_evaluator", "EvaluatorInput", "EvaluatorOutput", + "ReferenceInput", ] diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py index 236cffaf..f23e7dfd 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py @@ -41,6 +41,7 @@ def lambda_handler(event, context=None): target_trace_id=trace_ids[0] if trace_ids else None, target_span_id=span_ids[0] if span_ids else None, schema_version=event.get("schemaVersion", "1.0"), + reference_inputs=event.get("evaluationReferenceInputs") or [], ) result = fn(evaluator_input, context) diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py index 1b17d6c5..cf5a1046 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py @@ -1,8 +1,34 @@ """Typed models for code-based evaluator Lambda input and output.""" -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field + + +class ReferenceInput(BaseModel): + """A single ground-truth entry from the event's ``evaluationReferenceInputs`` list. + + Field shapes follow the AgentCore code-based-evaluator contract. ``extra="allow"`` + keeps unknown/future keys instead of dropping them. + + Attributes: + context: Span context for the entry, e.g. {"spanContext": {"sessionId", "traceId"}}. + expected_response: Expected response object, e.g. {"text": "..."} (NOT a bare string). + assertions: Assertion-style ground truth, e.g. [{"text": "..."}]. + expected_trajectory: Expected tool trajectory, e.g. {"toolNames": [...]}. + """ + + context: Dict[str, Any] = Field(default_factory=dict) + expected_response: Optional[Dict[str, Any]] = Field(default=None, alias="expectedResponse") + assertions: List[Dict[str, Any]] = Field(default_factory=list) + expected_trajectory: Optional[Dict[str, Any]] = Field(default=None, alias="expectedTrajectory") + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + @property + def expected_response_text(self) -> Optional[str]: + """The ``expected_response.text`` value, or None if not present.""" + return (self.expected_response or {}).get("text") class EvaluatorInput(BaseModel): @@ -14,6 +40,9 @@ class EvaluatorInput(BaseModel): target_trace_id: The target trace ID (set for TRACE level, None otherwise). target_span_id: The target span ID (set for TOOL_CALL level, None otherwise). schema_version: Schema version of the Lambda contract. + reference_inputs: Ground-truth reference inputs (from evaluationReferenceInputs), + filtered by the service according to evaluation level. Empty when no ground + truth is configured. """ evaluation_level: str @@ -21,6 +50,7 @@ class EvaluatorInput(BaseModel): target_trace_id: Optional[str] = None target_span_id: Optional[str] = None schema_version: str = "1.0" + reference_inputs: List[ReferenceInput] = Field(default_factory=list) class EvaluatorOutput(BaseModel): diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py index 325c48a8..e03b896a 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py @@ -75,6 +75,39 @@ def handler(inp, context): handler(event) +class TestReferenceInputs: + def test_reference_inputs_passed_through(self): + captured = [] + + @custom_code_based_evaluator() + def handler(inp, context): + captured.append(inp.reference_inputs) + return EvaluatorOutput(value=1.0, label="Pass") + + event = _make_event(level="TRACE", trace_ids=["abc123"]) + event["evaluationReferenceInputs"] = [ + { + "context": {"spanContext": {"sessionId": "sess", "traceId": "abc123"}}, + "expectedResponse": {"text": "Paris"}, + } + ] + handler(event) + + assert len(captured[0]) == 1 + assert captured[0][0].expected_response_text == "Paris" + + def test_reference_inputs_default_empty(self): + captured = [] + + @custom_code_based_evaluator() + def handler(inp, context): + captured.append(inp.reference_inputs) + return EvaluatorOutput(value=1.0, label="Pass") + + handler(_make_event()) # no evaluationReferenceInputs key + assert captured[0] == [] + + class TestExceptionPropagation: def test_exception_propagates(self): @custom_code_based_evaluator() diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py index fac26f61..2c1e58e0 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py @@ -1,6 +1,10 @@ """Tests for EvaluatorInput and EvaluatorOutput dataclasses.""" -from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput +from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import ( + EvaluatorInput, + EvaluatorOutput, + ReferenceInput, +) class TestEvaluatorInput: @@ -18,6 +22,28 @@ def test_all_fields(self): assert inp.target_span_id is None assert inp.schema_version == "1.0" + def test_reference_inputs_default_empty(self): + inp = EvaluatorInput(evaluation_level="SESSION", session_spans=[]) + assert inp.reference_inputs == [] + + def test_reference_inputs_coerced_from_dicts(self): + # The service sends camelCase dicts; pydantic coerces them via aliases. + inp = EvaluatorInput( + evaluation_level="TRACE", + session_spans=[], + reference_inputs=[ + { + "context": {"spanContext": {"sessionId": "sess", "traceId": "t1"}}, + "expectedResponse": {"text": "Paris"}, + } + ], + ) + assert len(inp.reference_inputs) == 1 + ref = inp.reference_inputs[0] + assert isinstance(ref, ReferenceInput) + assert ref.expected_response_text == "Paris" + assert ref.context["spanContext"]["traceId"] == "t1" + def test_session_level_no_targets(self): inp = EvaluatorInput( evaluation_level="SESSION", @@ -54,3 +80,31 @@ def test_label_only(self): out = EvaluatorOutput(label="Fail") assert out.label == "Fail" assert out.value is None + + +class TestReferenceInput: + def test_defaults(self): + ref = ReferenceInput() + assert ref.context == {} + assert ref.expected_response is None + assert ref.assertions == [] + assert ref.expected_trajectory is None + assert ref.expected_response_text is None + + def test_expected_response_text(self): + ref = ReferenceInput(expected_response={"text": "Paris"}) + assert ref.expected_response_text == "Paris" + + def test_alias_and_field_name_both_accepted(self): + by_alias = ReferenceInput(expectedResponse={"text": "x"}, expectedTrajectory={"toolNames": ["a"]}) + by_name = ReferenceInput(expected_response={"text": "x"}, expected_trajectory={"toolNames": ["a"]}) + assert by_alias.expected_response_text == "x" + assert by_name.expected_trajectory == {"toolNames": ["a"]} + + def test_extra_keys_preserved(self): + ref = ReferenceInput.model_validate({"expectedResponse": {"text": "x"}, "futureField": 42}) + assert ref.model_extra["futureField"] == 42 + + def test_assertions(self): + ref = ReferenceInput(assertions=[{"text": "must be polite"}]) + assert ref.assertions == [{"text": "must be polite"}] From 3fedb3cfa768230260c43467f32b4ecb25f47cc6 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 18 Jun 2026 19:33:45 +0000 Subject: [PATCH 2/2] feat(evaluation): also expose evaluator_id and evaluator_name on EvaluatorInput The Lambda event carries evaluatorId and evaluatorName (top-level, per the code-based-evaluator input schema) but EvaluatorInput dropped both. Map them to optional evaluator_id/evaluator_name fields so evaluators can log or branch on which evaluator config invoked them. --- .../custom_code_based_evaluators/decorator.py | 2 ++ .../custom_code_based_evaluators/models.py | 4 +++ .../test_decorator.py | 30 +++++++++++++++++++ .../test_models.py | 5 ++++ 4 files changed, 41 insertions(+) diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py index f23e7dfd..8e8b7c51 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py @@ -41,6 +41,8 @@ def lambda_handler(event, context=None): target_trace_id=trace_ids[0] if trace_ids else None, target_span_id=span_ids[0] if span_ids else None, schema_version=event.get("schemaVersion", "1.0"), + evaluator_id=event.get("evaluatorId"), + evaluator_name=event.get("evaluatorName"), reference_inputs=event.get("evaluationReferenceInputs") or [], ) diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py index cf5a1046..4e5092b9 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py @@ -40,6 +40,8 @@ class EvaluatorInput(BaseModel): target_trace_id: The target trace ID (set for TRACE level, None otherwise). target_span_id: The target span ID (set for TOOL_CALL level, None otherwise). schema_version: Schema version of the Lambda contract. + evaluator_id: The ID of the code-based evaluator that was invoked. + evaluator_name: The name of the code-based evaluator that was invoked. reference_inputs: Ground-truth reference inputs (from evaluationReferenceInputs), filtered by the service according to evaluation level. Empty when no ground truth is configured. @@ -50,6 +52,8 @@ class EvaluatorInput(BaseModel): target_trace_id: Optional[str] = None target_span_id: Optional[str] = None schema_version: str = "1.0" + evaluator_id: Optional[str] = None + evaluator_name: Optional[str] = None reference_inputs: List[ReferenceInput] = Field(default_factory=list) diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py index e03b896a..af87a25f 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py @@ -75,6 +75,36 @@ def handler(inp, context): handler(event) +class TestEvaluatorIdentity: + def test_evaluator_id_and_name_passed_through(self): + captured = {} + + @custom_code_based_evaluator() + def handler(inp, context): + captured["id"] = inp.evaluator_id + captured["name"] = inp.evaluator_name + return EvaluatorOutput(value=1.0, label="Pass") + + event = _make_event() + event["evaluatorId"] = "my-eval-abc1234567" + event["evaluatorName"] = "MyEvaluator" + handler(event) + + assert captured == {"id": "my-eval-abc1234567", "name": "MyEvaluator"} + + def test_evaluator_id_and_name_default_none(self): + captured = {} + + @custom_code_based_evaluator() + def handler(inp, context): + captured["id"] = inp.evaluator_id + captured["name"] = inp.evaluator_name + return EvaluatorOutput(value=1.0, label="Pass") + + handler(_make_event()) # no evaluatorId/evaluatorName keys + assert captured == {"id": None, "name": None} + + class TestReferenceInputs: def test_reference_inputs_passed_through(self): captured = [] diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py index 2c1e58e0..33ea73dc 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py @@ -26,6 +26,11 @@ def test_reference_inputs_default_empty(self): inp = EvaluatorInput(evaluation_level="SESSION", session_spans=[]) assert inp.reference_inputs == [] + def test_evaluator_id_and_name_default_none(self): + inp = EvaluatorInput(evaluation_level="SESSION", session_spans=[]) + assert inp.evaluator_id is None + assert inp.evaluator_name is None + def test_reference_inputs_coerced_from_dicts(self): # The service sends camelCase dicts; pydantic coerces them via aliases. inp = EvaluatorInput(