feat(evaluation): expose evaluationReferenceInputs on EvaluatorInput

src/bedrock_agentcore/evaluation/__init__.py

@@ -4,6 +4,7 @@
 from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
     EvaluatorInput,
     EvaluatorOutput,
+    ReferenceInput,
     custom_code_based_evaluator,
 )
 from bedrock_agentcore.evaluation.dataset_client import DatasetClient
@@ -103,6 +104,7 @@
     "Turn",
     "PredefinedScenario",
     "PredefinedScenarioExecutor",
+    "ReferenceInput",
     "SimulatedScenario",
     "SimulatedScenarioExecutor",
     "custom_code_based_evaluator",

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/__init__.py

@@ -1,10 +1,15 @@
 """Code-based evaluator support for AgentCore Evaluation."""
 
 from bedrock_agentcore.evaluation.custom_code_based_evaluators.decorator import custom_code_based_evaluator
-from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    ReferenceInput,
+)
 
 __all__ = [
     "custom_code_based_evaluator",
     "EvaluatorInput",
     "EvaluatorOutput",
+    "ReferenceInput",
 ]

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/decorator.py

@@ -41,6 +41,9 @@ def lambda_handler(event, context=None):
                 target_trace_id=trace_ids[0] if trace_ids else None,
                 target_span_id=span_ids[0] if span_ids else None,
                 schema_version=event.get("schemaVersion", "1.0"),
+                evaluator_id=event.get("evaluatorId"),
+                evaluator_name=event.get("evaluatorName"),
+                reference_inputs=event.get("evaluationReferenceInputs") or [],
             )
 
             result = fn(evaluator_input, context)

src/bedrock_agentcore/evaluation/custom_code_based_evaluators/models.py

@@ -1,8 +1,34 @@
 """Typed models for code-based evaluator Lambda input and output."""
 
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ReferenceInput(BaseModel):
+    """A single ground-truth entry from the event's ``evaluationReferenceInputs`` list.
+
+    Field shapes follow the AgentCore code-based-evaluator contract. ``extra="allow"``
+    keeps unknown/future keys instead of dropping them.
+
+    Attributes:
+        context: Span context for the entry, e.g. {"spanContext": {"sessionId", "traceId"}}.
+        expected_response: Expected response object, e.g. {"text": "..."} (NOT a bare string).
+        assertions: Assertion-style ground truth, e.g. [{"text": "..."}].
+        expected_trajectory: Expected tool trajectory, e.g. {"toolNames": [...]}.
+    """
+
+    context: Dict[str, Any] = Field(default_factory=dict)
+    expected_response: Optional[Dict[str, Any]] = Field(default=None, alias="expectedResponse")
+    assertions: List[Dict[str, Any]] = Field(default_factory=list)
+    expected_trajectory: Optional[Dict[str, Any]] = Field(default=None, alias="expectedTrajectory")
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    @property
+    def expected_response_text(self) -> Optional[str]:
+        """The ``expected_response.text`` value, or None if not present."""
+        return (self.expected_response or {}).get("text")
 
 
 class EvaluatorInput(BaseModel):
@@ -14,13 +40,21 @@ class EvaluatorInput(BaseModel):
         target_trace_id: The target trace ID (set for TRACE level, None otherwise).
         target_span_id: The target span ID (set for TOOL_CALL level, None otherwise).
         schema_version: Schema version of the Lambda contract.
+        evaluator_id: The ID of the code-based evaluator that was invoked.
+        evaluator_name: The name of the code-based evaluator that was invoked.
+        reference_inputs: Ground-truth reference inputs (from evaluationReferenceInputs),
+            filtered by the service according to evaluation level. Empty when no ground
+            truth is configured.
     """
 
     evaluation_level: str
     session_spans: List[Dict]
     target_trace_id: Optional[str] = None
     target_span_id: Optional[str] = None
     schema_version: str = "1.0"
+    evaluator_id: Optional[str] = None
+    evaluator_name: Optional[str] = None
+    reference_inputs: List[ReferenceInput] = Field(default_factory=list)
 
 
 class EvaluatorOutput(BaseModel):

tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_decorator.py

@@ -75,6 +75,69 @@ def handler(inp, context):
         handler(event)
 
 
+class TestEvaluatorIdentity:
+    def test_evaluator_id_and_name_passed_through(self):
+        captured = {}
+
+        @custom_code_based_evaluator()
+        def handler(inp, context):
+            captured["id"] = inp.evaluator_id
+            captured["name"] = inp.evaluator_name
+            return EvaluatorOutput(value=1.0, label="Pass")
+
+        event = _make_event()
+        event["evaluatorId"] = "my-eval-abc1234567"
+        event["evaluatorName"] = "MyEvaluator"
+        handler(event)
+
+        assert captured == {"id": "my-eval-abc1234567", "name": "MyEvaluator"}
+
+    def test_evaluator_id_and_name_default_none(self):
+        captured = {}
+
+        @custom_code_based_evaluator()
+        def handler(inp, context):
+            captured["id"] = inp.evaluator_id
+            captured["name"] = inp.evaluator_name
+            return EvaluatorOutput(value=1.0, label="Pass")
+
+        handler(_make_event())  # no evaluatorId/evaluatorName keys
+        assert captured == {"id": None, "name": None}
+
+
+class TestReferenceInputs:
+    def test_reference_inputs_passed_through(self):
+        captured = []
+
+        @custom_code_based_evaluator()
+        def handler(inp, context):
+            captured.append(inp.reference_inputs)
+            return EvaluatorOutput(value=1.0, label="Pass")
+
+        event = _make_event(level="TRACE", trace_ids=["abc123"])
+        event["evaluationReferenceInputs"] = [
+            {
+                "context": {"spanContext": {"sessionId": "sess", "traceId": "abc123"}},
+                "expectedResponse": {"text": "Paris"},
+            }
+        ]
+        handler(event)
+
+        assert len(captured[0]) == 1
+        assert captured[0][0].expected_response_text == "Paris"
+
+    def test_reference_inputs_default_empty(self):
+        captured = []
+
+        @custom_code_based_evaluator()
+        def handler(inp, context):
+            captured.append(inp.reference_inputs)
+            return EvaluatorOutput(value=1.0, label="Pass")
+
+        handler(_make_event())  # no evaluationReferenceInputs key
+        assert captured[0] == []
+
+
 class TestExceptionPropagation:
     def test_exception_propagates(self):
         @custom_code_based_evaluator()

tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/test_models.py

@@ -1,6 +1,10 @@
 """Tests for EvaluatorInput and EvaluatorOutput dataclasses."""
 
-from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    ReferenceInput,
+)
 
 
 class TestEvaluatorInput:
@@ -18,6 +22,33 @@ def test_all_fields(self):
         assert inp.target_span_id is None
         assert inp.schema_version == "1.0"
 
+    def test_reference_inputs_default_empty(self):
+        inp = EvaluatorInput(evaluation_level="SESSION", session_spans=[])
+        assert inp.reference_inputs == []
+
+    def test_evaluator_id_and_name_default_none(self):
+        inp = EvaluatorInput(evaluation_level="SESSION", session_spans=[])
+        assert inp.evaluator_id is None
+        assert inp.evaluator_name is None
+
+    def test_reference_inputs_coerced_from_dicts(self):
+        # The service sends camelCase dicts; pydantic coerces them via aliases.
+        inp = EvaluatorInput(
+            evaluation_level="TRACE",
+            session_spans=[],
+            reference_inputs=[
+                {
+                    "context": {"spanContext": {"sessionId": "sess", "traceId": "t1"}},
+                    "expectedResponse": {"text": "Paris"},
+                }
+            ],
+        )
+        assert len(inp.reference_inputs) == 1
+        ref = inp.reference_inputs[0]
+        assert isinstance(ref, ReferenceInput)
+        assert ref.expected_response_text == "Paris"
+        assert ref.context["spanContext"]["traceId"] == "t1"
+
     def test_session_level_no_targets(self):
         inp = EvaluatorInput(
             evaluation_level="SESSION",
@@ -54,3 +85,31 @@ def test_label_only(self):
         out = EvaluatorOutput(label="Fail")
         assert out.label == "Fail"
         assert out.value is None
+
+
+class TestReferenceInput:
+    def test_defaults(self):
+        ref = ReferenceInput()
+        assert ref.context == {}
+        assert ref.expected_response is None
+        assert ref.assertions == []
+        assert ref.expected_trajectory is None
+        assert ref.expected_response_text is None
+
+    def test_expected_response_text(self):
+        ref = ReferenceInput(expected_response={"text": "Paris"})
+        assert ref.expected_response_text == "Paris"
+
+    def test_alias_and_field_name_both_accepted(self):
+        by_alias = ReferenceInput(expectedResponse={"text": "x"}, expectedTrajectory={"toolNames": ["a"]})
+        by_name = ReferenceInput(expected_response={"text": "x"}, expected_trajectory={"toolNames": ["a"]})
+        assert by_alias.expected_response_text == "x"
+        assert by_name.expected_trajectory == {"toolNames": ["a"]}
+
+    def test_extra_keys_preserved(self):
+        ref = ReferenceInput.model_validate({"expectedResponse": {"text": "x"}, "futureField": 42})
+        assert ref.model_extra["futureField"] == 42
+
+    def test_assertions(self):
+        ref = ReferenceInput(assertions=[{"text": "must be polite"}])
+        assert ref.assertions == [{"text": "must be polite"}]