From ab134a0d6d5a762d83788228cefd0964d1f654eb Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Wed, 5 Aug 2026 21:23:46 +0900 Subject: [PATCH 1/4] fix: redact JSON validation errors --- src/agents/agent_output.py | 19 ++- src/agents/exceptions.py | 11 ++ src/agents/handoffs/__init__.py | 20 ++- src/agents/realtime/handoffs.py | 20 ++- src/agents/realtime/session.py | 30 +++- src/agents/run.py | 2 + src/agents/run_internal/run_loop.py | 2 + src/agents/util/_json.py | 35 ++++- tests/realtime/test_session.py | 94 +++++++++++- tests/test_error_logging_redaction.py | 200 ++++++++++++++++++++++++++ 10 files changed, 400 insertions(+), 33 deletions(-) diff --git a/src/agents/agent_output.py b/src/agents/agent_output.py index f2274280b0..20e7cf70b0 100644 --- a/src/agents/agent_output.py +++ b/src/agents/agent_output.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, TypeAdapter from typing_extensions import TypedDict -from .exceptions import ModelBehaviorError, UserError +from .exceptions import ModelBehaviorError, UserError, _clear_data_redacted_error_traceback from .strict_schema import ensure_strict_json_schema from .tracing import SpanError from .util import _error_tracing, _json @@ -137,12 +137,17 @@ def validate_json(self, json_str: str) -> Any: """Validate a JSON string against the output type. Returns the validated object, or raises a `ModelBehaviorError` if the JSON is invalid. """ - validated = _json.validate_json( - json_str, - self._type_adapter, - partial=False, - strict=True if self._strict_json_schema else None, - ) + try: + validated = _json.validate_json( + json_str, + self._type_adapter, + partial=False, + strict=True if self._strict_json_schema else None, + ) + except ModelBehaviorError as error: + json_str = "" + _clear_data_redacted_error_traceback(error) + raise if self._is_wrapped: if not isinstance(validated, dict): _error_tracing.attach_error_to_current_span( diff --git a/src/agents/exceptions.py b/src/agents/exceptions.py index 349004c97d..1ccb3d6334 100644 --- a/src/agents/exceptions.py +++ b/src/agents/exceptions.py @@ -1,5 +1,6 @@ from __future__ import annotations +import traceback from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any @@ -19,6 +20,7 @@ from .util._pretty_print import pretty_print_run_error_details _DRAIN_STREAM_EVENTS_ATTR = "_agents_drain_queued_stream_events" +_DATA_REDACTED_ATTR = "_agents_data_redacted" def _mark_error_to_drain_stream_events(error: Exception) -> None: @@ -29,6 +31,15 @@ def _should_drain_stream_events_before_raising(error: Exception) -> bool: return bool(getattr(error, _DRAIN_STREAM_EVENTS_ATTR, False)) +def _mark_error_data_redacted(error: Exception) -> None: + setattr(error, _DATA_REDACTED_ATTR, True) + + +def _clear_data_redacted_error_traceback(error: Exception) -> None: + if getattr(error, _DATA_REDACTED_ATTR, False) and error.__traceback__ is not None: + traceback.clear_frames(error.__traceback__) + + @dataclass class RunErrorDetails: """Data collected from an agent run when an exception occurs.""" diff --git a/src/agents/handoffs/__init__.py b/src/agents/handoffs/__init__.py index 79d1841760..53dafb65d1 100644 --- a/src/agents/handoffs/__init__.py +++ b/src/agents/handoffs/__init__.py @@ -10,7 +10,7 @@ from pydantic import TypeAdapter from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError, UserError +from ..exceptions import ModelBehaviorError, UserError, _clear_data_redacted_error_traceback from ..items import RunItem, TResponseInputItem from ..run_context import RunContextWrapper, TContext from ..strict_schema import ensure_strict_json_schema @@ -295,12 +295,18 @@ async def _invoke_handoff( ) raise ModelBehaviorError("Handoff function expected non-null input, but got None") - validated_input = _json.validate_json( - json_str=input_json, - type_adapter=type_adapter, - partial=False, - strict=True, - ) + try: + validated_input = _json.validate_json( + json_str=input_json, + type_adapter=type_adapter, + partial=False, + strict=True, + contains_tool_data=True, + ) + except ModelBehaviorError as error: + input_json = "" + _clear_data_redacted_error_traceback(error) + raise input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) if inspect.isawaitable(result): diff --git a/src/agents/realtime/handoffs.py b/src/agents/realtime/handoffs.py index a2026772ee..7f04f23edc 100644 --- a/src/agents/realtime/handoffs.py +++ b/src/agents/realtime/handoffs.py @@ -7,7 +7,7 @@ from pydantic import TypeAdapter from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError, UserError +from ..exceptions import ModelBehaviorError, UserError, _clear_data_redacted_error_traceback from ..handoffs import Handoff from ..run_context import RunContextWrapper, TContext from ..strict_schema import ensure_strict_json_schema @@ -159,12 +159,18 @@ async def _invoke_handoff( ) raise ModelBehaviorError("Handoff function expected non-null input, but got None") - validated_input = _json.validate_json( - json_str=input_json, - type_adapter=type_adapter, - partial=False, - strict=True, - ) + try: + validated_input = _json.validate_json( + json_str=input_json, + type_adapter=type_adapter, + partial=False, + strict=True, + contains_tool_data=True, + ) + except ModelBehaviorError as error: + input_json = "" + _clear_data_redacted_error_traceback(error) + raise input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) if inspect.isawaitable(result): diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index f224bbf068..2750d13ce3 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -18,7 +18,12 @@ get_function_tool_namespace, ) from ..agent import Agent -from ..exceptions import ToolInputGuardrailTripwireTriggered, UserError +from ..exceptions import ( + ModelBehaviorError, + ToolInputGuardrailTripwireTriggered, + UserError, + _clear_data_redacted_error_traceback, +) from ..handoffs import Handoff from ..items import ToolApprovalItem from ..logger import ( @@ -398,7 +403,18 @@ async def on_event(self, event: RealtimeModelEvent) -> None: handle_kwargs: dict[str, Any] = {"agent_snapshot": agent_snapshot} if dispatch_snapshot is not None: handle_kwargs["dispatch_snapshot"] = dispatch_snapshot - await self._handle_tool_call(event, **handle_kwargs) + try: + await self._handle_tool_call(event, **handle_kwargs) + except ModelBehaviorError as error: + # Remove model-generated arguments from this synchronous boundary before the + # exception escapes to the model listener. + event = RealtimeModelToolCallEvent( + name="", + call_id="", + arguments="", + ) + _clear_data_redacted_error_traceback(error) + raise elif event.type == "audio": if event.response_id not in self._interrupted_response_ids: await self._put_event( @@ -1625,14 +1641,16 @@ def _enqueue_tool_call_task( def _on_tool_call_task_done(self, task: asyncio.Task[Any]) -> None: self._tool_call_tasks.discard(task) - if self._closing or self._closed: - self._consume_task_result(task) - return - if task.cancelled(): return exception = task.exception() + if isinstance(exception, ModelBehaviorError): + _clear_data_redacted_error_traceback(exception) + + if self._closing or self._closed: + return + if exception is None: return diff --git a/src/agents/run.py b/src/agents/run.py index 00028cf406..1584a0466b 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -17,6 +17,7 @@ OutputGuardrailTripwireTriggered, RunErrorDetails, UserError, + _clear_data_redacted_error_traceback, ) from .guardrail import ( InputGuardrailResult, @@ -1621,6 +1622,7 @@ def _finalize_result(result: RunResult) -> RunResult: trace_include_sensitive_data=run_config.trace_include_sensitive_data, ) if isinstance(exc, AgentsException): + _clear_data_redacted_error_traceback(exc) exc.run_data = RunErrorDetails( input=original_input, new_items=session_items, diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 643238d914..f52110491b 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -40,6 +40,7 @@ OutputGuardrailTripwireTriggered, RunErrorDetails, UserError, + _clear_data_redacted_error_traceback, ) from ..handoffs import Handoff from ..items import ( @@ -1387,6 +1388,7 @@ async def _save_stream_items_without_count( except AgentsException as exc: streamed_result.is_complete = True streamed_result._event_queue.put_nowait(QueueCompleteSentinel()) + _clear_data_redacted_error_traceback(exc) exc.run_data = RunErrorDetails( input=streamed_result.input, new_items=streamed_result.new_items, diff --git a/src/agents/util/_json.py b/src/agents/util/_json.py index 67186328cd..e2ec3eeab9 100644 --- a/src/agents/util/_json.py +++ b/src/agents/util/_json.py @@ -6,7 +6,8 @@ from pydantic import TypeAdapter, ValidationError from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError +from .. import _debug +from ..exceptions import ModelBehaviorError, _mark_error_data_redacted from ..tracing import SpanError from ._error_tracing import attach_error_to_current_span @@ -14,8 +15,14 @@ def validate_json( - json_str: str, type_adapter: TypeAdapter[T], partial: bool, strict: bool | None = None + json_str: str, + type_adapter: TypeAdapter[T], + partial: bool, + strict: bool | None = None, + *, + contains_tool_data: bool = False, ) -> T: + should_redact = _debug.DONT_LOG_MODEL_DATA or (contains_tool_data and _debug.DONT_LOG_TOOL_DATA) partial_setting: bool | Literal["off", "on", "trailing-strings"] = ( "trailing-strings" if partial else False ) @@ -26,15 +33,33 @@ def validate_json( validated = type_adapter.validate_json(json_str, **kwargs) return validated except ValidationError as e: + if not should_redact: + attach_error_to_current_span( + SpanError( + message="Invalid JSON provided", + data={}, + ) + ) + raise ModelBehaviorError( + f"Invalid JSON when parsing {json_str} for {type_adapter}; {e}" + ) from e + + # Clear the payload before creating the redacted traceback frame. Raising outside the except + # block also prevents the payload-bearing ValidationError from becoming the cause or context. + json_str = "" + error = ModelBehaviorError(f"Invalid JSON when parsing model output for {type_adapter}") + _mark_error_data_redacted(error) + # Redacted error reporting is best-effort so tracing failures cannot replace the safe error. + try: attach_error_to_current_span( SpanError( message="Invalid JSON provided", data={}, ) ) - raise ModelBehaviorError( - f"Invalid JSON when parsing {json_str} for {type_adapter}; {e}" - ) from e + except Exception: + pass + raise error def _to_dump_compatible(obj: Any) -> Any: diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index 9a6309fcc5..4c0931cedb 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -3,6 +3,7 @@ import json import logging import threading +import traceback from typing import Any, cast from unittest.mock import AsyncMock, Mock, PropertyMock, patch @@ -11,9 +12,10 @@ import agents._debug as _debug from agents.agent import AgentBase -from agents.exceptions import ToolTimeoutError, UserError +from agents.exceptions import ModelBehaviorError, ToolTimeoutError, UserError from agents.guardrail import GuardrailFunctionOutput, OutputGuardrail from agents.handoffs import Handoff +from agents.realtime import realtime_handoff from agents.realtime.agent import RealtimeAgent from agents.realtime.config import RealtimeRunConfig, RealtimeSessionModelSettings from agents.realtime.events import ( @@ -780,6 +782,96 @@ async def failing_task() -> None: assert err.error["message"] == expected_message +@pytest.mark.asyncio +@pytest.mark.parametrize("state_name", [None, "_closing", "_closed"]) +async def test_on_tool_call_task_done_clears_redacted_error_traceback( + monkeypatch: pytest.MonkeyPatch, + state_name: str | None, +) -> None: + secret = "REALTIME_HANDOFF_TRACEBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + session = RealtimeSession(_DummyModel(), RealtimeAgent(name="agent"), None) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + + async def failing_task() -> None: + payload = f'"{secret}"' + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + + task = asyncio.create_task(failing_task()) + await asyncio.gather(task, return_exceptions=True) + error = task.exception() + assert isinstance(error, ModelBehaviorError) + + before = traceback.TracebackException.from_exception(error, capture_locals=True) + assert secret in "".join( + value for frame in before.stack for value in (frame.locals or {}).values() + ) + + if state_name is not None: + setattr(session, state_name, True) + session._on_tool_call_task_done(task) + + after = traceback.TracebackException.from_exception(error, capture_locals=True) + assert secret not in "".join( + value for frame in after.stack for value in (frame.locals or {}).values() + ) + if state_name is None: + assert session._stored_exception is error + event = session._event_queue.get_nowait() + assert isinstance(event, RealtimeError) + assert secret not in event.error["message"] + else: + assert session._stored_exception is None + assert session._event_queue.empty() + + +@pytest.mark.asyncio +async def test_synchronous_realtime_handoff_clears_redacted_error_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = "SYNCHRONOUS_REALTIME_HANDOFF_TRACEBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + agent = RealtimeAgent(name="agent", handoffs=[handoff_obj]) + session = RealtimeSession( + _DummyModel(), + agent, + None, + run_config={"async_tool_calls": False}, + ) + event = RealtimeModelToolCallEvent( + name=handoff_obj.tool_name, + call_id="call-1", + arguments=f'"{secret}"', + ) + + with pytest.raises(ModelBehaviorError) as exc_info: + await session.on_event(event) + + error = exc_info.value + traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) + sdk_frames = [frame for frame in traceback_exception.stack if "/src/agents/" in frame.filename] + assert sdk_frames + assert secret not in "".join( + value for frame in sdk_frames for value in (frame.locals or {}).values() + ) + assert secret not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + + @pytest.mark.asyncio async def test_get_handoffs_async_is_enabled(monkeypatch): # Agent includes both a direct Handoff and a RealtimeAgent (auto-converted) diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index b194c6f41b..afdccea96f 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -11,6 +11,7 @@ import logging import pickle import threading +import traceback from logging.handlers import QueueHandler from pathlib import Path from queue import SimpleQueue @@ -31,9 +32,12 @@ OpenAIResponsesModel, RunConfig, RunContextWrapper, + Runner, function_tool, + handoff, trace, ) +from agents.agent_output import AgentOutputSchema from agents.logger import ( log_model_action_debug, log_model_action_error, @@ -46,6 +50,7 @@ log_tool_action_error as log_shared_tool_action_error, log_tool_action_warning, ) +from agents.realtime import RealtimeAgent, realtime_handoff from agents.run_internal.tool_execution import ( log_tool_action_error, resolve_approval_rejection_message, @@ -57,6 +62,9 @@ from agents.tracing.spans import Span from agents.tracing.traces import Trace +from .fake_model import FakeModel +from .test_responses import get_text_message + _SECRET = "super secret prompt content" @@ -858,3 +866,195 @@ async def test_agent_tool_validation_error_preserves_diagnostics_when_tool_data_ error = exc_info.value assert _TOOL_ARGUMENT_SECRET in str(error) assert isinstance(error.__cause__, ValidationError) + + +_MODEL_OUTPUT_SECRET = "SECRET_MODEL_OUTPUT_123" + + +class _RequiredOutput(BaseModel): + answer: str + count: int + + +def _assert_secret_absent_from_agents_traceback(error: BaseException, secret: str) -> None: + traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) + agents_source = (Path(__file__).parents[1] / "src" / "agents").resolve() + agents_frames = [ + frame + for frame in traceback_exception.stack + if Path(frame.filename).resolve().is_relative_to(agents_source) + ] + assert agents_frames + for frame in agents_frames: + assert secret not in "".join((frame.locals or {}).values()) + + +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "expected_redacted"), + [ + (True, False, True), + (False, True, False), + ], +) +def test_output_schema_validation_error_follows_model_data_policy( + monkeypatch: pytest.MonkeyPatch, + model_redacted: bool, + tool_redacted: bool, + expected_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + with pytest.raises(ModelBehaviorError) as exc_info: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + + error = exc_info.value + if expected_redacted: + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + + +def test_output_schema_redaction_survives_trace_attachment_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + with patch( + "agents.util._json.attach_error_to_current_span", + side_effect=RuntimeError("trace attachment failed"), + ): + with pytest.raises(ModelBehaviorError) as exc_info: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + + error = exc_info.value + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "expected_redacted"), + [ + (True, False, True), + (False, True, True), + (True, True, True), + (False, False, False), + ], +) +@pytest.mark.asyncio +async def test_handoff_input_validation_error_follows_mixed_data_policy( + monkeypatch: pytest.MonkeyPatch, + model_redacted: bool, + tool_redacted: bool, + expected_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + target = Agent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: _RequiredOutput) -> None: + pass # pragma: no cover + + handoff_obj = handoff(target, input_type=_RequiredOutput, on_handoff=on_handoff) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + + error = exc_info.value + if expected_redacted: + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + + +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "expected_redacted"), + [ + (True, False, True), + (False, True, True), + (True, True, True), + (False, False, False), + ], +) +@pytest.mark.asyncio +async def test_realtime_handoff_input_validation_error_follows_mixed_data_policy( + monkeypatch: pytest.MonkeyPatch, + model_redacted: bool, + tool_redacted: bool, + expected_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: _RequiredOutput) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, input_type=_RequiredOutput, on_handoff=on_handoff) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + + error = exc_info.value + if expected_redacted: + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + + +@pytest.mark.asyncio +async def test_run_surfaces_redacted_output_validation_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + model = FakeModel() + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + + with pytest.raises(ModelBehaviorError) as exc_info: + await Runner.run(agent, "go") + + error = exc_info.value + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.asyncio +async def test_streamed_run_surfaces_redacted_output_validation_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + model = FakeModel() + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + result = Runner.run_streamed(agent, "go") + + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + + error = exc_info.value + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) From 5af632d74231a478739b1e99b4e043d8992e0ffb Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Wed, 5 Aug 2026 23:11:06 +0900 Subject: [PATCH 2/4] fix: close JSON validation redaction gaps --- src/agents/agent_output.py | 12 +- src/agents/exceptions.py | 11 +- src/agents/handoffs/__init__.py | 12 +- src/agents/realtime/handoffs.py | 12 +- src/agents/realtime/session.py | 23 +- src/agents/result.py | 27 +- src/agents/run.py | 94 +++-- src/agents/run_internal/run_loop.py | 29 +- src/agents/run_internal/turn_resolution.py | 8 +- src/agents/util/_json.py | 2 +- tests/realtime/test_session.py | 140 ++++++- tests/test_error_logging_redaction.py | 422 ++++++++++++++++++++- 12 files changed, 698 insertions(+), 94 deletions(-) diff --git a/src/agents/agent_output.py b/src/agents/agent_output.py index 20e7cf70b0..758b9dfba9 100644 --- a/src/agents/agent_output.py +++ b/src/agents/agent_output.py @@ -5,7 +5,12 @@ from pydantic import BaseModel, TypeAdapter from typing_extensions import TypedDict -from .exceptions import ModelBehaviorError, UserError, _clear_data_redacted_error_traceback +from .exceptions import ( + ModelBehaviorError, + UserError, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, +) from .strict_schema import ensure_strict_json_schema from .tracing import SpanError from .util import _error_tracing, _json @@ -145,8 +150,9 @@ def validate_json(self, json_str: str) -> Any: strict=True if self._strict_json_schema else None, ) except ModelBehaviorError as error: - json_str = "" - _clear_data_redacted_error_traceback(error) + if _is_error_data_redacted(error): + json_str = "" + _detach_data_redacted_error_traceback(error) raise if self._is_wrapped: if not isinstance(validated, dict): diff --git a/src/agents/exceptions.py b/src/agents/exceptions.py index 1ccb3d6334..e5e2fb8195 100644 --- a/src/agents/exceptions.py +++ b/src/agents/exceptions.py @@ -35,11 +35,20 @@ def _mark_error_data_redacted(error: Exception) -> None: setattr(error, _DATA_REDACTED_ATTR, True) +def _is_error_data_redacted(error: Exception) -> bool: + return bool(getattr(error, _DATA_REDACTED_ATTR, False)) + + def _clear_data_redacted_error_traceback(error: Exception) -> None: - if getattr(error, _DATA_REDACTED_ATTR, False) and error.__traceback__ is not None: + if _is_error_data_redacted(error) and error.__traceback__ is not None: traceback.clear_frames(error.__traceback__) +def _detach_data_redacted_error_traceback(error: Exception) -> None: + if _is_error_data_redacted(error): + error.__traceback__ = None + + @dataclass class RunErrorDetails: """Data collected from an agent run when an exception occurs.""" diff --git a/src/agents/handoffs/__init__.py b/src/agents/handoffs/__init__.py index 53dafb65d1..11b28a1cdc 100644 --- a/src/agents/handoffs/__init__.py +++ b/src/agents/handoffs/__init__.py @@ -10,7 +10,12 @@ from pydantic import TypeAdapter from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError, UserError, _clear_data_redacted_error_traceback +from ..exceptions import ( + ModelBehaviorError, + UserError, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, +) from ..items import RunItem, TResponseInputItem from ..run_context import RunContextWrapper, TContext from ..strict_schema import ensure_strict_json_schema @@ -304,8 +309,9 @@ async def _invoke_handoff( contains_tool_data=True, ) except ModelBehaviorError as error: - input_json = "" - _clear_data_redacted_error_traceback(error) + if _is_error_data_redacted(error): + input_json = "" + _detach_data_redacted_error_traceback(error) raise input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) diff --git a/src/agents/realtime/handoffs.py b/src/agents/realtime/handoffs.py index 7f04f23edc..e333403467 100644 --- a/src/agents/realtime/handoffs.py +++ b/src/agents/realtime/handoffs.py @@ -7,7 +7,12 @@ from pydantic import TypeAdapter from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError, UserError, _clear_data_redacted_error_traceback +from ..exceptions import ( + ModelBehaviorError, + UserError, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, +) from ..handoffs import Handoff from ..run_context import RunContextWrapper, TContext from ..strict_schema import ensure_strict_json_schema @@ -168,8 +173,9 @@ async def _invoke_handoff( contains_tool_data=True, ) except ModelBehaviorError as error: - input_json = "" - _clear_data_redacted_error_traceback(error) + if _is_error_data_redacted(error): + input_json = "" + _detach_data_redacted_error_traceback(error) raise input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index 2750d13ce3..f7417efec7 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -23,6 +23,8 @@ ToolInputGuardrailTripwireTriggered, UserError, _clear_data_redacted_error_traceback, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, ) from ..handoffs import Handoff from ..items import ToolApprovalItem @@ -309,7 +311,16 @@ async def __aiter__(self) -> AsyncIterator[RealtimeSessionEvent]: if self._stored_exception is not None: # Clean up resources before raising await self.close() - raise self._stored_exception + stored_exception = self._stored_exception + if isinstance(stored_exception, Exception) and _is_error_data_redacted( + stored_exception + ): + _detach_data_redacted_error_traceback(stored_exception) + # Do not retain the session or the previously yielded raw event in the + # traceback frame that exposes a redacted error to the caller. + self = cast(Any, None) + event = cast(Any, None) + raise stored_exception self._event_iterator_waiters += 1 try: @@ -406,14 +417,8 @@ async def on_event(self, event: RealtimeModelEvent) -> None: try: await self._handle_tool_call(event, **handle_kwargs) except ModelBehaviorError as error: - # Remove model-generated arguments from this synchronous boundary before the - # exception escapes to the model listener. - event = RealtimeModelToolCallEvent( - name="", - call_id="", - arguments="", - ) - _clear_data_redacted_error_traceback(error) + if _is_error_data_redacted(error): + _detach_data_redacted_error_traceback(error) raise elif event.type == "audio": if event.response_id not in self._interrupted_response_ids: diff --git a/src/agents/result.py b/src/agents/result.py index 6482cd2813..7ebc03f15e 100644 --- a/src/agents/result.py +++ b/src/agents/result.py @@ -18,6 +18,8 @@ InputGuardrailTripwireTriggered, MaxTurnsExceeded, RunErrorDetails, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, _should_drain_stream_events_before_raising, ) from .guardrail import InputGuardrailResult, OutputGuardrailResult @@ -927,8 +929,16 @@ def register_current_consumer() -> None: self._drain_event_queue() self._drain_input_guardrail_queue() - if self._stored_exception: - raise self._stored_exception + stored_exception = self._stored_exception + if stored_exception: + if _is_error_data_redacted(stored_exception): + _detach_data_redacted_error_traceback(stored_exception) + # The streaming result retains caller-visible run data. Drop the local reference + # before raising so the redacted exception cannot retain it through this frame. + self = cast(Any, None) + registered_consumer_task = None + item = cast(Any, None) + raise stored_exception def _create_error_details(self) -> RunErrorDetails | None: """Return a `RunErrorDetails` object considering the current attributes of the class. @@ -974,7 +984,11 @@ def _check_errors(self): if not self.run_loop_task.cancelled(): run_impl_exc = self.run_loop_task.exception() if run_impl_exc and isinstance(run_impl_exc, Exception): - if isinstance(run_impl_exc, AgentsException) and run_impl_exc.run_data is None: + if ( + isinstance(run_impl_exc, AgentsException) + and run_impl_exc.run_data is None + and not _is_error_data_redacted(run_impl_exc) + ): run_impl_exc.run_data = self._create_error_details() self._stored_exception = run_impl_exc @@ -982,7 +996,11 @@ def _check_errors(self): if not self._input_guardrails_task.cancelled(): in_guard_exc = self._input_guardrails_task.exception() if in_guard_exc and isinstance(in_guard_exc, Exception): - if isinstance(in_guard_exc, AgentsException) and in_guard_exc.run_data is None: + if ( + isinstance(in_guard_exc, AgentsException) + and in_guard_exc.run_data is None + and not _is_error_data_redacted(in_guard_exc) + ): in_guard_exc.run_data = self._create_error_details() self._stored_exception = in_guard_exc @@ -993,6 +1011,7 @@ def _check_errors(self): if ( isinstance(out_guard_exc, AgentsException) and out_guard_exc.run_data is None + and not _is_error_data_redacted(out_guard_exc) ): out_guard_exc.run_data = self._create_error_details() self._stored_exception = out_guard_exc diff --git a/src/agents/run.py b/src/agents/run.py index 1584a0466b..a1fd583267 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -14,10 +14,13 @@ AgentsException, InputGuardrailTripwireTriggered, MaxTurnsExceeded, + ModelBehaviorError, OutputGuardrailTripwireTriggered, RunErrorDetails, UserError, _clear_data_redacted_error_traceback, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, ) from .guardrail import ( InputGuardrailResult, @@ -276,19 +279,23 @@ async def run( """ runner = DEFAULT_AGENT_RUNNER - return await runner.run( - starting_agent, - input, - context=context, - max_turns=max_turns, - hooks=hooks, - run_config=run_config, - error_handlers=error_handlers, - previous_response_id=previous_response_id, - auto_previous_response_id=auto_previous_response_id, - conversation_id=conversation_id, - session=session, - ) + try: + return await runner.run( + starting_agent, + input, + context=context, + max_turns=max_turns, + hooks=hooks, + run_config=run_config, + error_handlers=error_handlers, + previous_response_id=previous_response_id, + auto_previous_response_id=auto_previous_response_id, + conversation_id=conversation_id, + session=session, + ) + except ModelBehaviorError as error: + _detach_data_redacted_error_traceback(error) + raise @classmethod def run_sync( @@ -358,19 +365,23 @@ def run_sync( """ runner = DEFAULT_AGENT_RUNNER - return runner.run_sync( - starting_agent, - input, - context=context, - max_turns=max_turns, - hooks=hooks, - run_config=run_config, - error_handlers=error_handlers, - previous_response_id=previous_response_id, - conversation_id=conversation_id, - session=session, - auto_previous_response_id=auto_previous_response_id, - ) + try: + return runner.run_sync( + starting_agent, + input, + context=context, + max_turns=max_turns, + hooks=hooks, + run_config=run_config, + error_handlers=error_handlers, + previous_response_id=previous_response_id, + conversation_id=conversation_id, + session=session, + auto_previous_response_id=auto_previous_response_id, + ) + except ModelBehaviorError as error: + _detach_data_redacted_error_traceback(error) + raise @classmethod def run_streamed( @@ -1622,18 +1633,21 @@ def _finalize_result(result: RunResult) -> RunResult: trace_include_sensitive_data=run_config.trace_include_sensitive_data, ) if isinstance(exc, AgentsException): - _clear_data_redacted_error_traceback(exc) - exc.run_data = RunErrorDetails( - input=original_input, - new_items=session_items, - raw_responses=model_responses, - last_agent=current_agent, - context_wrapper=context_wrapper, - input_guardrail_results=input_guardrail_results, - output_guardrail_results=output_guardrail_results, - tool_input_guardrail_results=tool_input_guardrail_results, - tool_output_guardrail_results=tool_output_guardrail_results, - ) + if _is_error_data_redacted(exc): + _detach_data_redacted_error_traceback(exc) + else: + _clear_data_redacted_error_traceback(exc) + exc.run_data = RunErrorDetails( + input=original_input, + new_items=session_items, + raw_responses=model_responses, + last_agent=current_agent, + context_wrapper=context_wrapper, + input_guardrail_results=input_guardrail_results, + output_guardrail_results=output_guardrail_results, + tool_input_guardrail_results=tool_input_guardrail_results, + tool_output_guardrail_results=tool_output_guardrail_results, + ) raise finally: await cleanup_models_after_run(tool_use_tracker) @@ -1758,13 +1772,15 @@ def run_sync( try: # Drive the coroutine to completion, harvesting the final RunResult. return default_loop.run_until_complete(task) - except BaseException: + except BaseException as error: # If the sync caller aborts (KeyboardInterrupt, etc.), make sure the scheduled task # does not linger on the shared loop by cancelling it and waiting for completion. if not task.done(): task.cancel() with contextlib.suppress(asyncio.CancelledError): default_loop.run_until_complete(task) + if isinstance(error, ModelBehaviorError): + _detach_data_redacted_error_traceback(error) raise finally: if not default_loop.is_closed(): diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index f52110491b..80ec1928f3 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -41,6 +41,8 @@ RunErrorDetails, UserError, _clear_data_redacted_error_traceback, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, ) from ..handoffs import Handoff from ..items import ( @@ -1388,18 +1390,21 @@ async def _save_stream_items_without_count( except AgentsException as exc: streamed_result.is_complete = True streamed_result._event_queue.put_nowait(QueueCompleteSentinel()) - _clear_data_redacted_error_traceback(exc) - exc.run_data = RunErrorDetails( - input=streamed_result.input, - new_items=streamed_result.new_items, - raw_responses=streamed_result.raw_responses, - last_agent=current_agent, - context_wrapper=context_wrapper, - input_guardrail_results=streamed_result.input_guardrail_results, - output_guardrail_results=streamed_result.output_guardrail_results, - tool_input_guardrail_results=streamed_result.tool_input_guardrail_results, - tool_output_guardrail_results=streamed_result.tool_output_guardrail_results, - ) + if _is_error_data_redacted(exc): + _detach_data_redacted_error_traceback(exc) + else: + _clear_data_redacted_error_traceback(exc) + exc.run_data = RunErrorDetails( + input=streamed_result.input, + new_items=streamed_result.new_items, + raw_responses=streamed_result.raw_responses, + last_agent=current_agent, + context_wrapper=context_wrapper, + input_guardrail_results=streamed_result.input_guardrail_results, + output_guardrail_results=streamed_result.output_guardrail_results, + tool_input_guardrail_results=streamed_result.tool_input_guardrail_results, + tool_output_guardrail_results=streamed_result.tool_output_guardrail_results, + ) raise except Exception as e: attach_generic_agent_error( diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 5bf9839533..f3099e6677 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -54,7 +54,12 @@ peek_agent_tool_run_result, record_agent_tool_run_result, ) -from ..exceptions import ModelBehaviorError, ModelRefusalError, UserError +from ..exceptions import ( + ModelBehaviorError, + ModelRefusalError, + UserError, + _detach_data_redacted_error_traceback, +) from ..handoffs import Handoff, HandoffInputData, HandoffInputFilter, nest_handoff_history from ..handoffs.history import ( _get_nested_history_owned_items, @@ -421,6 +426,7 @@ async def _resolve_invalid_final_output( raw_responses=[new_response], last_agent=public_agent, ) + _detach_data_redacted_error_traceback(error) handler_result = await resolve_run_error_handler_result( error_handlers=error_handlers, error_kind="invalid_final_output", diff --git a/src/agents/util/_json.py b/src/agents/util/_json.py index e2ec3eeab9..fd944b6129 100644 --- a/src/agents/util/_json.py +++ b/src/agents/util/_json.py @@ -47,7 +47,7 @@ def validate_json( # Clear the payload before creating the redacted traceback frame. Raising outside the except # block also prevents the payload-bearing ValidationError from becoming the cause or context. json_str = "" - error = ModelBehaviorError(f"Invalid JSON when parsing model output for {type_adapter}") + error = ModelBehaviorError("Invalid JSON when parsing model output") _mark_error_data_redacted(error) # Redacted error reporting is best-effort so tracing failures cannot replace the safe error. try: diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index 4c0931cedb..f5b1756b2d 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -4,7 +4,8 @@ import logging import threading import traceback -from typing import Any, cast +from pathlib import Path +from typing import Any, Literal, cast from unittest.mock import AsyncMock, Mock, PropertyMock, patch import pytest @@ -831,19 +832,43 @@ async def failing_task() -> None: assert session._event_queue.empty() +def _realtime_sdk_traceback_frame_locals(error: BaseException) -> list[dict[str, Any]]: + agents_source = (Path(__file__).parents[2] / "src" / "agents").resolve() + frame_locals: list[dict[str, Any]] = [] + traceback_object = error.__traceback__ + while traceback_object is not None: + if ( + Path(traceback_object.tb_frame.f_code.co_filename) + .resolve() + .is_relative_to(agents_source) + ): + frame_locals.append(traceback_object.tb_frame.f_locals) + traceback_object = traceback_object.tb_next + return frame_locals + + @pytest.mark.asyncio async def test_synchronous_realtime_handoff_clears_redacted_error_traceback( monkeypatch: pytest.MonkeyPatch, ) -> None: - secret = "SYNCHRONOUS_REALTIME_HANDOFF_TRACEBACK_SECRET" + payload_secret = "SYNCHRONOUS_REALTIME_HANDOFF_TRACEBACK_SECRET" + schema_secret = "SYNCHRONOUS_REALTIME_HANDOFF_SCHEMA_SECRET_4207" + sensitive_input_type = cast( + type[Any], + Literal["SYNCHRONOUS_REALTIME_HANDOFF_SCHEMA_SECRET_4207"], + ) monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) target = RealtimeAgent(name="target") - async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: pass # pragma: no cover - handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + handoff_obj = realtime_handoff( + target, + on_handoff=on_handoff, + input_type=sensitive_input_type, + ) agent = RealtimeAgent(name="agent", handoffs=[handoff_obj]) session = RealtimeSession( _DummyModel(), @@ -854,7 +879,7 @@ async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: event = RealtimeModelToolCallEvent( name=handoff_obj.tool_name, call_id="call-1", - arguments=f'"{secret}"', + arguments=f'"{payload_secret}"', ) with pytest.raises(ModelBehaviorError) as exc_info: @@ -862,16 +887,119 @@ async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: error = exc_info.value traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) - sdk_frames = [frame for frame in traceback_exception.stack if "/src/agents/" in frame.filename] + agents_source = (Path(__file__).parents[2] / "src" / "agents").resolve() + sdk_frames = [ + frame + for frame in traceback_exception.stack + if Path(frame.filename).resolve().is_relative_to(agents_source) + ] + assert not sdk_frames + assert payload_secret not in "".join( + value for frame in sdk_frames for value in (frame.locals or {}).values() + ) + assert schema_secret not in "".join( + value for frame in sdk_frames for value in (frame.locals or {}).values() + ) + actual_frame_locals = _realtime_sdk_traceback_frame_locals(error) + assert not actual_frame_locals + assert payload_secret not in str(error) + assert schema_secret not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + + +@pytest.mark.asyncio +async def test_async_realtime_handoff_detaches_session_from_redacted_error_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = "ASYNCHRONOUS_REALTIME_HANDOFF_TRACEBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + agent = RealtimeAgent(name="agent", handoffs=[handoff_obj]) + session = RealtimeSession(_DummyModel(), agent, None) + event = RealtimeModelToolCallEvent( + name=handoff_obj.tool_name, + call_id="call-1", + arguments=f'"{secret}"', + ) + event_iterator = session.__aiter__() + + await session.on_event(event) + raw_event = await anext(event_iterator) + assert isinstance(raw_event, RealtimeRawModelEvent) + + for _ in range(20): + if session._stored_exception is not None: + break + await asyncio.sleep(0) + assert isinstance(session._stored_exception, ModelBehaviorError) + + with pytest.raises(ModelBehaviorError) as exc_info: + await anext(event_iterator) + + error = exc_info.value + traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) + agents_source = (Path(__file__).parents[2] / "src" / "agents").resolve() + sdk_frames = [ + frame + for frame in traceback_exception.stack + if Path(frame.filename).resolve().is_relative_to(agents_source) + ] assert sdk_frames assert secret not in "".join( value for frame in sdk_frames for value in (frame.locals or {}).values() ) + actual_frame_locals = _realtime_sdk_traceback_frame_locals(error) + assert actual_frame_locals + assert all(session is not value for frame in actual_frame_locals for value in frame.values()) assert secret not in str(error) assert error.__cause__ is None assert error.__context__ is None +@pytest.mark.asyncio +async def test_synchronous_realtime_handoff_preserves_diagnostic_traceback_locals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = "SYNCHRONOUS_REALTIME_HANDOFF_DIAGNOSTIC_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + agent = RealtimeAgent(name="agent", handoffs=[handoff_obj]) + session = RealtimeSession( + _DummyModel(), + agent, + None, + run_config={"async_tool_calls": False}, + ) + event = RealtimeModelToolCallEvent( + name=handoff_obj.tool_name, + call_id="call-1", + arguments=f'"{secret}"', + ) + + with pytest.raises(ModelBehaviorError) as exc_info: + await session.on_event(event) + + error = exc_info.value + assert secret in str(error) + assert any( + frame_locals.get("event") is event + for frame_locals in _realtime_sdk_traceback_frame_locals(error) + ) + + @pytest.mark.asyncio async def test_get_handoffs_async_is_enabled(monkeypatch): # Agent includes both a direct Handoff and a RealtimeAgent (auto-converted) diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index afdccea96f..4acca6ffef 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -8,6 +8,7 @@ from __future__ import annotations +import json import logging import pickle import threading @@ -15,7 +16,7 @@ from logging.handlers import QueueHandler from pathlib import Path from queue import SimpleQueue -from typing import Any +from typing import Any, Literal, cast from unittest.mock import patch import httpx @@ -26,12 +27,16 @@ import agents._debug as _debug from agents import ( Agent, + GuardrailFunctionOutput, + InputGuardrail, ModelBehaviorError, ModelSettings, ModelTracing, OpenAIResponsesModel, + OutputGuardrail, RunConfig, RunContextWrapper, + RunErrorHandlerInput, Runner, function_tool, handoff, @@ -63,7 +68,8 @@ from agents.tracing.traces import Trace from .fake_model import FakeModel -from .test_responses import get_text_message +from .test_responses import get_function_tool_call, get_text_message +from .utils.simple_session import SimpleListSession _SECRET = "super secret prompt content" @@ -869,6 +875,10 @@ async def test_agent_tool_validation_error_preserves_diagnostics_when_tool_data_ _MODEL_OUTPUT_SECRET = "SECRET_MODEL_OUTPUT_123" +_SENSITIVE_SCHEMA_SECRET = "SENSITIVE_HANDOFF_SCHEMA_SECRET_4207" +_SENSITIVE_OUTPUT_SCHEMA_SECRET = "SENSITIVE_OUTPUT_SCHEMA_SECRET_4207" +_SensitiveHandoffInput = Literal["SENSITIVE_HANDOFF_SCHEMA_SECRET_4207"] +_SensitiveOutput = Literal["SENSITIVE_OUTPUT_SCHEMA_SECRET_4207"] class _RequiredOutput(BaseModel): @@ -876,7 +886,12 @@ class _RequiredOutput(BaseModel): count: int -def _assert_secret_absent_from_agents_traceback(error: BaseException, secret: str) -> None: +def _assert_secret_absent_from_agents_traceback( + error: BaseException, + secret: str, + *, + require_agents_frames: bool = True, +) -> None: traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) agents_source = (Path(__file__).parents[1] / "src" / "agents").resolve() agents_frames = [ @@ -884,11 +899,27 @@ def _assert_secret_absent_from_agents_traceback(error: BaseException, secret: st for frame in traceback_exception.stack if Path(frame.filename).resolve().is_relative_to(agents_source) ] - assert agents_frames + if require_agents_frames: + assert agents_frames for frame in agents_frames: assert secret not in "".join((frame.locals or {}).values()) +def _agents_traceback_frame_locals(error: BaseException) -> list[dict[str, Any]]: + agents_source = (Path(__file__).parents[1] / "src" / "agents").resolve() + frame_locals: list[dict[str, Any]] = [] + traceback_object = error.__traceback__ + while traceback_object is not None: + if ( + Path(traceback_object.tb_frame.f_code.co_filename) + .resolve() + .is_relative_to(agents_source) + ): + frame_locals.append(traceback_object.tb_frame.f_locals) + traceback_object = traceback_object.tb_next + return frame_locals + + @pytest.mark.parametrize( ("model_redacted", "tool_redacted", "expected_redacted"), [ @@ -914,10 +945,18 @@ def test_output_schema_validation_error_follows_model_data_policy( assert _MODEL_OUTPUT_SECRET not in str(error) assert error.__cause__ is None assert error.__context__ is None - _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) else: assert _MODEL_OUTPUT_SECRET in str(error) assert isinstance(error.__cause__, ValidationError) + assert any( + frame_locals.get("json_str") == payload + for frame_locals in _agents_traceback_frame_locals(error) + ) def test_output_schema_redaction_survives_trace_attachment_failure( @@ -937,7 +976,31 @@ def test_output_schema_redaction_survives_trace_attachment_failure( assert _MODEL_OUTPUT_SECRET not in str(error) assert error.__cause__ is None assert error.__context__ is None - _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + + +def test_output_schema_redaction_omits_sensitive_schema_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + output_type = cast(type[Any], _SensitiveOutput) + + with pytest.raises(ModelBehaviorError) as exc_info: + AgentOutputSchema(output_type).validate_json('"invalid"') + + error = exc_info.value + assert _SENSITIVE_OUTPUT_SCHEMA_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _SENSITIVE_OUTPUT_SCHEMA_SECRET, + require_agents_frames=False, + ) @pytest.mark.parametrize( @@ -960,23 +1023,100 @@ async def test_handoff_input_validation_error_follows_mixed_data_policy( monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' target = Agent(name="target") + handoff_calls = 0 async def on_handoff(_ctx: RunContextWrapper[Any], _input: _RequiredOutput) -> None: - pass # pragma: no cover + nonlocal handoff_calls + handoff_calls += 1 handoff_obj = handoff(target, input_type=_RequiredOutput, on_handoff=on_handoff) with pytest.raises(ModelBehaviorError) as exc_info: await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + assert handoff_calls == 0 error = exc_info.value if expected_redacted: assert _MODEL_OUTPUT_SECRET not in str(error) assert error.__cause__ is None assert error.__context__ is None - _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) else: assert _MODEL_OUTPUT_SECRET in str(error) assert isinstance(error.__cause__, ValidationError) + assert any( + frame_locals.get("input_json") == payload + for frame_locals in _agents_traceback_frame_locals(error) + ) + + +@pytest.mark.asyncio +async def test_handoff_redaction_omits_sensitive_schema_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = Agent(name="target") + handoff_calls = 0 + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: + nonlocal handoff_calls + handoff_calls += 1 + + handoff_obj = handoff( + target, + input_type=cast(type[Any], _SensitiveHandoffInput), + on_handoff=on_handoff, + ) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), '"invalid"') + + error = exc_info.value + assert handoff_calls == 0 + assert _SENSITIVE_SCHEMA_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _SENSITIVE_SCHEMA_SECRET, + require_agents_frames=False, + ) + + +@pytest.mark.asyncio +async def test_realtime_handoff_redaction_omits_sensitive_schema_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = RealtimeAgent(name="target") + handoff_calls = 0 + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: + nonlocal handoff_calls + handoff_calls += 1 + + handoff_obj = realtime_handoff( + target, + input_type=cast(type[Any], _SensitiveHandoffInput), + on_handoff=on_handoff, + ) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), '"invalid"') + + error = exc_info.value + assert handoff_calls == 0 + assert _SENSITIVE_SCHEMA_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _SENSITIVE_SCHEMA_SECRET, + require_agents_frames=False, + ) @pytest.mark.parametrize( @@ -999,23 +1139,34 @@ async def test_realtime_handoff_input_validation_error_follows_mixed_data_policy monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' target = RealtimeAgent(name="target") + handoff_calls = 0 async def on_handoff(_ctx: RunContextWrapper[Any], _input: _RequiredOutput) -> None: - pass # pragma: no cover + nonlocal handoff_calls + handoff_calls += 1 handoff_obj = realtime_handoff(target, input_type=_RequiredOutput, on_handoff=on_handoff) with pytest.raises(ModelBehaviorError) as exc_info: await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + assert handoff_calls == 0 error = exc_info.value if expected_redacted: assert _MODEL_OUTPUT_SECRET not in str(error) assert error.__cause__ is None assert error.__context__ is None - _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) else: assert _MODEL_OUTPUT_SECRET in str(error) assert isinstance(error.__cause__, ValidationError) + assert any( + frame_locals.get("input_json") == payload + for frame_locals in _agents_traceback_frame_locals(error) + ) @pytest.mark.asyncio @@ -1027,15 +1178,93 @@ async def test_run_surfaces_redacted_output_validation_error( model = FakeModel() agent = Agent(name="A", model=model, output_type=_RequiredOutput) model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + session = SimpleListSession( + session_id="redacted-run", + history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}], + ) with pytest.raises(ModelBehaviorError) as exc_info: - await Runner.run(agent, "go") + await Runner.run(agent, _MODEL_OUTPUT_SECRET, session=session) error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) assert _MODEL_OUTPUT_SECRET not in str(error) assert error.__cause__ is None assert error.__context__ is None - _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + assert all(session is not value for frame in frame_locals for value in frame.values()) + + +def test_run_sync_surfaces_redacted_output_validation_error_without_runner_data( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + model = FakeModel() + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + session = SimpleListSession( + session_id="redacted-run-sync", + history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}], + ) + + with pytest.raises(ModelBehaviorError) as exc_info: + Runner.run_sync(agent, _MODEL_OUTPUT_SECRET, session=session) + + error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + assert all(session is not value for frame in frame_locals for value in frame.values()) + + +@pytest.mark.asyncio +async def test_run_preserves_diagnostic_wrapper_traceback_locals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + diagnostic_input = "DIAGNOSTIC_RUNNER_INPUT_SECRET" + model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + session = SimpleListSession(session_id="diagnostic-runner") + + with pytest.raises(ModelBehaviorError) as exc_info: + await Runner.run(agent, diagnostic_input, session=session) + + error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) + assert any(frame.get("input") == diagnostic_input for frame in frame_locals) + assert any(frame.get("session") is session for frame in frame_locals) + + +def test_run_sync_preserves_diagnostic_wrapper_traceback_locals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + diagnostic_input = "DIAGNOSTIC_RUNNER_SYNC_INPUT_SECRET" + model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + session = SimpleListSession(session_id="diagnostic-runner-sync") + + with pytest.raises(ModelBehaviorError) as exc_info: + Runner.run_sync(agent, diagnostic_input, session=session) + + error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) + assert any(frame.get("input") == diagnostic_input for frame in frame_locals) + assert any(frame.get("session") is session for frame in frame_locals) @pytest.mark.asyncio @@ -1058,3 +1287,172 @@ async def test_streamed_run_surfaces_redacted_output_validation_error( assert error.__cause__ is None assert error.__context__ is None _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.asyncio +async def test_streamed_output_guardrail_omits_run_data_from_redacted_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + def output_guardrail( + _context: RunContextWrapper[Any], + _agent: Agent[Any], + _agent_output: Any, + ) -> GuardrailFunctionOutput: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + raise AssertionError("validation should fail") # pragma: no cover + + model = FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)]) + agent = Agent( + name="A", + model=model, + output_guardrails=[OutputGuardrail(guardrail_function=output_guardrail)], + ) + result = Runner.run_streamed(agent, "go") + + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + + error = exc_info.value + assert error.run_data is None + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.asyncio +async def test_streamed_input_guardrail_omits_run_data_from_redacted_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + def input_guardrail( + _context: RunContextWrapper[Any], + _agent: Agent[Any], + _input: str | list[Any], + ) -> GuardrailFunctionOutput: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + raise AssertionError("validation should fail") # pragma: no cover + + model = FakeModel(initial_output=[get_text_message("unused")]) + agent = Agent( + name="A", + model=model, + input_guardrails=[InputGuardrail(guardrail_function=input_guardrail)], + ) + result = Runner.run_streamed(agent, _MODEL_OUTPUT_SECRET) + + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + + error = exc_info.value + assert error.run_data is None + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_receives_detached_redacted_error( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + retained_errors: list[ModelBehaviorError] = [] + + def recover(data: RunErrorHandlerInput[None]) -> _RequiredOutput: + assert isinstance(data.error, ModelBehaviorError) + retained_errors.append(data.error) + return _RequiredOutput(answer="safe", count=1) + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": recover}, + ) + async for _ in result.stream_events(): + pass + else: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": recover}, + ) + + assert len(retained_errors) == 1 + error = retained_errors[0] + assert error.__traceback__ is None + assert error.__cause__ is None + assert error.__context__ is None + assert _MODEL_OUTPUT_SECRET not in str(error) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize("redacted", [False, True]) +@pytest.mark.asyncio +async def test_multiturn_output_validation_error_run_data_follows_redaction_policy( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + + @function_tool + def record_value(value: str) -> str: + return "recorded" + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call( + record_value.name, + json.dumps({"value": _MODEL_OUTPUT_SECRET}), + ) + ], + [get_text_message('{"answer": "missing count"}')], + ] + ) + agent = Agent( + name="A", + model=model, + tools=[record_value], + output_type=_RequiredOutput, + ) + + if streamed: + result = Runner.run_streamed(agent, "go") + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(ModelBehaviorError) as exc_info: + await Runner.run(agent, "go") + + error = exc_info.value + if redacted: + assert error.run_data is None + assert _MODEL_OUTPUT_SECRET not in str(error) + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=streamed, + ) + else: + assert error.run_data is not None + assert error.run_data.raw_responses + assert error.run_data.new_items From 19c596ab7ede62dbd8c617cb08f768a785ab181c Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Thu, 6 Aug 2026 04:30:54 +0900 Subject: [PATCH 3/4] Harden validation error data redaction --- src/agents/agent_output.py | 17 ++- src/agents/exceptions.py | 8 +- src/agents/handoffs/__init__.py | 40 ++++-- src/agents/realtime/handoffs.py | 40 ++++-- src/agents/realtime/session.py | 17 ++- src/agents/result.py | 5 +- src/agents/run.py | 42 +++++- src/agents/run_internal/error_handlers.py | 12 +- src/agents/run_internal/turn_resolution.py | 90 +++++++++--- tests/realtime/test_session.py | 9 +- tests/test_error_logging_redaction.py | 157 ++++++++++++++++++++- tests/test_invalid_final_output_handler.py | 6 +- 12 files changed, 375 insertions(+), 68 deletions(-) diff --git a/src/agents/agent_output.py b/src/agents/agent_output.py index 758b9dfba9..32df9cb712 100644 --- a/src/agents/agent_output.py +++ b/src/agents/agent_output.py @@ -1,6 +1,6 @@ import abc from dataclasses import dataclass -from typing import Any, get_args, get_origin +from typing import Any, cast, get_args, get_origin from pydantic import BaseModel, TypeAdapter from typing_extensions import TypedDict @@ -10,6 +10,7 @@ UserError, _detach_data_redacted_error_traceback, _is_error_data_redacted, + _raise_data_redacted_error, ) from .strict_schema import ensure_strict_json_schema from .tracing import SpanError @@ -142,6 +143,7 @@ def validate_json(self, json_str: str) -> Any: """Validate a JSON string against the output type. Returns the validated object, or raises a `ModelBehaviorError` if the JSON is invalid. """ + redacted_error: ModelBehaviorError | None = None try: validated = _json.validate_json( json_str, @@ -150,10 +152,15 @@ def validate_json(self, json_str: str) -> Any: strict=True if self._strict_json_schema else None, ) except ModelBehaviorError as error: - if _is_error_data_redacted(error): - json_str = "" - _detach_data_redacted_error_traceback(error) - raise + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + if redacted_error is not None: + self = cast(Any, None) + json_str = "" + _raise_data_redacted_error(redacted_error) if self._is_wrapped: if not isinstance(validated, dict): _error_tracing.attach_error_to_current_span( diff --git a/src/agents/exceptions.py b/src/agents/exceptions.py index e5e2fb8195..887ea910ba 100644 --- a/src/agents/exceptions.py +++ b/src/agents/exceptions.py @@ -2,7 +2,7 @@ import traceback from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NoReturn if TYPE_CHECKING: from .agent import Agent @@ -21,6 +21,7 @@ _DRAIN_STREAM_EVENTS_ATTR = "_agents_drain_queued_stream_events" _DATA_REDACTED_ATTR = "_agents_data_redacted" +_DATA_REDACTED_ERROR_MESSAGE = "Error details are redacted." def _mark_error_to_drain_stream_events(error: Exception) -> None: @@ -49,6 +50,11 @@ def _detach_data_redacted_error_traceback(error: Exception) -> None: error.__traceback__ = None +def _raise_data_redacted_error(error: Exception) -> NoReturn: + """Raise a detached redacted error from a frame that owns no payload data.""" + raise error from None + + @dataclass class RunErrorDetails: """Data collected from an agent run when an exception occurs.""" diff --git a/src/agents/handoffs/__init__.py b/src/agents/handoffs/__init__.py index 11b28a1cdc..727f708324 100644 --- a/src/agents/handoffs/__init__.py +++ b/src/agents/handoffs/__init__.py @@ -15,6 +15,7 @@ UserError, _detach_data_redacted_error_traceback, _is_error_data_redacted, + _raise_data_redacted_error, ) from ..items import RunItem, TResponseInputItem from ..run_context import RunContextWrapper, TContext @@ -287,7 +288,7 @@ def handoff( if len(sig.parameters) != 1: raise UserError("on_handoff must take one argument: context") - async def _invoke_handoff( + async def _invoke_handoff_impl( ctx: RunContextWrapper[Any], input_json: str | None = None ) -> Agent[TContext]: if input_type is not None and type_adapter is not None: @@ -300,19 +301,13 @@ async def _invoke_handoff( ) raise ModelBehaviorError("Handoff function expected non-null input, but got None") - try: - validated_input = _json.validate_json( - json_str=input_json, - type_adapter=type_adapter, - partial=False, - strict=True, - contains_tool_data=True, - ) - except ModelBehaviorError as error: - if _is_error_data_redacted(error): - input_json = "" - _detach_data_redacted_error_traceback(error) - raise + validated_input = _json.validate_json( + json_str=input_json, + type_adapter=type_adapter, + partial=False, + strict=True, + contains_tool_data=True, + ) input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) if inspect.isawaitable(result): @@ -325,6 +320,23 @@ async def _invoke_handoff( return agent + async def _invoke_handoff( + ctx: RunContextWrapper[Any], input_json: str | None = None + ) -> Agent[TContext]: + redacted_error: ModelBehaviorError | None = None + try: + return await _invoke_handoff_impl(ctx, input_json) + except ModelBehaviorError as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + ctx = cast(Any, None) + input_json = "" + assert redacted_error is not None + _raise_data_redacted_error(redacted_error) + tool_name = tool_name_override or Handoff.default_tool_name(agent) tool_description = tool_description_override or Handoff.default_tool_description(agent) diff --git a/src/agents/realtime/handoffs.py b/src/agents/realtime/handoffs.py index e333403467..8f99b81567 100644 --- a/src/agents/realtime/handoffs.py +++ b/src/agents/realtime/handoffs.py @@ -12,6 +12,7 @@ UserError, _detach_data_redacted_error_traceback, _is_error_data_redacted, + _raise_data_redacted_error, ) from ..handoffs import Handoff from ..run_context import RunContextWrapper, TContext @@ -151,7 +152,7 @@ def realtime_handoff( if len(sig.parameters) != 1: raise UserError("on_handoff must take one argument: context") - async def _invoke_handoff( + async def _invoke_handoff_impl( ctx: RunContextWrapper[Any], input_json: str | None = None ) -> RealtimeAgent[TContext]: if input_type is not None and type_adapter is not None: @@ -164,19 +165,13 @@ async def _invoke_handoff( ) raise ModelBehaviorError("Handoff function expected non-null input, but got None") - try: - validated_input = _json.validate_json( - json_str=input_json, - type_adapter=type_adapter, - partial=False, - strict=True, - contains_tool_data=True, - ) - except ModelBehaviorError as error: - if _is_error_data_redacted(error): - input_json = "" - _detach_data_redacted_error_traceback(error) - raise + validated_input = _json.validate_json( + json_str=input_json, + type_adapter=type_adapter, + partial=False, + strict=True, + contains_tool_data=True, + ) input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) if inspect.isawaitable(result): @@ -189,6 +184,23 @@ async def _invoke_handoff( return agent + async def _invoke_handoff( + ctx: RunContextWrapper[Any], input_json: str | None = None + ) -> RealtimeAgent[TContext]: + redacted_error: ModelBehaviorError | None = None + try: + return await _invoke_handoff_impl(ctx, input_json) + except ModelBehaviorError as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + ctx = cast(Any, None) + input_json = "" + assert redacted_error is not None + _raise_data_redacted_error(redacted_error) + tool_name = tool_name_override or Handoff.default_tool_name(agent) tool_description = tool_description_override or Handoff.default_tool_description(agent) diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index f7417efec7..7d6265a96c 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -25,6 +25,7 @@ _clear_data_redacted_error_traceback, _detach_data_redacted_error_traceback, _is_error_data_redacted, + _raise_data_redacted_error, ) from ..handoffs import Handoff from ..items import ToolApprovalItem @@ -414,12 +415,22 @@ async def on_event(self, event: RealtimeModelEvent) -> None: handle_kwargs: dict[str, Any] = {"agent_snapshot": agent_snapshot} if dispatch_snapshot is not None: handle_kwargs["dispatch_snapshot"] = dispatch_snapshot + redacted_error: ModelBehaviorError | None = None try: await self._handle_tool_call(event, **handle_kwargs) except ModelBehaviorError as error: - if _is_error_data_redacted(error): - _detach_data_redacted_error_traceback(error) - raise + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + if redacted_error is not None: + self = cast(Any, None) + event = cast(Any, None) + agent_snapshot = cast(Any, None) + dispatch_snapshot = cast(Any, None) + handle_kwargs = {} + _raise_data_redacted_error(redacted_error) elif event.type == "audio": if event.response_id not in self._interrupted_response_ids: await self._put_event( diff --git a/src/agents/result.py b/src/agents/result.py index 7ebc03f15e..afd4990393 100644 --- a/src/agents/result.py +++ b/src/agents/result.py @@ -731,7 +731,10 @@ def run_loop_exception(self) -> BaseException | None: task = self.run_loop_task if task is None or not task.done() or task.cancelled(): return None - return task.exception() + error = task.exception() + if isinstance(error, Exception) and _is_error_data_redacted(error): + _detach_data_redacted_error_traceback(error) + return error def cancel(self, mode: Literal["immediate", "after_turn"] = "immediate") -> None: """Cancel the streaming run. diff --git a/src/agents/run.py b/src/agents/run.py index a1fd583267..68b0d51cd7 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -279,6 +279,7 @@ async def run( """ runner = DEFAULT_AGENT_RUNNER + redacted_error: AgentsException | None = None try: return await runner.run( starting_agent, @@ -293,9 +294,25 @@ async def run( conversation_id=conversation_id, session=session, ) - except ModelBehaviorError as error: + except AgentsException as error: + if not _is_error_data_redacted(error): + raise _detach_data_redacted_error_traceback(error) - raise + redacted_error = error + + starting_agent = cast(Any, None) + input = cast(Any, None) + context = cast(Any, None) + hooks = cast(Any, None) + run_config = cast(Any, None) + error_handlers = cast(Any, None) + previous_response_id = None + auto_previous_response_id = cast(Any, None) + conversation_id = None + session = cast(Any, None) + runner = cast(Any, None) + assert redacted_error is not None + raise redacted_error from None @classmethod def run_sync( @@ -365,6 +382,7 @@ def run_sync( """ runner = DEFAULT_AGENT_RUNNER + redacted_error: AgentsException | None = None try: return runner.run_sync( starting_agent, @@ -379,9 +397,25 @@ def run_sync( session=session, auto_previous_response_id=auto_previous_response_id, ) - except ModelBehaviorError as error: + except AgentsException as error: + if not _is_error_data_redacted(error): + raise _detach_data_redacted_error_traceback(error) - raise + redacted_error = error + + starting_agent = cast(Any, None) + input = cast(Any, None) + context = cast(Any, None) + hooks = cast(Any, None) + run_config = cast(Any, None) + error_handlers = cast(Any, None) + previous_response_id = None + auto_previous_response_id = cast(Any, None) + conversation_id = None + session = cast(Any, None) + runner = cast(Any, None) + assert redacted_error is not None + raise redacted_error from None @classmethod def run_streamed( diff --git a/src/agents/run_internal/error_handlers.py b/src/agents/run_internal/error_handlers.py index 8c30f54d95..63745133df 100644 --- a/src/agents/run_internal/error_handlers.py +++ b/src/agents/run_internal/error_handlers.py @@ -159,7 +159,12 @@ def format_final_output_text(agent: Agent[Any], final_output: Any) -> str: return str(final_output) -def validate_handler_final_output(agent: Agent[Any], final_output: Any) -> Any: +def validate_handler_final_output( + agent: Agent[Any], + final_output: Any, + *, + data_redacted: bool = False, +) -> Any: output_schema = get_output_schema(agent) if output_schema is None or output_schema.is_plain_text(): return final_output @@ -171,7 +176,10 @@ def validate_handler_final_output(agent: Agent[Any], final_output: Any) -> Any: payload_value = {_WRAPPER_DICT_KEY: final_output} try: if isinstance(output_schema, AgentOutputSchema): - payload_bytes = output_schema._type_adapter.dump_json(payload_value) + payload_bytes = output_schema._type_adapter.dump_json( + payload_value, + warnings="none" if data_redacted else "warn", + ) payload = ( payload_bytes.decode() if isinstance(payload_bytes, bytes | bytearray) diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index f3099e6677..cae3f5943c 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -55,10 +55,13 @@ record_agent_tool_run_result, ) from ..exceptions import ( + _DATA_REDACTED_ERROR_MESSAGE, ModelBehaviorError, ModelRefusalError, UserError, _detach_data_redacted_error_traceback, + _is_error_data_redacted, + _mark_error_data_redacted, ) from ..handoffs import Handoff, HandoffInputData, HandoffInputFilter, nest_handoff_history from ..handoffs.history import ( @@ -420,6 +423,7 @@ async def _resolve_invalid_final_output( new_items: list[RunItem], context_wrapper: RunContextWrapper[TContext], ) -> tuple[Any, MessageOutputItem | None] | None: + redacted = _is_error_data_redacted(error) run_error_data = build_run_error_data( input=original_input, new_items=new_items, @@ -427,26 +431,54 @@ async def _resolve_invalid_final_output( last_agent=public_agent, ) _detach_data_redacted_error_traceback(error) - handler_result = await resolve_run_error_handler_result( - error_handlers=error_handlers, - error_kind="invalid_final_output", - error=error, - context_wrapper=context_wrapper, - run_data=run_error_data, - ) - if handler_result is None: - return None + safe_error: UserError | None = None + try: + handler_result = await resolve_run_error_handler_result( + error_handlers=error_handlers, + error_kind="invalid_final_output", + error=error, + context_wrapper=context_wrapper, + run_data=run_error_data, + ) + if handler_result is None: + return None - final_output = validate_handler_final_output(public_agent, handler_result.final_output) - message_item = ( - create_message_output_item( + final_output = validate_handler_final_output( public_agent, - format_final_output_text(public_agent, final_output), + handler_result.final_output, + data_redacted=redacted, ) - if handler_result.include_in_history - else None - ) - return final_output, message_item + message_item = ( + create_message_output_item( + public_agent, + format_final_output_text(public_agent, final_output), + ) + if handler_result.include_in_history + else None + ) + return final_output, message_item + except Exception as handler_error: + if not redacted: + raise + handler_error.__traceback__ = None + handler_error.__cause__ = None + handler_error.__context__ = None + safe_error = UserError(_DATA_REDACTED_ERROR_MESSAGE) + _mark_error_data_redacted(safe_error) + + error = cast(Any, None) + error_handlers = cast(Any, None) + public_agent = cast(Any, None) + original_input = cast(Any, None) + new_response = cast(Any, None) + new_items = cast(Any, None) + context_wrapper = cast(Any, None) + run_error_data = cast(Any, None) + handler_result = cast(Any, None) + final_output = cast(Any, None) + message_item = cast(Any, None) + assert safe_error is not None + raise safe_error from None def _resolve_server_managed_handoff_behavior( @@ -908,12 +940,32 @@ async def execute_tools_and_side_effects( ) if output_schema and not output_schema.is_plain_text(): if potential_final_output_text: + validation_error: ModelBehaviorError | None = None try: final_output = output_schema.validate_json(potential_final_output_text) except ModelBehaviorError as error: + if _is_error_data_redacted(error): + validation_error = error + else: + resolved_handler_output = await _resolve_invalid_final_output( + error_handlers=error_handlers, + error=error, + public_agent=public_agent, + original_input=original_input, + new_response=new_response, + new_items=pre_step_items + new_step_items, + context_wrapper=context_wrapper, + ) + if resolved_handler_output is None: + raise + final_output, message_item = resolved_handler_output + if message_item is not None: + new_step_items.append(message_item) + + if validation_error is not None: resolved_handler_output = await _resolve_invalid_final_output( error_handlers=error_handlers, - error=error, + error=validation_error, public_agent=public_agent, original_input=original_input, new_response=new_response, @@ -921,7 +973,7 @@ async def execute_tools_and_side_effects( context_wrapper=context_wrapper, ) if resolved_handler_output is None: - raise + raise validation_error final_output, message_item = resolved_handler_output if message_item is not None: new_step_items.append(message_item) diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index f5b1756b2d..a52b89f241 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -893,7 +893,7 @@ async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: for frame in traceback_exception.stack if Path(frame.filename).resolve().is_relative_to(agents_source) ] - assert not sdk_frames + assert sdk_frames assert payload_secret not in "".join( value for frame in sdk_frames for value in (frame.locals or {}).values() ) @@ -901,7 +901,12 @@ async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: value for frame in sdk_frames for value in (frame.locals or {}).values() ) actual_frame_locals = _realtime_sdk_traceback_frame_locals(error) - assert not actual_frame_locals + assert actual_frame_locals + assert all( + value is not session and value is not event + for frame in actual_frame_locals + for value in frame.values() + ) assert payload_secret not in str(error) assert schema_secret not in str(error) assert error.__cause__ is None diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index 4acca6ffef..13570f21c1 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -8,11 +8,13 @@ from __future__ import annotations +import asyncio import json import logging import pickle import threading import traceback +import warnings from logging.handlers import QueueHandler from pathlib import Path from queue import SimpleQueue @@ -38,6 +40,7 @@ RunContextWrapper, RunErrorHandlerInput, Runner, + UserError, function_tool, handoff, trace, @@ -1194,7 +1197,6 @@ async def test_run_surfaces_redacted_output_validation_error( _assert_secret_absent_from_agents_traceback( error, _MODEL_OUTPUT_SECRET, - require_agents_frames=False, ) assert all(session is not value for frame in frame_locals for value in frame.values()) @@ -1223,7 +1225,6 @@ def test_run_sync_surfaces_redacted_output_validation_error_without_runner_data( _assert_secret_absent_from_agents_traceback( error, _MODEL_OUTPUT_SECRET, - require_agents_frames=False, ) assert all(session is not value for frame in frame_locals for value in frame.values()) @@ -1289,6 +1290,41 @@ async def test_streamed_run_surfaces_redacted_output_validation_error( _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) +@pytest.mark.parametrize("redacted", [False, True]) +@pytest.mark.asyncio +async def test_streamed_run_loop_exception_follows_model_data_policy( + monkeypatch: pytest.MonkeyPatch, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + result = Runner.run_streamed(agent, "go") + + assert result.run_loop_task is not None + while not result.run_loop_task.done(): + await asyncio.sleep(0) + + error = result.run_loop_exception + assert isinstance(error, ModelBehaviorError) + if redacted: + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + assert any( + _MODEL_OUTPUT_SECRET in repr(frame_locals) + for frame_locals in _agents_traceback_frame_locals(error) + ) + + @pytest.mark.asyncio async def test_streamed_output_guardrail_omits_run_data_from_redacted_error( monkeypatch: pytest.MonkeyPatch, @@ -1400,6 +1436,123 @@ def recover(data: RunErrorHandlerInput[None]) -> _RequiredOutput: assert _MODEL_OUTPUT_SECRET not in str(error) +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_invalid_fallback_preserves_redaction( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + fallback_secret = "INVALID_HANDLER_FALLBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def invalid_fallback(_data: RunErrorHandlerInput[None]) -> dict[str, str]: + return {"answer": fallback_secret} + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": invalid_fallback}, + ) + with pytest.raises(UserError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(UserError) as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": invalid_fallback}, + ) + + error = exc_info.value + assert not caught_warnings + assert str(error) == "Error details are redacted." + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback(error, fallback_secret) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_failure_preserves_redaction( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def fail(data: RunErrorHandlerInput[None]) -> None: + raise RuntimeError(repr(data.run_data.raw_responses)) + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + with pytest.raises(UserError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(UserError) as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + + error = exc_info.value + assert str(error) == "Error details are redacted." + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_failure_preserves_diagnostic_context( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def fail(_data: RunErrorHandlerInput[None]) -> None: + raise RuntimeError("handler failed") + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + with pytest.raises(RuntimeError, match="handler failed") as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(RuntimeError, match="handler failed") as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + + validation_error = exc_info.value.__context__ + assert isinstance(validation_error, ModelBehaviorError) + assert _MODEL_OUTPUT_SECRET in str(validation_error) + assert isinstance(validation_error.__cause__, ValidationError) + + @pytest.mark.parametrize("streamed", [False, True]) @pytest.mark.parametrize("redacted", [False, True]) @pytest.mark.asyncio diff --git a/tests/test_invalid_final_output_handler.py b/tests/test_invalid_final_output_handler.py index b4519debdc..667d9a746d 100644 --- a/tests/test_invalid_final_output_handler.py +++ b/tests/test_invalid_final_output_handler.py @@ -7,6 +7,7 @@ from openai.types.responses import ResponseOutputMessage from pydantic import BaseModel +import agents._debug as _debug from agents import ( Agent, AgentHookContext, @@ -114,7 +115,10 @@ async def test_invalid_final_output_handler_can_skip_fallback_history() -> None: @pytest.mark.asyncio -async def test_invalid_final_output_handler_rejects_invalid_fallback() -> None: +async def test_invalid_final_output_handler_rejects_invalid_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) model = FakeModel(initial_output=[get_text_message("not valid json")]) agent = Agent(name="test", model=model, output_type=FinalOutput) From f812591a176286dae5352e5db375cf129fe95d45 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Thu, 6 Aug 2026 08:17:11 +0900 Subject: [PATCH 4/4] Harden redacted recovery boundaries --- src/agents/handoffs/__init__.py | 41 +++-- src/agents/realtime/handoffs.py | 25 +-- src/agents/run_internal/error_handlers.py | 12 +- src/agents/run_internal/turn_resolution.py | 13 +- tests/test_error_logging_redaction.py | 193 ++++++++++++++++++++- 5 files changed, 235 insertions(+), 49 deletions(-) diff --git a/src/agents/handoffs/__init__.py b/src/agents/handoffs/__init__.py index 727f708324..17eaed1de7 100644 --- a/src/agents/handoffs/__init__.py +++ b/src/agents/handoffs/__init__.py @@ -5,6 +5,7 @@ import weakref from collections.abc import Awaitable, Callable from dataclasses import dataclass, field, replace as dataclasses_replace +from functools import partial from typing import TYPE_CHECKING, Any, Generic, TypeAlias, cast, overload from pydantic import TypeAdapter @@ -45,6 +46,27 @@ OnHandoffWithoutInput = Callable[[RunContextWrapper[Any]], Any] +async def _invoke_handoff_with_redaction( + invoke_handoff: Callable[[RunContextWrapper[Any], str | None], Awaitable[TAgent]], + ctx: RunContextWrapper[Any], + input_json: str | None = None, +) -> TAgent: + redacted_error: ModelBehaviorError | None = None + try: + return await invoke_handoff(ctx, input_json) + except ModelBehaviorError as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + invoke_handoff = cast(Any, None) + ctx = cast(Any, None) + input_json = "" + assert redacted_error is not None + _raise_data_redacted_error(redacted_error) + + @dataclass(frozen=True) class HandoffInputData: input_history: str | tuple[TResponseInputItem, ...] @@ -320,23 +342,6 @@ async def _invoke_handoff_impl( return agent - async def _invoke_handoff( - ctx: RunContextWrapper[Any], input_json: str | None = None - ) -> Agent[TContext]: - redacted_error: ModelBehaviorError | None = None - try: - return await _invoke_handoff_impl(ctx, input_json) - except ModelBehaviorError as error: - if not _is_error_data_redacted(error): - raise - _detach_data_redacted_error_traceback(error) - redacted_error = error - - ctx = cast(Any, None) - input_json = "" - assert redacted_error is not None - _raise_data_redacted_error(redacted_error) - tool_name = tool_name_override or Handoff.default_tool_name(agent) tool_description = tool_description_override or Handoff.default_tool_description(agent) @@ -358,7 +363,7 @@ async def _is_enabled(ctx: RunContextWrapper[Any], agent_base: AgentBase[Any]) - tool_name=tool_name, tool_description=tool_description, input_json_schema=input_json_schema, - on_invoke_handoff=_invoke_handoff, + on_invoke_handoff=partial(_invoke_handoff_with_redaction, _invoke_handoff_impl), input_filter=input_filter, nest_handoff_history=nest_handoff_history, agent_name=agent.name, diff --git a/src/agents/realtime/handoffs.py b/src/agents/realtime/handoffs.py index 8f99b81567..3373e24cbb 100644 --- a/src/agents/realtime/handoffs.py +++ b/src/agents/realtime/handoffs.py @@ -2,6 +2,7 @@ import inspect from collections.abc import Callable, Iterable +from functools import partial from typing import TYPE_CHECKING, Any, cast, overload from pydantic import TypeAdapter @@ -10,11 +11,8 @@ from ..exceptions import ( ModelBehaviorError, UserError, - _detach_data_redacted_error_traceback, - _is_error_data_redacted, - _raise_data_redacted_error, ) -from ..handoffs import Handoff +from ..handoffs import Handoff, _invoke_handoff_with_redaction from ..run_context import RunContextWrapper, TContext from ..strict_schema import ensure_strict_json_schema from ..tracing.spans import SpanError @@ -184,23 +182,6 @@ async def _invoke_handoff_impl( return agent - async def _invoke_handoff( - ctx: RunContextWrapper[Any], input_json: str | None = None - ) -> RealtimeAgent[TContext]: - redacted_error: ModelBehaviorError | None = None - try: - return await _invoke_handoff_impl(ctx, input_json) - except ModelBehaviorError as error: - if not _is_error_data_redacted(error): - raise - _detach_data_redacted_error_traceback(error) - redacted_error = error - - ctx = cast(Any, None) - input_json = "" - assert redacted_error is not None - _raise_data_redacted_error(redacted_error) - tool_name = tool_name_override or Handoff.default_tool_name(agent) tool_description = tool_description_override or Handoff.default_tool_description(agent) @@ -220,7 +201,7 @@ async def _is_enabled(ctx: RunContextWrapper[Any], agent_base: AgentBase[Any]) - tool_name=tool_name, tool_description=tool_description, input_json_schema=input_json_schema, - on_invoke_handoff=_invoke_handoff, + on_invoke_handoff=partial(_invoke_handoff_with_redaction, _invoke_handoff_impl), input_filter=None, # Not supported for RealtimeAgent handoffs agent_name=agent.name, is_enabled=_is_enabled if callable(is_enabled) else is_enabled, diff --git a/src/agents/run_internal/error_handlers.py b/src/agents/run_internal/error_handlers.py index 63745133df..f55e8b9929 100644 --- a/src/agents/run_internal/error_handlers.py +++ b/src/agents/run_internal/error_handlers.py @@ -136,7 +136,12 @@ def build_run_error_data( ) -def format_final_output_text(agent: Agent[Any], final_output: Any) -> str: +def format_final_output_text( + agent: Agent[Any], + final_output: Any, + *, + data_redacted: bool = False, +) -> str: output_schema = get_output_schema(agent) if output_schema is None or output_schema.is_plain_text(): return str(final_output) @@ -148,7 +153,10 @@ def format_final_output_text(agent: Agent[Any], final_output: Any) -> str: payload_value = {_WRAPPER_DICT_KEY: final_output} try: if isinstance(output_schema, AgentOutputSchema): - payload_bytes = output_schema._type_adapter.dump_json(payload_value) + payload_bytes = output_schema._type_adapter.dump_json( + payload_value, + warnings="none" if data_redacted else "warn", + ) return ( payload_bytes.decode() if isinstance(payload_bytes, bytes | bytearray) diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index cae3f5943c..2302a9d1a5 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -423,7 +423,7 @@ async def _resolve_invalid_final_output( new_items: list[RunItem], context_wrapper: RunContextWrapper[TContext], ) -> tuple[Any, MessageOutputItem | None] | None: - redacted = _is_error_data_redacted(error) + redacted = _is_error_data_redacted(error) or _debug.DONT_LOG_MODEL_DATA run_error_data = build_run_error_data( input=original_input, new_items=new_items, @@ -451,18 +451,19 @@ async def _resolve_invalid_final_output( message_item = ( create_message_output_item( public_agent, - format_final_output_text(public_agent, final_output), + format_final_output_text( + public_agent, + final_output, + data_redacted=redacted, + ), ) if handler_result.include_in_history else None ) return final_output, message_item - except Exception as handler_error: + except Exception: if not redacted: raise - handler_error.__traceback__ = None - handler_error.__cause__ = None - handler_error.__context__ = None safe_error = UserError(_DATA_REDACTED_ERROR_MESSAGE) _mark_error_data_redacted(safe_error) diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index 13570f21c1..821907a4ce 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -24,7 +24,7 @@ import httpx import pytest from openai import AsyncOpenAI -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, SkipValidation, ValidationError import agents._debug as _debug from agents import ( @@ -39,6 +39,7 @@ RunConfig, RunContextWrapper, RunErrorHandlerInput, + RunErrorHandlerResult, Runner, UserError, function_tool, @@ -99,6 +100,11 @@ def __getattribute__(self, name: str): return super().__getattribute__(name) +class _HostileAttributeWriteException(Exception): + def __setattr__(self, name: str, value: Any) -> None: + raise RuntimeError("redacted handling mutated the handler exception") + + class _TruthinessException(Exception): def __init__(self, *, truthy: bool) -> None: super().__init__("diagnostic failure") @@ -889,6 +895,11 @@ class _RequiredOutput(BaseModel): count: int +class _PermissiveFallbackOutput(BaseModel): + payload: SkipValidation[str] + count: int + + def _assert_secret_absent_from_agents_traceback( error: BaseException, secret: str, @@ -923,6 +934,22 @@ def _agents_traceback_frame_locals(error: BaseException) -> list[dict[str, Any]] return frame_locals +def _assert_handoff_closure_absent_from_traceback( + error: BaseException, + *, + callback: Any, + schema_secret: str, +) -> None: + for frame_locals in _agents_traceback_frame_locals(error): + for value in frame_locals.values(): + closure = getattr(value, "__closure__", None) + if closure is None: + continue + closure_values = [cell.cell_contents for cell in closure] + assert all(item is not callback for item in closure_values) + assert schema_secret not in repr(closure_values) + + @pytest.mark.parametrize( ("model_redacted", "tool_redacted", "expected_redacted"), [ @@ -1087,6 +1114,11 @@ async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: _SENSITIVE_SCHEMA_SECRET, require_agents_frames=False, ) + _assert_handoff_closure_absent_from_traceback( + error, + callback=on_handoff, + schema_secret=_SENSITIVE_SCHEMA_SECRET, + ) @pytest.mark.asyncio @@ -1120,6 +1152,11 @@ async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: _SENSITIVE_SCHEMA_SECRET, require_agents_frames=False, ) + _assert_handoff_closure_absent_from_traceback( + error, + callback=on_handoff, + schema_secret=_SENSITIVE_SCHEMA_SECRET, + ) @pytest.mark.parametrize( @@ -1479,6 +1516,120 @@ def invalid_fallback(_data: RunErrorHandlerInput[None]) -> dict[str, str]: _assert_secret_absent_from_agents_traceback(error, fallback_secret) +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize("redacted", [True, False]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_fallback_serialization_follows_redaction_policy( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, + redacted: bool, +) -> None: + fallback_secret = "PERMISSIVE_HANDLER_FALLBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + model = FakeModel( + initial_output=[ + get_text_message(f'{{"payload": "{_MODEL_OUTPUT_SECRET}", "count": "invalid"}}') + ] + ) + agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput) + + def permissive_fallback(_data: RunErrorHandlerInput[None]) -> _PermissiveFallbackOutput: + return _PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + if streamed: + streaming_result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + async for _ in streaming_result.stream_events(): + pass + actual_final_output = streaming_result.final_output + else: + run_result = await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + actual_final_output = run_result.final_output + + rendered_warnings = "\n".join(str(warning.message) for warning in caught_warnings) + if redacted: + assert not caught_warnings + else: + assert fallback_secret in rendered_warnings + assert actual_final_output == _PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ) + + +@pytest.mark.parametrize( + ("streamed", "include_in_history", "redacted"), + [ + (False, True, True), + (False, False, True), + (True, True, True), + (True, False, True), + (False, True, False), + ], +) +@pytest.mark.asyncio +async def test_empty_final_output_handler_fallback_serialization_follows_redaction_policy( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, + include_in_history: bool, + redacted: bool, +) -> None: + fallback_secret = "EMPTY_HANDLER_FALLBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + model = FakeModel(initial_output=[]) + agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput) + + def permissive_fallback(_data: RunErrorHandlerInput[None]) -> RunErrorHandlerResult: + return RunErrorHandlerResult( + final_output=_PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ), + include_in_history=include_in_history, + ) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + if streamed: + streaming_result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + async for _ in streaming_result.stream_events(): + pass + actual_final_output = streaming_result.final_output + else: + run_result = await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + actual_final_output = run_result.final_output + + rendered_warnings = "\n".join(str(warning.message) for warning in caught_warnings) + if redacted: + assert not caught_warnings + else: + assert fallback_secret in rendered_warnings + assert actual_final_output == _PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ) + + @pytest.mark.parametrize("streamed", [False, True]) @pytest.mark.asyncio async def test_invalid_final_output_handler_failure_preserves_redaction( @@ -1517,6 +1668,46 @@ def fail(data: RunErrorHandlerInput[None]) -> None: _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_hostile_failure_preserves_redaction( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + handler_secret = "HOSTILE_HANDLER_FAILURE_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def fail(_data: RunErrorHandlerInput[None]) -> None: + raise _HostileAttributeWriteException(handler_secret) + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + with pytest.raises(UserError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(UserError) as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + + error = exc_info.value + assert str(error) == "Error details are redacted." + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback(error, handler_secret) + + @pytest.mark.parametrize("streamed", [False, True]) @pytest.mark.asyncio async def test_invalid_final_output_handler_failure_preserves_diagnostic_context(