diff --git a/src/agents/agent_output.py b/src/agents/agent_output.py index f2274280b0..32df9cb712 100644 --- a/src/agents/agent_output.py +++ b/src/agents/agent_output.py @@ -1,11 +1,17 @@ import abc from dataclasses import dataclass -from typing import Any, get_args, get_origin +from typing import Any, cast, get_args, get_origin from pydantic import BaseModel, TypeAdapter from typing_extensions import TypedDict -from .exceptions import ModelBehaviorError, UserError +from .exceptions import ( + ModelBehaviorError, + UserError, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, + _raise_data_redacted_error, +) from .strict_schema import ensure_strict_json_schema from .tracing import SpanError from .util import _error_tracing, _json @@ -137,12 +143,24 @@ def validate_json(self, json_str: str) -> Any: """Validate a JSON string against the output type. Returns the validated object, or raises a `ModelBehaviorError` if the JSON is invalid. """ - validated = _json.validate_json( - json_str, - self._type_adapter, - partial=False, - strict=True if self._strict_json_schema else None, - ) + redacted_error: ModelBehaviorError | None = None + try: + validated = _json.validate_json( + json_str, + self._type_adapter, + partial=False, + strict=True if self._strict_json_schema else None, + ) + except ModelBehaviorError as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + if redacted_error is not None: + self = cast(Any, None) + json_str = "" + _raise_data_redacted_error(redacted_error) if self._is_wrapped: if not isinstance(validated, dict): _error_tracing.attach_error_to_current_span( diff --git a/src/agents/exceptions.py b/src/agents/exceptions.py index 349004c97d..887ea910ba 100644 --- a/src/agents/exceptions.py +++ b/src/agents/exceptions.py @@ -1,7 +1,8 @@ from __future__ import annotations +import traceback from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NoReturn if TYPE_CHECKING: from .agent import Agent @@ -19,6 +20,8 @@ from .util._pretty_print import pretty_print_run_error_details _DRAIN_STREAM_EVENTS_ATTR = "_agents_drain_queued_stream_events" +_DATA_REDACTED_ATTR = "_agents_data_redacted" +_DATA_REDACTED_ERROR_MESSAGE = "Error details are redacted." def _mark_error_to_drain_stream_events(error: Exception) -> None: @@ -29,6 +32,29 @@ def _should_drain_stream_events_before_raising(error: Exception) -> bool: return bool(getattr(error, _DRAIN_STREAM_EVENTS_ATTR, False)) +def _mark_error_data_redacted(error: Exception) -> None: + setattr(error, _DATA_REDACTED_ATTR, True) + + +def _is_error_data_redacted(error: Exception) -> bool: + return bool(getattr(error, _DATA_REDACTED_ATTR, False)) + + +def _clear_data_redacted_error_traceback(error: Exception) -> None: + if _is_error_data_redacted(error) and error.__traceback__ is not None: + traceback.clear_frames(error.__traceback__) + + +def _detach_data_redacted_error_traceback(error: Exception) -> None: + if _is_error_data_redacted(error): + error.__traceback__ = None + + +def _raise_data_redacted_error(error: Exception) -> NoReturn: + """Raise a detached redacted error from a frame that owns no payload data.""" + raise error from None + + @dataclass class RunErrorDetails: """Data collected from an agent run when an exception occurs.""" diff --git a/src/agents/handoffs/__init__.py b/src/agents/handoffs/__init__.py index 79d1841760..17eaed1de7 100644 --- a/src/agents/handoffs/__init__.py +++ b/src/agents/handoffs/__init__.py @@ -5,12 +5,19 @@ import weakref from collections.abc import Awaitable, Callable from dataclasses import dataclass, field, replace as dataclasses_replace +from functools import partial from typing import TYPE_CHECKING, Any, Generic, TypeAlias, cast, overload from pydantic import TypeAdapter from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError, UserError +from ..exceptions import ( + ModelBehaviorError, + UserError, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, + _raise_data_redacted_error, +) from ..items import RunItem, TResponseInputItem from ..run_context import RunContextWrapper, TContext from ..strict_schema import ensure_strict_json_schema @@ -39,6 +46,27 @@ OnHandoffWithoutInput = Callable[[RunContextWrapper[Any]], Any] +async def _invoke_handoff_with_redaction( + invoke_handoff: Callable[[RunContextWrapper[Any], str | None], Awaitable[TAgent]], + ctx: RunContextWrapper[Any], + input_json: str | None = None, +) -> TAgent: + redacted_error: ModelBehaviorError | None = None + try: + return await invoke_handoff(ctx, input_json) + except ModelBehaviorError as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + invoke_handoff = cast(Any, None) + ctx = cast(Any, None) + input_json = "" + assert redacted_error is not None + _raise_data_redacted_error(redacted_error) + + @dataclass(frozen=True) class HandoffInputData: input_history: str | tuple[TResponseInputItem, ...] @@ -282,7 +310,7 @@ def handoff( if len(sig.parameters) != 1: raise UserError("on_handoff must take one argument: context") - async def _invoke_handoff( + async def _invoke_handoff_impl( ctx: RunContextWrapper[Any], input_json: str | None = None ) -> Agent[TContext]: if input_type is not None and type_adapter is not None: @@ -300,6 +328,7 @@ async def _invoke_handoff( type_adapter=type_adapter, partial=False, strict=True, + contains_tool_data=True, ) input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) @@ -334,7 +363,7 @@ async def _is_enabled(ctx: RunContextWrapper[Any], agent_base: AgentBase[Any]) - tool_name=tool_name, tool_description=tool_description, input_json_schema=input_json_schema, - on_invoke_handoff=_invoke_handoff, + on_invoke_handoff=partial(_invoke_handoff_with_redaction, _invoke_handoff_impl), input_filter=input_filter, nest_handoff_history=nest_handoff_history, agent_name=agent.name, diff --git a/src/agents/realtime/handoffs.py b/src/agents/realtime/handoffs.py index a2026772ee..3373e24cbb 100644 --- a/src/agents/realtime/handoffs.py +++ b/src/agents/realtime/handoffs.py @@ -2,13 +2,17 @@ import inspect from collections.abc import Callable, Iterable +from functools import partial from typing import TYPE_CHECKING, Any, cast, overload from pydantic import TypeAdapter from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError, UserError -from ..handoffs import Handoff +from ..exceptions import ( + ModelBehaviorError, + UserError, +) +from ..handoffs import Handoff, _invoke_handoff_with_redaction from ..run_context import RunContextWrapper, TContext from ..strict_schema import ensure_strict_json_schema from ..tracing.spans import SpanError @@ -146,7 +150,7 @@ def realtime_handoff( if len(sig.parameters) != 1: raise UserError("on_handoff must take one argument: context") - async def _invoke_handoff( + async def _invoke_handoff_impl( ctx: RunContextWrapper[Any], input_json: str | None = None ) -> RealtimeAgent[TContext]: if input_type is not None and type_adapter is not None: @@ -164,6 +168,7 @@ async def _invoke_handoff( type_adapter=type_adapter, partial=False, strict=True, + contains_tool_data=True, ) input_func = cast(OnHandoffWithInput[THandoffInput], on_handoff) result = input_func(ctx, validated_input) @@ -196,7 +201,7 @@ async def _is_enabled(ctx: RunContextWrapper[Any], agent_base: AgentBase[Any]) - tool_name=tool_name, tool_description=tool_description, input_json_schema=input_json_schema, - on_invoke_handoff=_invoke_handoff, + on_invoke_handoff=partial(_invoke_handoff_with_redaction, _invoke_handoff_impl), input_filter=None, # Not supported for RealtimeAgent handoffs agent_name=agent.name, is_enabled=_is_enabled if callable(is_enabled) else is_enabled, diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index f224bbf068..7d6265a96c 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -18,7 +18,15 @@ get_function_tool_namespace, ) from ..agent import Agent -from ..exceptions import ToolInputGuardrailTripwireTriggered, UserError +from ..exceptions import ( + ModelBehaviorError, + ToolInputGuardrailTripwireTriggered, + UserError, + _clear_data_redacted_error_traceback, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, + _raise_data_redacted_error, +) from ..handoffs import Handoff from ..items import ToolApprovalItem from ..logger import ( @@ -304,7 +312,16 @@ async def __aiter__(self) -> AsyncIterator[RealtimeSessionEvent]: if self._stored_exception is not None: # Clean up resources before raising await self.close() - raise self._stored_exception + stored_exception = self._stored_exception + if isinstance(stored_exception, Exception) and _is_error_data_redacted( + stored_exception + ): + _detach_data_redacted_error_traceback(stored_exception) + # Do not retain the session or the previously yielded raw event in the + # traceback frame that exposes a redacted error to the caller. + self = cast(Any, None) + event = cast(Any, None) + raise stored_exception self._event_iterator_waiters += 1 try: @@ -398,7 +415,22 @@ async def on_event(self, event: RealtimeModelEvent) -> None: handle_kwargs: dict[str, Any] = {"agent_snapshot": agent_snapshot} if dispatch_snapshot is not None: handle_kwargs["dispatch_snapshot"] = dispatch_snapshot - await self._handle_tool_call(event, **handle_kwargs) + redacted_error: ModelBehaviorError | None = None + try: + await self._handle_tool_call(event, **handle_kwargs) + except ModelBehaviorError as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + if redacted_error is not None: + self = cast(Any, None) + event = cast(Any, None) + agent_snapshot = cast(Any, None) + dispatch_snapshot = cast(Any, None) + handle_kwargs = {} + _raise_data_redacted_error(redacted_error) elif event.type == "audio": if event.response_id not in self._interrupted_response_ids: await self._put_event( @@ -1625,14 +1657,16 @@ def _enqueue_tool_call_task( def _on_tool_call_task_done(self, task: asyncio.Task[Any]) -> None: self._tool_call_tasks.discard(task) - if self._closing or self._closed: - self._consume_task_result(task) - return - if task.cancelled(): return exception = task.exception() + if isinstance(exception, ModelBehaviorError): + _clear_data_redacted_error_traceback(exception) + + if self._closing or self._closed: + return + if exception is None: return diff --git a/src/agents/result.py b/src/agents/result.py index 6482cd2813..afd4990393 100644 --- a/src/agents/result.py +++ b/src/agents/result.py @@ -18,6 +18,8 @@ InputGuardrailTripwireTriggered, MaxTurnsExceeded, RunErrorDetails, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, _should_drain_stream_events_before_raising, ) from .guardrail import InputGuardrailResult, OutputGuardrailResult @@ -729,7 +731,10 @@ def run_loop_exception(self) -> BaseException | None: task = self.run_loop_task if task is None or not task.done() or task.cancelled(): return None - return task.exception() + error = task.exception() + if isinstance(error, Exception) and _is_error_data_redacted(error): + _detach_data_redacted_error_traceback(error) + return error def cancel(self, mode: Literal["immediate", "after_turn"] = "immediate") -> None: """Cancel the streaming run. @@ -927,8 +932,16 @@ def register_current_consumer() -> None: self._drain_event_queue() self._drain_input_guardrail_queue() - if self._stored_exception: - raise self._stored_exception + stored_exception = self._stored_exception + if stored_exception: + if _is_error_data_redacted(stored_exception): + _detach_data_redacted_error_traceback(stored_exception) + # The streaming result retains caller-visible run data. Drop the local reference + # before raising so the redacted exception cannot retain it through this frame. + self = cast(Any, None) + registered_consumer_task = None + item = cast(Any, None) + raise stored_exception def _create_error_details(self) -> RunErrorDetails | None: """Return a `RunErrorDetails` object considering the current attributes of the class. @@ -974,7 +987,11 @@ def _check_errors(self): if not self.run_loop_task.cancelled(): run_impl_exc = self.run_loop_task.exception() if run_impl_exc and isinstance(run_impl_exc, Exception): - if isinstance(run_impl_exc, AgentsException) and run_impl_exc.run_data is None: + if ( + isinstance(run_impl_exc, AgentsException) + and run_impl_exc.run_data is None + and not _is_error_data_redacted(run_impl_exc) + ): run_impl_exc.run_data = self._create_error_details() self._stored_exception = run_impl_exc @@ -982,7 +999,11 @@ def _check_errors(self): if not self._input_guardrails_task.cancelled(): in_guard_exc = self._input_guardrails_task.exception() if in_guard_exc and isinstance(in_guard_exc, Exception): - if isinstance(in_guard_exc, AgentsException) and in_guard_exc.run_data is None: + if ( + isinstance(in_guard_exc, AgentsException) + and in_guard_exc.run_data is None + and not _is_error_data_redacted(in_guard_exc) + ): in_guard_exc.run_data = self._create_error_details() self._stored_exception = in_guard_exc @@ -993,6 +1014,7 @@ def _check_errors(self): if ( isinstance(out_guard_exc, AgentsException) and out_guard_exc.run_data is None + and not _is_error_data_redacted(out_guard_exc) ): out_guard_exc.run_data = self._create_error_details() self._stored_exception = out_guard_exc diff --git a/src/agents/run.py b/src/agents/run.py index 00028cf406..68b0d51cd7 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -14,9 +14,13 @@ AgentsException, InputGuardrailTripwireTriggered, MaxTurnsExceeded, + ModelBehaviorError, OutputGuardrailTripwireTriggered, RunErrorDetails, UserError, + _clear_data_redacted_error_traceback, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, ) from .guardrail import ( InputGuardrailResult, @@ -275,19 +279,40 @@ async def run( """ runner = DEFAULT_AGENT_RUNNER - return await runner.run( - starting_agent, - input, - context=context, - max_turns=max_turns, - hooks=hooks, - run_config=run_config, - error_handlers=error_handlers, - previous_response_id=previous_response_id, - auto_previous_response_id=auto_previous_response_id, - conversation_id=conversation_id, - session=session, - ) + redacted_error: AgentsException | None = None + try: + return await runner.run( + starting_agent, + input, + context=context, + max_turns=max_turns, + hooks=hooks, + run_config=run_config, + error_handlers=error_handlers, + previous_response_id=previous_response_id, + auto_previous_response_id=auto_previous_response_id, + conversation_id=conversation_id, + session=session, + ) + except AgentsException as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + starting_agent = cast(Any, None) + input = cast(Any, None) + context = cast(Any, None) + hooks = cast(Any, None) + run_config = cast(Any, None) + error_handlers = cast(Any, None) + previous_response_id = None + auto_previous_response_id = cast(Any, None) + conversation_id = None + session = cast(Any, None) + runner = cast(Any, None) + assert redacted_error is not None + raise redacted_error from None @classmethod def run_sync( @@ -357,19 +382,40 @@ def run_sync( """ runner = DEFAULT_AGENT_RUNNER - return runner.run_sync( - starting_agent, - input, - context=context, - max_turns=max_turns, - hooks=hooks, - run_config=run_config, - error_handlers=error_handlers, - previous_response_id=previous_response_id, - conversation_id=conversation_id, - session=session, - auto_previous_response_id=auto_previous_response_id, - ) + redacted_error: AgentsException | None = None + try: + return runner.run_sync( + starting_agent, + input, + context=context, + max_turns=max_turns, + hooks=hooks, + run_config=run_config, + error_handlers=error_handlers, + previous_response_id=previous_response_id, + conversation_id=conversation_id, + session=session, + auto_previous_response_id=auto_previous_response_id, + ) + except AgentsException as error: + if not _is_error_data_redacted(error): + raise + _detach_data_redacted_error_traceback(error) + redacted_error = error + + starting_agent = cast(Any, None) + input = cast(Any, None) + context = cast(Any, None) + hooks = cast(Any, None) + run_config = cast(Any, None) + error_handlers = cast(Any, None) + previous_response_id = None + auto_previous_response_id = cast(Any, None) + conversation_id = None + session = cast(Any, None) + runner = cast(Any, None) + assert redacted_error is not None + raise redacted_error from None @classmethod def run_streamed( @@ -1621,17 +1667,21 @@ def _finalize_result(result: RunResult) -> RunResult: trace_include_sensitive_data=run_config.trace_include_sensitive_data, ) if isinstance(exc, AgentsException): - exc.run_data = RunErrorDetails( - input=original_input, - new_items=session_items, - raw_responses=model_responses, - last_agent=current_agent, - context_wrapper=context_wrapper, - input_guardrail_results=input_guardrail_results, - output_guardrail_results=output_guardrail_results, - tool_input_guardrail_results=tool_input_guardrail_results, - tool_output_guardrail_results=tool_output_guardrail_results, - ) + if _is_error_data_redacted(exc): + _detach_data_redacted_error_traceback(exc) + else: + _clear_data_redacted_error_traceback(exc) + exc.run_data = RunErrorDetails( + input=original_input, + new_items=session_items, + raw_responses=model_responses, + last_agent=current_agent, + context_wrapper=context_wrapper, + input_guardrail_results=input_guardrail_results, + output_guardrail_results=output_guardrail_results, + tool_input_guardrail_results=tool_input_guardrail_results, + tool_output_guardrail_results=tool_output_guardrail_results, + ) raise finally: await cleanup_models_after_run(tool_use_tracker) @@ -1756,13 +1806,15 @@ def run_sync( try: # Drive the coroutine to completion, harvesting the final RunResult. return default_loop.run_until_complete(task) - except BaseException: + except BaseException as error: # If the sync caller aborts (KeyboardInterrupt, etc.), make sure the scheduled task # does not linger on the shared loop by cancelling it and waiting for completion. if not task.done(): task.cancel() with contextlib.suppress(asyncio.CancelledError): default_loop.run_until_complete(task) + if isinstance(error, ModelBehaviorError): + _detach_data_redacted_error_traceback(error) raise finally: if not default_loop.is_closed(): diff --git a/src/agents/run_internal/error_handlers.py b/src/agents/run_internal/error_handlers.py index 8c30f54d95..f55e8b9929 100644 --- a/src/agents/run_internal/error_handlers.py +++ b/src/agents/run_internal/error_handlers.py @@ -136,7 +136,12 @@ def build_run_error_data( ) -def format_final_output_text(agent: Agent[Any], final_output: Any) -> str: +def format_final_output_text( + agent: Agent[Any], + final_output: Any, + *, + data_redacted: bool = False, +) -> str: output_schema = get_output_schema(agent) if output_schema is None or output_schema.is_plain_text(): return str(final_output) @@ -148,7 +153,10 @@ def format_final_output_text(agent: Agent[Any], final_output: Any) -> str: payload_value = {_WRAPPER_DICT_KEY: final_output} try: if isinstance(output_schema, AgentOutputSchema): - payload_bytes = output_schema._type_adapter.dump_json(payload_value) + payload_bytes = output_schema._type_adapter.dump_json( + payload_value, + warnings="none" if data_redacted else "warn", + ) return ( payload_bytes.decode() if isinstance(payload_bytes, bytes | bytearray) @@ -159,7 +167,12 @@ def format_final_output_text(agent: Agent[Any], final_output: Any) -> str: return str(final_output) -def validate_handler_final_output(agent: Agent[Any], final_output: Any) -> Any: +def validate_handler_final_output( + agent: Agent[Any], + final_output: Any, + *, + data_redacted: bool = False, +) -> Any: output_schema = get_output_schema(agent) if output_schema is None or output_schema.is_plain_text(): return final_output @@ -171,7 +184,10 @@ def validate_handler_final_output(agent: Agent[Any], final_output: Any) -> Any: payload_value = {_WRAPPER_DICT_KEY: final_output} try: if isinstance(output_schema, AgentOutputSchema): - payload_bytes = output_schema._type_adapter.dump_json(payload_value) + payload_bytes = output_schema._type_adapter.dump_json( + payload_value, + warnings="none" if data_redacted else "warn", + ) payload = ( payload_bytes.decode() if isinstance(payload_bytes, bytes | bytearray) diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 643238d914..80ec1928f3 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -40,6 +40,9 @@ OutputGuardrailTripwireTriggered, RunErrorDetails, UserError, + _clear_data_redacted_error_traceback, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, ) from ..handoffs import Handoff from ..items import ( @@ -1387,17 +1390,21 @@ async def _save_stream_items_without_count( except AgentsException as exc: streamed_result.is_complete = True streamed_result._event_queue.put_nowait(QueueCompleteSentinel()) - exc.run_data = RunErrorDetails( - input=streamed_result.input, - new_items=streamed_result.new_items, - raw_responses=streamed_result.raw_responses, - last_agent=current_agent, - context_wrapper=context_wrapper, - input_guardrail_results=streamed_result.input_guardrail_results, - output_guardrail_results=streamed_result.output_guardrail_results, - tool_input_guardrail_results=streamed_result.tool_input_guardrail_results, - tool_output_guardrail_results=streamed_result.tool_output_guardrail_results, - ) + if _is_error_data_redacted(exc): + _detach_data_redacted_error_traceback(exc) + else: + _clear_data_redacted_error_traceback(exc) + exc.run_data = RunErrorDetails( + input=streamed_result.input, + new_items=streamed_result.new_items, + raw_responses=streamed_result.raw_responses, + last_agent=current_agent, + context_wrapper=context_wrapper, + input_guardrail_results=streamed_result.input_guardrail_results, + output_guardrail_results=streamed_result.output_guardrail_results, + tool_input_guardrail_results=streamed_result.tool_input_guardrail_results, + tool_output_guardrail_results=streamed_result.tool_output_guardrail_results, + ) raise except Exception as e: attach_generic_agent_error( diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 5bf9839533..2302a9d1a5 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -54,7 +54,15 @@ peek_agent_tool_run_result, record_agent_tool_run_result, ) -from ..exceptions import ModelBehaviorError, ModelRefusalError, UserError +from ..exceptions import ( + _DATA_REDACTED_ERROR_MESSAGE, + ModelBehaviorError, + ModelRefusalError, + UserError, + _detach_data_redacted_error_traceback, + _is_error_data_redacted, + _mark_error_data_redacted, +) from ..handoffs import Handoff, HandoffInputData, HandoffInputFilter, nest_handoff_history from ..handoffs.history import ( _get_nested_history_owned_items, @@ -415,32 +423,63 @@ async def _resolve_invalid_final_output( new_items: list[RunItem], context_wrapper: RunContextWrapper[TContext], ) -> tuple[Any, MessageOutputItem | None] | None: + redacted = _is_error_data_redacted(error) or _debug.DONT_LOG_MODEL_DATA run_error_data = build_run_error_data( input=original_input, new_items=new_items, raw_responses=[new_response], last_agent=public_agent, ) - handler_result = await resolve_run_error_handler_result( - error_handlers=error_handlers, - error_kind="invalid_final_output", - error=error, - context_wrapper=context_wrapper, - run_data=run_error_data, - ) - if handler_result is None: - return None + _detach_data_redacted_error_traceback(error) + safe_error: UserError | None = None + try: + handler_result = await resolve_run_error_handler_result( + error_handlers=error_handlers, + error_kind="invalid_final_output", + error=error, + context_wrapper=context_wrapper, + run_data=run_error_data, + ) + if handler_result is None: + return None - final_output = validate_handler_final_output(public_agent, handler_result.final_output) - message_item = ( - create_message_output_item( + final_output = validate_handler_final_output( public_agent, - format_final_output_text(public_agent, final_output), + handler_result.final_output, + data_redacted=redacted, ) - if handler_result.include_in_history - else None - ) - return final_output, message_item + message_item = ( + create_message_output_item( + public_agent, + format_final_output_text( + public_agent, + final_output, + data_redacted=redacted, + ), + ) + if handler_result.include_in_history + else None + ) + return final_output, message_item + except Exception: + if not redacted: + raise + safe_error = UserError(_DATA_REDACTED_ERROR_MESSAGE) + _mark_error_data_redacted(safe_error) + + error = cast(Any, None) + error_handlers = cast(Any, None) + public_agent = cast(Any, None) + original_input = cast(Any, None) + new_response = cast(Any, None) + new_items = cast(Any, None) + context_wrapper = cast(Any, None) + run_error_data = cast(Any, None) + handler_result = cast(Any, None) + final_output = cast(Any, None) + message_item = cast(Any, None) + assert safe_error is not None + raise safe_error from None def _resolve_server_managed_handoff_behavior( @@ -902,12 +941,32 @@ async def execute_tools_and_side_effects( ) if output_schema and not output_schema.is_plain_text(): if potential_final_output_text: + validation_error: ModelBehaviorError | None = None try: final_output = output_schema.validate_json(potential_final_output_text) except ModelBehaviorError as error: + if _is_error_data_redacted(error): + validation_error = error + else: + resolved_handler_output = await _resolve_invalid_final_output( + error_handlers=error_handlers, + error=error, + public_agent=public_agent, + original_input=original_input, + new_response=new_response, + new_items=pre_step_items + new_step_items, + context_wrapper=context_wrapper, + ) + if resolved_handler_output is None: + raise + final_output, message_item = resolved_handler_output + if message_item is not None: + new_step_items.append(message_item) + + if validation_error is not None: resolved_handler_output = await _resolve_invalid_final_output( error_handlers=error_handlers, - error=error, + error=validation_error, public_agent=public_agent, original_input=original_input, new_response=new_response, @@ -915,7 +974,7 @@ async def execute_tools_and_side_effects( context_wrapper=context_wrapper, ) if resolved_handler_output is None: - raise + raise validation_error final_output, message_item = resolved_handler_output if message_item is not None: new_step_items.append(message_item) diff --git a/src/agents/util/_json.py b/src/agents/util/_json.py index 67186328cd..fd944b6129 100644 --- a/src/agents/util/_json.py +++ b/src/agents/util/_json.py @@ -6,7 +6,8 @@ from pydantic import TypeAdapter, ValidationError from typing_extensions import TypeVar -from ..exceptions import ModelBehaviorError +from .. import _debug +from ..exceptions import ModelBehaviorError, _mark_error_data_redacted from ..tracing import SpanError from ._error_tracing import attach_error_to_current_span @@ -14,8 +15,14 @@ def validate_json( - json_str: str, type_adapter: TypeAdapter[T], partial: bool, strict: bool | None = None + json_str: str, + type_adapter: TypeAdapter[T], + partial: bool, + strict: bool | None = None, + *, + contains_tool_data: bool = False, ) -> T: + should_redact = _debug.DONT_LOG_MODEL_DATA or (contains_tool_data and _debug.DONT_LOG_TOOL_DATA) partial_setting: bool | Literal["off", "on", "trailing-strings"] = ( "trailing-strings" if partial else False ) @@ -26,15 +33,33 @@ def validate_json( validated = type_adapter.validate_json(json_str, **kwargs) return validated except ValidationError as e: + if not should_redact: + attach_error_to_current_span( + SpanError( + message="Invalid JSON provided", + data={}, + ) + ) + raise ModelBehaviorError( + f"Invalid JSON when parsing {json_str} for {type_adapter}; {e}" + ) from e + + # Clear the payload before creating the redacted traceback frame. Raising outside the except + # block also prevents the payload-bearing ValidationError from becoming the cause or context. + json_str = "" + error = ModelBehaviorError("Invalid JSON when parsing model output") + _mark_error_data_redacted(error) + # Redacted error reporting is best-effort so tracing failures cannot replace the safe error. + try: attach_error_to_current_span( SpanError( message="Invalid JSON provided", data={}, ) ) - raise ModelBehaviorError( - f"Invalid JSON when parsing {json_str} for {type_adapter}; {e}" - ) from e + except Exception: + pass + raise error def _to_dump_compatible(obj: Any) -> Any: diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index 9a6309fcc5..a52b89f241 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -3,7 +3,9 @@ import json import logging import threading -from typing import Any, cast +import traceback +from pathlib import Path +from typing import Any, Literal, cast from unittest.mock import AsyncMock, Mock, PropertyMock, patch import pytest @@ -11,9 +13,10 @@ import agents._debug as _debug from agents.agent import AgentBase -from agents.exceptions import ToolTimeoutError, UserError +from agents.exceptions import ModelBehaviorError, ToolTimeoutError, UserError from agents.guardrail import GuardrailFunctionOutput, OutputGuardrail from agents.handoffs import Handoff +from agents.realtime import realtime_handoff from agents.realtime.agent import RealtimeAgent from agents.realtime.config import RealtimeRunConfig, RealtimeSessionModelSettings from agents.realtime.events import ( @@ -780,6 +783,228 @@ async def failing_task() -> None: assert err.error["message"] == expected_message +@pytest.mark.asyncio +@pytest.mark.parametrize("state_name", [None, "_closing", "_closed"]) +async def test_on_tool_call_task_done_clears_redacted_error_traceback( + monkeypatch: pytest.MonkeyPatch, + state_name: str | None, +) -> None: + secret = "REALTIME_HANDOFF_TRACEBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + session = RealtimeSession(_DummyModel(), RealtimeAgent(name="agent"), None) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + + async def failing_task() -> None: + payload = f'"{secret}"' + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + + task = asyncio.create_task(failing_task()) + await asyncio.gather(task, return_exceptions=True) + error = task.exception() + assert isinstance(error, ModelBehaviorError) + + before = traceback.TracebackException.from_exception(error, capture_locals=True) + assert secret in "".join( + value for frame in before.stack for value in (frame.locals or {}).values() + ) + + if state_name is not None: + setattr(session, state_name, True) + session._on_tool_call_task_done(task) + + after = traceback.TracebackException.from_exception(error, capture_locals=True) + assert secret not in "".join( + value for frame in after.stack for value in (frame.locals or {}).values() + ) + if state_name is None: + assert session._stored_exception is error + event = session._event_queue.get_nowait() + assert isinstance(event, RealtimeError) + assert secret not in event.error["message"] + else: + assert session._stored_exception is None + assert session._event_queue.empty() + + +def _realtime_sdk_traceback_frame_locals(error: BaseException) -> list[dict[str, Any]]: + agents_source = (Path(__file__).parents[2] / "src" / "agents").resolve() + frame_locals: list[dict[str, Any]] = [] + traceback_object = error.__traceback__ + while traceback_object is not None: + if ( + Path(traceback_object.tb_frame.f_code.co_filename) + .resolve() + .is_relative_to(agents_source) + ): + frame_locals.append(traceback_object.tb_frame.f_locals) + traceback_object = traceback_object.tb_next + return frame_locals + + +@pytest.mark.asyncio +async def test_synchronous_realtime_handoff_clears_redacted_error_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload_secret = "SYNCHRONOUS_REALTIME_HANDOFF_TRACEBACK_SECRET" + schema_secret = "SYNCHRONOUS_REALTIME_HANDOFF_SCHEMA_SECRET_4207" + sensitive_input_type = cast( + type[Any], + Literal["SYNCHRONOUS_REALTIME_HANDOFF_SCHEMA_SECRET_4207"], + ) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff( + target, + on_handoff=on_handoff, + input_type=sensitive_input_type, + ) + agent = RealtimeAgent(name="agent", handoffs=[handoff_obj]) + session = RealtimeSession( + _DummyModel(), + agent, + None, + run_config={"async_tool_calls": False}, + ) + event = RealtimeModelToolCallEvent( + name=handoff_obj.tool_name, + call_id="call-1", + arguments=f'"{payload_secret}"', + ) + + with pytest.raises(ModelBehaviorError) as exc_info: + await session.on_event(event) + + error = exc_info.value + traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) + agents_source = (Path(__file__).parents[2] / "src" / "agents").resolve() + sdk_frames = [ + frame + for frame in traceback_exception.stack + if Path(frame.filename).resolve().is_relative_to(agents_source) + ] + assert sdk_frames + assert payload_secret not in "".join( + value for frame in sdk_frames for value in (frame.locals or {}).values() + ) + assert schema_secret not in "".join( + value for frame in sdk_frames for value in (frame.locals or {}).values() + ) + actual_frame_locals = _realtime_sdk_traceback_frame_locals(error) + assert actual_frame_locals + assert all( + value is not session and value is not event + for frame in actual_frame_locals + for value in frame.values() + ) + assert payload_secret not in str(error) + assert schema_secret not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + + +@pytest.mark.asyncio +async def test_async_realtime_handoff_detaches_session_from_redacted_error_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = "ASYNCHRONOUS_REALTIME_HANDOFF_TRACEBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + agent = RealtimeAgent(name="agent", handoffs=[handoff_obj]) + session = RealtimeSession(_DummyModel(), agent, None) + event = RealtimeModelToolCallEvent( + name=handoff_obj.tool_name, + call_id="call-1", + arguments=f'"{secret}"', + ) + event_iterator = session.__aiter__() + + await session.on_event(event) + raw_event = await anext(event_iterator) + assert isinstance(raw_event, RealtimeRawModelEvent) + + for _ in range(20): + if session._stored_exception is not None: + break + await asyncio.sleep(0) + assert isinstance(session._stored_exception, ModelBehaviorError) + + with pytest.raises(ModelBehaviorError) as exc_info: + await anext(event_iterator) + + error = exc_info.value + traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) + agents_source = (Path(__file__).parents[2] / "src" / "agents").resolve() + sdk_frames = [ + frame + for frame in traceback_exception.stack + if Path(frame.filename).resolve().is_relative_to(agents_source) + ] + assert sdk_frames + assert secret not in "".join( + value for frame in sdk_frames for value in (frame.locals or {}).values() + ) + actual_frame_locals = _realtime_sdk_traceback_frame_locals(error) + assert actual_frame_locals + assert all(session is not value for frame in actual_frame_locals for value in frame.values()) + assert secret not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + + +@pytest.mark.asyncio +async def test_synchronous_realtime_handoff_preserves_diagnostic_traceback_locals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = "SYNCHRONOUS_REALTIME_HANDOFF_DIAGNOSTIC_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + target = RealtimeAgent(name="target") + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: int) -> None: + pass # pragma: no cover + + handoff_obj = realtime_handoff(target, on_handoff=on_handoff, input_type=int) + agent = RealtimeAgent(name="agent", handoffs=[handoff_obj]) + session = RealtimeSession( + _DummyModel(), + agent, + None, + run_config={"async_tool_calls": False}, + ) + event = RealtimeModelToolCallEvent( + name=handoff_obj.tool_name, + call_id="call-1", + arguments=f'"{secret}"', + ) + + with pytest.raises(ModelBehaviorError) as exc_info: + await session.on_event(event) + + error = exc_info.value + assert secret in str(error) + assert any( + frame_locals.get("event") is event + for frame_locals in _realtime_sdk_traceback_frame_locals(error) + ) + + @pytest.mark.asyncio async def test_get_handoffs_async_is_enabled(monkeypatch): # Agent includes both a direct Handoff and a RealtimeAgent (auto-converted) diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index b194c6f41b..821907a4ce 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -8,32 +8,45 @@ from __future__ import annotations +import asyncio +import json import logging import pickle import threading +import traceback +import warnings from logging.handlers import QueueHandler from pathlib import Path from queue import SimpleQueue -from typing import Any +from typing import Any, Literal, cast from unittest.mock import patch import httpx import pytest from openai import AsyncOpenAI -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, SkipValidation, ValidationError import agents._debug as _debug from agents import ( Agent, + GuardrailFunctionOutput, + InputGuardrail, ModelBehaviorError, ModelSettings, ModelTracing, OpenAIResponsesModel, + OutputGuardrail, RunConfig, RunContextWrapper, + RunErrorHandlerInput, + RunErrorHandlerResult, + Runner, + UserError, function_tool, + handoff, trace, ) +from agents.agent_output import AgentOutputSchema from agents.logger import ( log_model_action_debug, log_model_action_error, @@ -46,6 +59,7 @@ log_tool_action_error as log_shared_tool_action_error, log_tool_action_warning, ) +from agents.realtime import RealtimeAgent, realtime_handoff from agents.run_internal.tool_execution import ( log_tool_action_error, resolve_approval_rejection_message, @@ -57,6 +71,10 @@ from agents.tracing.spans import Span from agents.tracing.traces import Trace +from .fake_model import FakeModel +from .test_responses import get_function_tool_call, get_text_message +from .utils.simple_session import SimpleListSession + _SECRET = "super secret prompt content" @@ -82,6 +100,11 @@ def __getattribute__(self, name: str): return super().__getattribute__(name) +class _HostileAttributeWriteException(Exception): + def __setattr__(self, name: str, value: Any) -> None: + raise RuntimeError("redacted handling mutated the handler exception") + + class _TruthinessException(Exception): def __init__(self, *, truthy: bool) -> None: super().__init__("diagnostic failure") @@ -858,3 +881,922 @@ async def test_agent_tool_validation_error_preserves_diagnostics_when_tool_data_ error = exc_info.value assert _TOOL_ARGUMENT_SECRET in str(error) assert isinstance(error.__cause__, ValidationError) + + +_MODEL_OUTPUT_SECRET = "SECRET_MODEL_OUTPUT_123" +_SENSITIVE_SCHEMA_SECRET = "SENSITIVE_HANDOFF_SCHEMA_SECRET_4207" +_SENSITIVE_OUTPUT_SCHEMA_SECRET = "SENSITIVE_OUTPUT_SCHEMA_SECRET_4207" +_SensitiveHandoffInput = Literal["SENSITIVE_HANDOFF_SCHEMA_SECRET_4207"] +_SensitiveOutput = Literal["SENSITIVE_OUTPUT_SCHEMA_SECRET_4207"] + + +class _RequiredOutput(BaseModel): + answer: str + count: int + + +class _PermissiveFallbackOutput(BaseModel): + payload: SkipValidation[str] + count: int + + +def _assert_secret_absent_from_agents_traceback( + error: BaseException, + secret: str, + *, + require_agents_frames: bool = True, +) -> None: + traceback_exception = traceback.TracebackException.from_exception(error, capture_locals=True) + agents_source = (Path(__file__).parents[1] / "src" / "agents").resolve() + agents_frames = [ + frame + for frame in traceback_exception.stack + if Path(frame.filename).resolve().is_relative_to(agents_source) + ] + if require_agents_frames: + assert agents_frames + for frame in agents_frames: + assert secret not in "".join((frame.locals or {}).values()) + + +def _agents_traceback_frame_locals(error: BaseException) -> list[dict[str, Any]]: + agents_source = (Path(__file__).parents[1] / "src" / "agents").resolve() + frame_locals: list[dict[str, Any]] = [] + traceback_object = error.__traceback__ + while traceback_object is not None: + if ( + Path(traceback_object.tb_frame.f_code.co_filename) + .resolve() + .is_relative_to(agents_source) + ): + frame_locals.append(traceback_object.tb_frame.f_locals) + traceback_object = traceback_object.tb_next + return frame_locals + + +def _assert_handoff_closure_absent_from_traceback( + error: BaseException, + *, + callback: Any, + schema_secret: str, +) -> None: + for frame_locals in _agents_traceback_frame_locals(error): + for value in frame_locals.values(): + closure = getattr(value, "__closure__", None) + if closure is None: + continue + closure_values = [cell.cell_contents for cell in closure] + assert all(item is not callback for item in closure_values) + assert schema_secret not in repr(closure_values) + + +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "expected_redacted"), + [ + (True, False, True), + (False, True, False), + ], +) +def test_output_schema_validation_error_follows_model_data_policy( + monkeypatch: pytest.MonkeyPatch, + model_redacted: bool, + tool_redacted: bool, + expected_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + with pytest.raises(ModelBehaviorError) as exc_info: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + + error = exc_info.value + if expected_redacted: + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + assert any( + frame_locals.get("json_str") == payload + for frame_locals in _agents_traceback_frame_locals(error) + ) + + +def test_output_schema_redaction_survives_trace_attachment_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + with patch( + "agents.util._json.attach_error_to_current_span", + side_effect=RuntimeError("trace attachment failed"), + ): + with pytest.raises(ModelBehaviorError) as exc_info: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + + error = exc_info.value + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + + +def test_output_schema_redaction_omits_sensitive_schema_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + output_type = cast(type[Any], _SensitiveOutput) + + with pytest.raises(ModelBehaviorError) as exc_info: + AgentOutputSchema(output_type).validate_json('"invalid"') + + error = exc_info.value + assert _SENSITIVE_OUTPUT_SCHEMA_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _SENSITIVE_OUTPUT_SCHEMA_SECRET, + require_agents_frames=False, + ) + + +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "expected_redacted"), + [ + (True, False, True), + (False, True, True), + (True, True, True), + (False, False, False), + ], +) +@pytest.mark.asyncio +async def test_handoff_input_validation_error_follows_mixed_data_policy( + monkeypatch: pytest.MonkeyPatch, + model_redacted: bool, + tool_redacted: bool, + expected_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + target = Agent(name="target") + handoff_calls = 0 + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: _RequiredOutput) -> None: + nonlocal handoff_calls + handoff_calls += 1 + + handoff_obj = handoff(target, input_type=_RequiredOutput, on_handoff=on_handoff) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + + assert handoff_calls == 0 + error = exc_info.value + if expected_redacted: + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + assert any( + frame_locals.get("input_json") == payload + for frame_locals in _agents_traceback_frame_locals(error) + ) + + +@pytest.mark.asyncio +async def test_handoff_redaction_omits_sensitive_schema_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = Agent(name="target") + handoff_calls = 0 + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: + nonlocal handoff_calls + handoff_calls += 1 + + handoff_obj = handoff( + target, + input_type=cast(type[Any], _SensitiveHandoffInput), + on_handoff=on_handoff, + ) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), '"invalid"') + + error = exc_info.value + assert handoff_calls == 0 + assert _SENSITIVE_SCHEMA_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _SENSITIVE_SCHEMA_SECRET, + require_agents_frames=False, + ) + _assert_handoff_closure_absent_from_traceback( + error, + callback=on_handoff, + schema_secret=_SENSITIVE_SCHEMA_SECRET, + ) + + +@pytest.mark.asyncio +async def test_realtime_handoff_redaction_omits_sensitive_schema_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + target = RealtimeAgent(name="target") + handoff_calls = 0 + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: Any) -> None: + nonlocal handoff_calls + handoff_calls += 1 + + handoff_obj = realtime_handoff( + target, + input_type=cast(type[Any], _SensitiveHandoffInput), + on_handoff=on_handoff, + ) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), '"invalid"') + + error = exc_info.value + assert handoff_calls == 0 + assert _SENSITIVE_SCHEMA_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _SENSITIVE_SCHEMA_SECRET, + require_agents_frames=False, + ) + _assert_handoff_closure_absent_from_traceback( + error, + callback=on_handoff, + schema_secret=_SENSITIVE_SCHEMA_SECRET, + ) + + +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "expected_redacted"), + [ + (True, False, True), + (False, True, True), + (True, True, True), + (False, False, False), + ], +) +@pytest.mark.asyncio +async def test_realtime_handoff_input_validation_error_follows_mixed_data_policy( + monkeypatch: pytest.MonkeyPatch, + model_redacted: bool, + tool_redacted: bool, + expected_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + target = RealtimeAgent(name="target") + handoff_calls = 0 + + async def on_handoff(_ctx: RunContextWrapper[Any], _input: _RequiredOutput) -> None: + nonlocal handoff_calls + handoff_calls += 1 + + handoff_obj = realtime_handoff(target, input_type=_RequiredOutput, on_handoff=on_handoff) + with pytest.raises(ModelBehaviorError) as exc_info: + await handoff_obj.on_invoke_handoff(RunContextWrapper(None), payload) + + assert handoff_calls == 0 + error = exc_info.value + if expected_redacted: + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + assert any( + frame_locals.get("input_json") == payload + for frame_locals in _agents_traceback_frame_locals(error) + ) + + +@pytest.mark.asyncio +async def test_run_surfaces_redacted_output_validation_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + model = FakeModel() + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + session = SimpleListSession( + session_id="redacted-run", + history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}], + ) + + with pytest.raises(ModelBehaviorError) as exc_info: + await Runner.run(agent, _MODEL_OUTPUT_SECRET, session=session) + + error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + ) + assert all(session is not value for frame in frame_locals for value in frame.values()) + + +def test_run_sync_surfaces_redacted_output_validation_error_without_runner_data( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + model = FakeModel() + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + session = SimpleListSession( + session_id="redacted-run-sync", + history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}], + ) + + with pytest.raises(ModelBehaviorError) as exc_info: + Runner.run_sync(agent, _MODEL_OUTPUT_SECRET, session=session) + + error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + ) + assert all(session is not value for frame in frame_locals for value in frame.values()) + + +@pytest.mark.asyncio +async def test_run_preserves_diagnostic_wrapper_traceback_locals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + diagnostic_input = "DIAGNOSTIC_RUNNER_INPUT_SECRET" + model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + session = SimpleListSession(session_id="diagnostic-runner") + + with pytest.raises(ModelBehaviorError) as exc_info: + await Runner.run(agent, diagnostic_input, session=session) + + error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) + assert any(frame.get("input") == diagnostic_input for frame in frame_locals) + assert any(frame.get("session") is session for frame in frame_locals) + + +def test_run_sync_preserves_diagnostic_wrapper_traceback_locals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + diagnostic_input = "DIAGNOSTIC_RUNNER_SYNC_INPUT_SECRET" + model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + session = SimpleListSession(session_id="diagnostic-runner-sync") + + with pytest.raises(ModelBehaviorError) as exc_info: + Runner.run_sync(agent, diagnostic_input, session=session) + + error = exc_info.value + frame_locals = _agents_traceback_frame_locals(error) + assert any(frame.get("input") == diagnostic_input for frame in frame_locals) + assert any(frame.get("session") is session for frame in frame_locals) + + +@pytest.mark.asyncio +async def test_streamed_run_surfaces_redacted_output_validation_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + model = FakeModel() + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + result = Runner.run_streamed(agent, "go") + + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + + error = exc_info.value + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.parametrize("redacted", [False, True]) +@pytest.mark.asyncio +async def test_streamed_run_loop_exception_follows_model_data_policy( + monkeypatch: pytest.MonkeyPatch, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + result = Runner.run_streamed(agent, "go") + + assert result.run_loop_task is not None + while not result.run_loop_task.done(): + await asyncio.sleep(0) + + error = result.run_loop_exception + assert isinstance(error, ModelBehaviorError) + if redacted: + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=False, + ) + else: + assert _MODEL_OUTPUT_SECRET in str(error) + assert isinstance(error.__cause__, ValidationError) + assert any( + _MODEL_OUTPUT_SECRET in repr(frame_locals) + for frame_locals in _agents_traceback_frame_locals(error) + ) + + +@pytest.mark.asyncio +async def test_streamed_output_guardrail_omits_run_data_from_redacted_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + def output_guardrail( + _context: RunContextWrapper[Any], + _agent: Agent[Any], + _agent_output: Any, + ) -> GuardrailFunctionOutput: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + raise AssertionError("validation should fail") # pragma: no cover + + model = FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)]) + agent = Agent( + name="A", + model=model, + output_guardrails=[OutputGuardrail(guardrail_function=output_guardrail)], + ) + result = Runner.run_streamed(agent, "go") + + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + + error = exc_info.value + assert error.run_data is None + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.asyncio +async def test_streamed_input_guardrail_omits_run_data_from_redacted_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + payload = f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}' + + def input_guardrail( + _context: RunContextWrapper[Any], + _agent: Agent[Any], + _input: str | list[Any], + ) -> GuardrailFunctionOutput: + AgentOutputSchema(_RequiredOutput).validate_json(payload) + raise AssertionError("validation should fail") # pragma: no cover + + model = FakeModel(initial_output=[get_text_message("unused")]) + agent = Agent( + name="A", + model=model, + input_guardrails=[InputGuardrail(guardrail_function=input_guardrail)], + ) + result = Runner.run_streamed(agent, _MODEL_OUTPUT_SECRET) + + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + + error = exc_info.value + assert error.run_data is None + assert _MODEL_OUTPUT_SECRET not in str(error) + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_receives_detached_redacted_error( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + retained_errors: list[ModelBehaviorError] = [] + + def recover(data: RunErrorHandlerInput[None]) -> _RequiredOutput: + assert isinstance(data.error, ModelBehaviorError) + retained_errors.append(data.error) + return _RequiredOutput(answer="safe", count=1) + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": recover}, + ) + async for _ in result.stream_events(): + pass + else: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": recover}, + ) + + assert len(retained_errors) == 1 + error = retained_errors[0] + assert error.__traceback__ is None + assert error.__cause__ is None + assert error.__context__ is None + assert _MODEL_OUTPUT_SECRET not in str(error) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_invalid_fallback_preserves_redaction( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + fallback_secret = "INVALID_HANDLER_FALLBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def invalid_fallback(_data: RunErrorHandlerInput[None]) -> dict[str, str]: + return {"answer": fallback_secret} + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": invalid_fallback}, + ) + with pytest.raises(UserError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(UserError) as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": invalid_fallback}, + ) + + error = exc_info.value + assert not caught_warnings + assert str(error) == "Error details are redacted." + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback(error, fallback_secret) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize("redacted", [True, False]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_fallback_serialization_follows_redaction_policy( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, + redacted: bool, +) -> None: + fallback_secret = "PERMISSIVE_HANDLER_FALLBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + model = FakeModel( + initial_output=[ + get_text_message(f'{{"payload": "{_MODEL_OUTPUT_SECRET}", "count": "invalid"}}') + ] + ) + agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput) + + def permissive_fallback(_data: RunErrorHandlerInput[None]) -> _PermissiveFallbackOutput: + return _PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + if streamed: + streaming_result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + async for _ in streaming_result.stream_events(): + pass + actual_final_output = streaming_result.final_output + else: + run_result = await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + actual_final_output = run_result.final_output + + rendered_warnings = "\n".join(str(warning.message) for warning in caught_warnings) + if redacted: + assert not caught_warnings + else: + assert fallback_secret in rendered_warnings + assert actual_final_output == _PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ) + + +@pytest.mark.parametrize( + ("streamed", "include_in_history", "redacted"), + [ + (False, True, True), + (False, False, True), + (True, True, True), + (True, False, True), + (False, True, False), + ], +) +@pytest.mark.asyncio +async def test_empty_final_output_handler_fallback_serialization_follows_redaction_policy( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, + include_in_history: bool, + redacted: bool, +) -> None: + fallback_secret = "EMPTY_HANDLER_FALLBACK_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + model = FakeModel(initial_output=[]) + agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput) + + def permissive_fallback(_data: RunErrorHandlerInput[None]) -> RunErrorHandlerResult: + return RunErrorHandlerResult( + final_output=_PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ), + include_in_history=include_in_history, + ) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + if streamed: + streaming_result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + async for _ in streaming_result.stream_events(): + pass + actual_final_output = streaming_result.final_output + else: + run_result = await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": permissive_fallback}, + ) + actual_final_output = run_result.final_output + + rendered_warnings = "\n".join(str(warning.message) for warning in caught_warnings) + if redacted: + assert not caught_warnings + else: + assert fallback_secret in rendered_warnings + assert actual_final_output == _PermissiveFallbackOutput( + payload=cast(Any, {"secret": fallback_secret}), + count=1, + ) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_failure_preserves_redaction( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def fail(data: RunErrorHandlerInput[None]) -> None: + raise RuntimeError(repr(data.run_data.raw_responses)) + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + with pytest.raises(UserError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(UserError) as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + + error = exc_info.value + assert str(error) == "Error details are redacted." + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_hostile_failure_preserves_redaction( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + handler_secret = "HOSTILE_HANDLER_FAILURE_SECRET" + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def fail(_data: RunErrorHandlerInput[None]) -> None: + raise _HostileAttributeWriteException(handler_secret) + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + with pytest.raises(UserError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(UserError) as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + + error = exc_info.value + assert str(error) == "Error details are redacted." + assert error.run_data is None + assert error.__cause__ is None + assert error.__context__ is None + _assert_secret_absent_from_agents_traceback(error, _MODEL_OUTPUT_SECRET) + _assert_secret_absent_from_agents_traceback(error, handler_secret) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_invalid_final_output_handler_failure_preserves_diagnostic_context( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]) + agent = Agent(name="A", model=model, output_type=_RequiredOutput) + + def fail(_data: RunErrorHandlerInput[None]) -> None: + raise RuntimeError("handler failed") + + if streamed: + result = Runner.run_streamed( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + with pytest.raises(RuntimeError, match="handler failed") as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(RuntimeError, match="handler failed") as exc_info: + await Runner.run( + agent, + "go", + error_handlers={"invalid_final_output": fail}, + ) + + validation_error = exc_info.value.__context__ + assert isinstance(validation_error, ModelBehaviorError) + assert _MODEL_OUTPUT_SECRET in str(validation_error) + assert isinstance(validation_error.__cause__, ValidationError) + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize("redacted", [False, True]) +@pytest.mark.asyncio +async def test_multiturn_output_validation_error_run_data_follows_redaction_policy( + monkeypatch: pytest.MonkeyPatch, + streamed: bool, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + + @function_tool + def record_value(value: str) -> str: + return "recorded" + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call( + record_value.name, + json.dumps({"value": _MODEL_OUTPUT_SECRET}), + ) + ], + [get_text_message('{"answer": "missing count"}')], + ] + ) + agent = Agent( + name="A", + model=model, + tools=[record_value], + output_type=_RequiredOutput, + ) + + if streamed: + result = Runner.run_streamed(agent, "go") + with pytest.raises(ModelBehaviorError) as exc_info: + async for _ in result.stream_events(): + pass + else: + with pytest.raises(ModelBehaviorError) as exc_info: + await Runner.run(agent, "go") + + error = exc_info.value + if redacted: + assert error.run_data is None + assert _MODEL_OUTPUT_SECRET not in str(error) + _assert_secret_absent_from_agents_traceback( + error, + _MODEL_OUTPUT_SECRET, + require_agents_frames=streamed, + ) + else: + assert error.run_data is not None + assert error.run_data.raw_responses + assert error.run_data.new_items diff --git a/tests/test_invalid_final_output_handler.py b/tests/test_invalid_final_output_handler.py index b4519debdc..667d9a746d 100644 --- a/tests/test_invalid_final_output_handler.py +++ b/tests/test_invalid_final_output_handler.py @@ -7,6 +7,7 @@ from openai.types.responses import ResponseOutputMessage from pydantic import BaseModel +import agents._debug as _debug from agents import ( Agent, AgentHookContext, @@ -114,7 +115,10 @@ async def test_invalid_final_output_handler_can_skip_fallback_history() -> None: @pytest.mark.asyncio -async def test_invalid_final_output_handler_rejects_invalid_fallback() -> None: +async def test_invalid_final_output_handler_rejects_invalid_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) model = FakeModel(initial_output=[get_text_message("not valid json")]) agent = Agent(name="test", model=model, output_type=FinalOutput)