diff --git a/src/agents/extensions/tool_output_trimmer.py b/src/agents/extensions/tool_output_trimmer.py index d6fab350a0..3955c22aa9 100644 --- a/src/agents/extensions/tool_output_trimmer.py +++ b/src/agents/extensions/tool_output_trimmer.py @@ -44,23 +44,44 @@ # still call the tool without them. _PROSE_SCHEMA_KEYWORDS = frozenset({"description", "title", "$comment", "examples"}) -# Keywords whose value is a map keyed by *user-chosen names* — parameter names, definition -# names, regexes — rather than by schema keywords. Their keys must survive even when they -# spell one of the prose keywords above, so they are recursed into by value only. -_NAME_KEYED_SCHEMA_MAPS = frozenset( +# Keywords whose value is itself a schema. Unknown keywords are intentionally not traversed: +# preserving unfamiliar data is safer than treating every nested mapping as a schema and +# accidentally deleting user-controlled values. +_SCHEMA_VALUE_KEYWORDS = frozenset( { - "properties", - "patternProperties", - "$defs", - "definitions", - "dependentSchemas", - "dependentRequired", + "additionalItems", + "additionalProperties", + "contains", + "contentSchema", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", } ) -# Keywords whose value is instance *data* rather than a subschema. Nothing inside them is a -# schema keyword, so they are copied through untouched. -_DATA_SCHEMA_KEYWORDS = frozenset({"default", "const", "enum"}) +# Keywords whose value is a list of schemas. +_SCHEMA_LIST_KEYWORDS = frozenset({"allOf", "anyOf", "oneOf", "prefixItems"}) + +# Keywords whose value is a map keyed by user-chosen names and whose values are schemas. +_SCHEMA_MAP_KEYWORDS = frozenset( + {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"} +) + +# The legacy ``dependencies`` keyword is also name-keyed, but each value may be either a +# schema or a list of property names. +_SCHEMA_OR_PROPERTY_LIST_MAP_KEYWORDS = frozenset({"dependencies"}) + +_STRUCTURED_OUTPUT_FIELDS = { + "input_text": frozenset({"type", "text"}), + "input_image": frozenset({"type", "image_url", "file_id", "detail"}), + "input_file": frozenset({"type", "file_data", "file_url", "file_id", "filename"}), +} +_IMAGE_DETAILS = frozenset({"low", "high", "auto"}) @dataclass @@ -76,9 +97,12 @@ class ToolOutputTrimmer: recent_turns: Number of recent user messages whose surrounding items are never trimmed. Defaults to 2. max_output_chars: Tool outputs above this character count are candidates for - trimming. Defaults to 500. - preview_chars: How many characters of the original output to preserve as a - preview when trimming. Defaults to 200. + trimming. Structured outputs count their model-facing string payloads without + Python or JSON representation overhead, and their replacements fit within this + budget. Defaults to 500. + preview_chars: Maximum number of characters of a string output, or the text parts of + a structured output, to preserve as a preview when trimming. Structured previews + may be shorter when needed to fit ``max_output_chars``. Defaults to 200. trimmable_tools: Optional tool name or set of tool names whose outputs can be trimmed. For namespaced tools, both bare names and qualified ``namespace.name`` entries are supported. If ``None``, all tool outputs are eligible for trimming. Defaults @@ -223,6 +247,9 @@ def _trim_function_call_output( ) -> tuple[dict[str, Any] | None, int]: """Trim a function_call_output item when its serialized output is too large.""" output = item.get("output", "") + if isinstance(output, list): + return self._trim_structured_function_call_output(item, output, tool_names) + output_str = output if isinstance(output, str) else str(output) output_len = len(output_str) if output_len <= self.max_output_chars: @@ -242,6 +269,126 @@ def _trim_function_call_output( trimmed_item["output"] = summary return trimmed_item, output_len - len(summary) + def _trim_structured_function_call_output( + self, + item: dict[str, Any], + parts: list[Any], + tool_names: tuple[str, ...], + ) -> tuple[dict[str, Any] | None, int]: + """Trim a canonical structured function output without previewing opaque payloads.""" + details = self._structured_output_details(parts) + if details is None: + return None, 0 + + output_len, text_content, dropped_part_types = details + if output_len <= self.max_output_chars: + return None, 0 + + display_name = (tool_names[0] if tool_names else "") or "unknown_tool" + dropped_note = "" + if dropped_part_types: + dropped_note = "; dropped " + ", ".join( + f"{count} {part_type}" for part_type, count in sorted(dropped_part_types.items()) + ) + + minimal_header = "[Trimmed]" + if self.max_output_chars < len(minimal_header): + summary = minimal_header[: self.max_output_chars] + else: + preview_budget = self.max_output_chars - len(minimal_header) - 1 + preview_len = min(len(text_content), self.preview_chars, max(0, preview_budget)) + body = f"\n{text_content[:preview_len]}" if preview_len else "" + if ( + preview_len < len(text_content) + and len(minimal_header) + len(body) + len("...") <= self.max_output_chars + ): + body += "..." + + preview_note = f"; preview {preview_len}" if text_content else "" + headers = [ + f"[Trimmed: {display_name}; payload {output_len}{preview_note}{dropped_note}]" + ] + if dropped_part_types: + dropped_types = ", ".join(sorted(dropped_part_types)) + headers.extend( + [ + f"[Trimmed: {display_name}{dropped_note}]", + f"[Trimmed{dropped_note}]", + f"[Trimmed: {dropped_types}]", + f"[Trimmed: dropped {sum(dropped_part_types.values())} opaque]", + ] + ) + headers.extend( + [ + f"[Trimmed: payload {output_len}]", + f"[Trimmed: {display_name}]", + minimal_header, + ] + ) + summary = next( + header + body + for header in headers + if len(header) + len(body) <= self.max_output_chars + ) + + trimmed_item = dict(item) + trimmed_item["output"] = summary + return trimmed_item, output_len - len(summary) + + def _structured_output_details( + self, + parts: list[Any], + ) -> tuple[int, str, dict[str, int]] | None: + """Return payload size, readable text, and dropped-part counts for canonical parts.""" + if not parts: + return None + + output_len = 0 + text_segments: list[str] = [] + dropped_part_types: dict[str, int] = {} + + for part in parts: + if not isinstance(part, dict): + return None + + part_type = part.get("type") + if not isinstance(part_type, str): + return None + allowed_fields = _STRUCTURED_OUTPUT_FIELDS.get(part_type) + if allowed_fields is None or not set(part).issubset(allowed_fields): + return None + if any(key != "type" and not isinstance(value, str) for key, value in part.items()): + return None + + if part_type == "input_text": + text = part.get("text") + if not isinstance(text, str): + return None + text_segments.append(text) + elif part_type == "input_image": + if not isinstance(part.get("image_url"), str) and not isinstance( + part.get("file_id"), str + ): + return None + if "detail" in part and part["detail"] not in _IMAGE_DETAILS: + return None + dropped_part_types[part_type] = dropped_part_types.get(part_type, 0) + 1 + elif part_type == "input_file": + if not any( + isinstance(part.get(field), str) + for field in ("file_data", "file_url", "file_id") + ): + return None + dropped_part_types[part_type] = dropped_part_types.get(part_type, 0) + 1 + + output_len += sum( + len(value) + for key, value in part.items() + if key != "type" and isinstance(value, str) + ) + + return output_len, "\n".join(text_segments), dropped_part_types + def _trim_tool_search_output(self, item: dict[str, Any]) -> tuple[dict[str, Any] | None, int]: """Trim a tool_search_output item while keeping a valid replayable shape.""" if isinstance(item.get("results"), list): @@ -311,31 +458,40 @@ def _trim_json_schema(self, schema: dict[str, Any]) -> dict[str, Any]: """Remove verbose prose from a JSON schema while preserving its structure.""" trimmed_schema: dict[str, Any] = {} for key, value in schema.items(): - # A name-keyed map is keyed by parameter/definition names, not by schema - # keywords, so recurse into its values while keeping its keys verbatim. - # Dropping a key here would delete a declared parameter or dangle a $ref. - if key in _NAME_KEYED_SCHEMA_MAPS and isinstance(value, dict): - trimmed_schema[key] = { - name: self._trim_json_schema(sub) if isinstance(sub, dict) else sub - for name, sub in value.items() - } - continue - # These hold instance data. A "title" key inside a default value is part of the - # value, so trimming it would silently change the tool's contract. - if key in _DATA_SCHEMA_KEYWORDS: - trimmed_schema[key] = value - continue if key in _PROSE_SCHEMA_KEYWORDS: continue - if isinstance(value, dict): - trimmed_schema[key] = self._trim_json_schema(value) - elif isinstance(value, list): + if key in _SCHEMA_VALUE_KEYWORDS: + if isinstance(value, dict): + trimmed_schema[key] = self._trim_json_schema(value) + elif key == "items" and isinstance(value, list): + trimmed_schema[key] = [ + self._trim_json_schema(item) if isinstance(item, dict) else item + for item in value + ] + else: + trimmed_schema[key] = value + continue + if key in _SCHEMA_LIST_KEYWORDS and isinstance(value, list): trimmed_schema[key] = [ self._trim_json_schema(item) if isinstance(item, dict) else item for item in value ] - else: - trimmed_schema[key] = value + continue + if key in _SCHEMA_MAP_KEYWORDS and isinstance(value, dict): + trimmed_schema[key] = { + name: self._trim_json_schema(sub) if isinstance(sub, dict) else sub + for name, sub in value.items() + } + continue + if key in _SCHEMA_OR_PROPERTY_LIST_MAP_KEYWORDS and isinstance(value, dict): + trimmed_schema[key] = { + name: self._trim_json_schema(dependency) + if isinstance(dependency, dict) + else dependency + for name, dependency in value.items() + } + continue + trimmed_schema[key] = value return trimmed_schema def _serialize_json_like(self, value: Any) -> str: diff --git a/tests/extensions/test_tool_output_trimmer.py b/tests/extensions/test_tool_output_trimmer.py index 58fcdbd191..fb615e99b2 100644 --- a/tests/extensions/test_tool_output_trimmer.py +++ b/tests/extensions/test_tool_output_trimmer.py @@ -10,8 +10,11 @@ from unittest.mock import MagicMock import pytest +from openai.types.responses import ResponseFunctionToolCall +from agents import ItemHelpers, ToolOutputFileContent, ToolOutputImage, ToolOutputText from agents.extensions.tool_output_trimmer import ToolOutputTrimmer +from agents.models.chatcmpl_converter import Converter from agents.run_config import CallModelData, ModelInputData # --------------------------------------------------------------------------- @@ -34,7 +37,7 @@ def _func_call(call_id: str, name: str, *, namespace: str | None = None) -> dict return item -def _func_output(call_id: str, output: str) -> dict[str, Any]: +def _func_output(call_id: str, output: Any) -> dict[str, Any]: return {"type": "function_call_output", "call_id": call_id, "output": output} @@ -220,6 +223,329 @@ def test_preserves_small_old_output(self) -> None: result = trimmer(_make_data(items)) assert _output(result, 2) == small + @pytest.mark.parametrize("opaque_first", [True, False]) + def test_structured_output_previews_text_and_drops_opaque_parts( + self, opaque_first: bool + ) -> None: + """Canonical structured outputs use text content instead of representation order.""" + caption = "Revenue chart: Q3 up 12% YoY, driven by EMEA." + image_part = { + "type": "input_image", + "image_url": "data:image/png;base64," + "Q" * 3000, + "detail": "auto", + } + text_part = {"type": "input_text", "text": caption} + parts = [image_part, text_part] if opaque_first else [text_part, image_part] + items = [ + _user("q1"), + _func_call("c1", "plot"), + _func_output("c1", parts), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=500, preview_chars=200) + result = trimmer(_make_data(items)) + trimmed = _output(result, 2) + + assert isinstance(trimmed, str) + assert caption in trimmed + assert "base64," not in trimmed + assert f"preview {len(caption)}" in trimmed + assert "dropped 1 input_image" in trimmed + assert not trimmed.endswith("...") + + repeated = trimmer(_make_data(result.input)) + assert repeated.input == result.input + + def test_structured_output_truncates_long_text_with_exact_preview_length(self) -> None: + """The summary reports and marks truncation only when text itself is shortened.""" + text = "abcdefghij" * 40 + parts = [{"type": "input_text", "text": text}] + items = [ + _user("q1"), + _func_call("c1", "search"), + _func_output("c1", parts), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=100, preview_chars=40) + result = trimmer(_make_data(items)) + trimmed = _output(result, 2) + + assert len(trimmed) <= 100 + assert "payload 400" in trimmed + assert "preview 40" in trimmed + assert trimmed.endswith(f"{'abcdefghij' * 4}...") + + repeated = trimmer(_make_data(result.input)) + assert repeated.input == result.input + + def test_structured_output_prioritizes_text_at_tight_budget(self) -> None: + """A tight structured budget preserves feasible text before optional metadata.""" + text = "abcdefghijklmnopqrstuvwxyz" * 10 + items = [ + _user("q1"), + _func_call("c1", "search"), + _func_output("c1", [{"type": "input_text", "text": text}]), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=40, preview_chars=100) + result = trimmer(_make_data(items)) + trimmed = _output(result, 2) + + assert isinstance(trimmed, str) + assert len(trimmed) <= 40 + assert trimmed.startswith("[Trimmed]\n") + assert text[:20] in trimmed + + repeated = trimmer(_make_data(result.input)) + assert repeated.input == result.input + + def test_structured_output_without_text_names_dropped_parts(self) -> None: + """Image-only and file-only outputs are summarized without leaking their payloads.""" + parts = [ + {"type": "input_image", "image_url": "data:image/png;base64," + "Q" * 1000}, + {"type": "input_file", "file_data": "R" * 1000, "filename": "report.pdf"}, + ] + items = [ + _user("q1"), + _func_call("c1", "render"), + _func_output("c1", parts), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=500, preview_chars=200) + result = trimmer(_make_data(items)) + trimmed = _output(result, 2) + + assert "base64," not in trimmed + assert "report.pdf" not in trimmed + assert "dropped 1 input_file, 1 input_image" in trimmed + assert "char preview" not in trimmed + assert not trimmed.endswith("...") + + def test_structured_output_prioritizes_text_over_opaque_metadata(self) -> None: + """Mixed outputs retain their feasible text before optional dropped-part details.""" + text = "useful-text-preview-more" + parts = [ + {"type": "input_text", "text": text}, + {"type": "input_image", "image_url": "image-payload-" + "Q" * 1000}, + {"type": "input_file", "file_data": "file-payload-" + "R" * 1000}, + ] + items = [ + _user("q1"), + _func_call("c1", "render"), + _func_output("c1", parts), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=70, preview_chars=20) + result = trimmer(_make_data(items)) + trimmed = _output(result, 2) + + assert isinstance(trimmed, str) + assert len(trimmed) <= 70 + assert text[:20] in trimmed + assert "image-payload" not in trimmed + assert "file-payload" not in trimmed + + repeated = trimmer(_make_data(result.input)) + assert repeated.input == result.input + + @pytest.mark.parametrize( + "part,payload_fragment", + [ + ({"type": "input_image", "image_url": "image-payload-12345"}, "image-payload"), + ({"type": "input_file", "file_data": "file-payload-12345"}, "file-payload"), + ], + ) + def test_structured_opaque_output_respects_tight_budget( + self, part: dict[str, str], payload_fragment: str + ) -> None: + """Canonical opaque payloads become stable bounded summaries at tight budgets.""" + items = [ + _user("q1"), + _func_call("c1", "render"), + _func_output("c1", [part]), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=10, preview_chars=0) + result = trimmer(_make_data(items)) + trimmed = _output(result, 2) + + assert isinstance(trimmed, str) + assert len(trimmed) <= 10 + assert payload_fragment not in trimmed + + repeated = trimmer(_make_data(result.input)) + assert repeated.input == result.input + + def test_structured_opaque_output_uses_exact_type_header_when_it_fits(self) -> None: + """A compact summary reports the exact omitted type before generic metadata.""" + part = {"type": "input_image", "image_url": "image-payload-" + "Q" * 1000} + items = [ + _user("q1"), + _func_call("c1", "render"), + _func_output("c1", [part]), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + expected = "[Trimmed: input_image]" + + result = ToolOutputTrimmer(max_output_chars=len(expected), preview_chars=0)( + _make_data(items) + ) + + assert _output(result, 2) == expected + + @pytest.mark.parametrize( + "max_output_chars", + [1, len("[Trimmed]"), len("[Trimmed: input_image]"), 70, 200], + ) + def test_canonical_structured_output_replays_through_chat_completions( + self, max_output_chars: int + ) -> None: + """SDK-produced structured output stays bounded and replayable after trimming.""" + call = ResponseFunctionToolCall( + id="fc1", + call_id="c1", + name="render", + arguments="{}", + type="function_call", + ) + output_item = ItemHelpers.tool_call_output_item( + call, + [ + ToolOutputText(text="useful-text-preview-" + "T" * 1000), + ToolOutputImage( + image_url="image-payload-" + "I" * 1000, + file_id="image-file-id", + detail="high", + ), + ToolOutputFileContent( + file_data="file-payload-" + "F" * 1000, + file_url="https://example.com/report.pdf", + file_id="file-id", + filename="report.pdf", + ), + ], + ) + produced_output = output_item["output"] + assert isinstance(produced_output, list) + assert produced_output[1] == { + "type": "input_image", + "image_url": "image-payload-" + "I" * 1000, + "file_id": "image-file-id", + "detail": "high", + } + assert produced_output[2] == { + "type": "input_file", + "file_data": "file-payload-" + "F" * 1000, + "file_url": "https://example.com/report.pdf", + "file_id": "file-id", + "filename": "report.pdf", + } + + items = [ + _user("q1"), + _func_call("c1", "render"), + output_item, + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + original = copy.deepcopy(items) + trimmer = ToolOutputTrimmer(max_output_chars=max_output_chars, preview_chars=20) + result = trimmer(_make_data(items)) + trimmed = _output(result, 2) + + assert isinstance(trimmed, str) + assert len(trimmed) <= max_output_chars + assert "image-payload" not in trimmed + assert "file-payload" not in trimmed + assert items == original + assert trimmer(_make_data(result.input)).input == result.input + + messages = Converter.items_to_messages([result.input[2]]) + assert messages == [{"role": "tool", "tool_call_id": "c1", "content": trimmed}] + + def test_structured_output_threshold_uses_payload_characters(self) -> None: + """Structured syntax and field names do not cause a small payload to be trimmed.""" + parts = [{"type": "input_text", "text": "short"}] + items = [ + _user("q1"), + _func_call("c1", "search"), + _func_output("c1", parts), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=5, preview_chars=2) + result = trimmer(_make_data(items)) + + assert _output(result, 2) == parts + + @pytest.mark.parametrize( + "parts", + [ + [{"type": "output_text", "text": "x" * 1000}], + [{"type": "input_text", "text": "x" * 1000, "metadata": "unsupported"}], + [{"type": "input_image", "image_url": "x" * 1000, "detail": "invalid"}], + ["not a content part"], + ], + ) + def test_unsupported_structured_output_is_preserved(self, parts: list[Any]) -> None: + """The built-in trimmer does not infer semantics for non-canonical list shapes.""" + items = [ + _user("q1"), + _func_call("c1", "search"), + _func_output("c1", parts), + _assistant("a1"), + _user("q2"), + _assistant("a2"), + _user("q3"), + _assistant("a3"), + ] + + trimmer = ToolOutputTrimmer(max_output_chars=100, preview_chars=40) + result = trimmer(_make_data(items)) + + assert _output(result, 2) == parts + def test_respects_trimmable_tools_allowlist(self) -> None: """Only outputs from tools in trimmable_tools should be trimmed.""" large = "x" * 1000 @@ -441,6 +767,10 @@ def test_keeps_definition_names_and_instance_data_in_schema(self) -> None: "patternProperties": {"title": {"type": "string"}}, "dependentSchemas": {"title": {"required": ["note"]}}, "dependentRequired": {"title": ["note"]}, + "dependencies": { + "description": ["note"], + "title": {"type": "string", "description": "dependency prose " * 200}, + }, "properties": { "note": {"$ref": "#/$defs/description"}, "prio": {"$ref": "#/$defs/Priority"}, @@ -452,6 +782,10 @@ def test_keeps_definition_names_and_instance_data_in_schema(self) -> None: "mode": {"const": {"title": "A", "kind": "fast"}}, "choice": {"enum": [{"title": "A", "id": 1}, {"title": "B", "id": 2}]}, }, + "x-tool-metadata": { + "description": "application data", + "nested": {"title": "must survive"}, + }, "required": ["note"], } items = [ @@ -492,6 +826,8 @@ def test_keeps_definition_names_and_instance_data_in_schema(self) -> None: assert sorted(trimmed["patternProperties"]) == ["title"] assert sorted(trimmed["dependentSchemas"]) == ["title"] assert trimmed["dependentRequired"] == {"title": ["note"]} + assert sorted(trimmed["dependencies"]) == ["description", "title"] + assert "description" not in trimmed["dependencies"]["title"] # Instance data is preserved byte for byte. assert trimmed["properties"]["opts"]["default"] == { @@ -504,6 +840,10 @@ def test_keeps_definition_names_and_instance_data_in_schema(self) -> None: {"title": "A", "id": 1}, {"title": "B", "id": 2}, ] + assert trimmed["x-tool-metadata"] == { + "description": "application data", + "nested": {"title": "must survive"}, + } # Prose is still trimmed, at the schema level and inside a nested subschema. assert "description" not in trimmed @@ -524,6 +864,7 @@ def test_trims_prose_inside_genuine_subschema_keywords(self) -> None: "propertyNames": {"pattern": "^x", "description": "a key " * 200}, }, }, + "allOf": [{"type": "object", "description": "combined schema " * 200}], } items = [ _user("q1"), @@ -554,6 +895,7 @@ def test_trims_prose_inside_genuine_subschema_keywords(self) -> None: assert trimmed["properties"]["tags"]["items"] == {"type": "string"} assert trimmed["properties"]["bag"]["propertyNames"] == {"pattern": "^x"} + assert trimmed["allOf"] == [{"type": "object"}] def test_trims_legacy_tool_search_output_results(self) -> None: """Legacy tool_search_output snapshots with free-text results should still trim."""