diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_callback_chat_target.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_callback_chat_target.py index b33888cc14fb..a41eb7dc715d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_callback_chat_target.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_callback_chat_target.py @@ -94,6 +94,37 @@ async def _send_prompt_with_retry(self, *, message: Message) -> List[Message]: """ return await self._send_prompt_impl(message=message) + def _resolve_content(self, piece: Any) -> str: + """Resolve the text content for a message piece, reading file content for binary_path pieces. + + XPIA (indirect jailbreak) strategy creates file-based context prompts with + binary_path data type, but model targets only support text content. This helper + reads the file and returns its contents for binary_path pieces, or returns the + converted/original value as-is for other data types. + + Args: + piece: A message piece with converted_value, original_value, and + converted_value_data_type attributes. + + Returns: + The resolved text content string. + """ + value = piece.converted_value or piece.original_value or "" + if getattr(piece, "converted_value_data_type", None) == "binary_path" and isinstance(value, str) and value: + try: + # Synchronous read is intentional here — XPIA context files are small + # text files, so the blocking I/O is negligible. + with open(value, "r", encoding="utf-8", errors="replace") as f: + return f.read() + except (OSError, IOError) as exc: + logger.warning( + "Failed to read binary_path file %s: %s. Falling back to file path string.", + value, + exc, + ) + return value + return value + async def _send_prompt_impl(self, *, message: Message) -> List[Message]: """ Core implementation of send_prompt_async. @@ -105,6 +136,8 @@ async def _send_prompt_impl(self, *, message: Message) -> List[Message]: self._validate_request(prompt_request=message) request = message.get_piece(0) + request_content = self._resolve_content(request) + # Get conversation history and convert to chat message format conversation_history = self._memory.get_conversation(conversation_id=request.conversation_id) messages: List[Dict[str, str]] = [] @@ -113,7 +146,7 @@ async def _send_prompt_impl(self, *, message: Message) -> List[Message]: messages.append( { "role": (piece.api_role if hasattr(piece, "api_role") else str(piece.role)), - "content": piece.converted_value or piece.original_value or "", + "content": self._resolve_content(piece), } ) @@ -121,7 +154,7 @@ async def _send_prompt_impl(self, *, message: Message) -> List[Message]: messages.append( { "role": (request.api_role if hasattr(request, "api_role") else str(request.role)), - "content": request.converted_value or request.original_value or "", + "content": request_content, } ) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_redteam/test_callback_chat_target.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_redteam/test_callback_chat_target.py index cd8de9006848..afc2308d418a 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_redteam/test_callback_chat_target.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_redteam/test_callback_chat_target.py @@ -609,3 +609,139 @@ async def test_empty_messages_list_raises_valueerror(self, mock_callback): with pytest.raises(ValueError, match="invalid response"): await target.send_prompt_async(message=mock_request) + + +@pytest.mark.unittest +class TestCallbackChatTargetBinaryPath: + """Tests for binary_path resolution via _resolve_content helper.""" + + def test_resolve_content_reads_binary_path_file(self, tmp_path): + """binary_path piece with a valid temp file returns file contents.""" + file = tmp_path / "xpia_context.txt" + file.write_text("injected XPIA payload", encoding="utf-8") + + piece = MagicMock() + piece.converted_value = str(file) + piece.original_value = str(file) + piece.converted_value_data_type = "binary_path" + + target = _CallbackChatTarget(callback=AsyncMock()) + assert target._resolve_content(piece) == "injected XPIA payload" + + def test_resolve_content_falls_back_on_missing_file(self): + """Missing binary_path file logs warning and returns the path string.""" + piece = MagicMock() + piece.converted_value = "/nonexistent/path/to/file.txt" + piece.original_value = "/nonexistent/path/to/file.txt" + piece.converted_value_data_type = "binary_path" + + target = _CallbackChatTarget(callback=AsyncMock()) + result = target._resolve_content(piece) + assert result == "/nonexistent/path/to/file.txt" + + def test_resolve_content_returns_text_as_is(self): + """Text data type pieces are returned without file I/O.""" + piece = MagicMock() + piece.converted_value = "plain text prompt" + piece.original_value = "plain text prompt" + piece.converted_value_data_type = "text" + + target = _CallbackChatTarget(callback=AsyncMock()) + assert target._resolve_content(piece) == "plain text prompt" + + @pytest.mark.asyncio + async def test_binary_path_content_sent_to_callback(self, tmp_path): + """Callback receives file *contents* (not the path) for binary_path requests.""" + file = tmp_path / "context.txt" + file.write_text("file content for callback", encoding="utf-8") + + mock_callback = AsyncMock( + return_value={ + "messages": [{"role": "assistant", "content": "ok"}], + "stream": False, + "session_state": None, + "context": {}, + } + ) + + target = _CallbackChatTarget(callback=mock_callback, retry_enabled=False) + + mock_piece = MagicMock() + mock_piece.id = "piece-1" + mock_piece.converted_value = str(file) + mock_piece.original_value = str(file) + mock_piece.converted_value_data_type = "binary_path" + mock_piece.conversation_id = "conv-bp" + mock_piece.api_role = "user" + mock_piece.role = "user" + mock_piece.labels = {} + + mock_request = MagicMock() + mock_request.message_pieces = [mock_piece] + mock_request.get_piece.return_value = mock_piece + + with patch.object(target, "_memory") as mock_memory, patch( + "azure.ai.evaluation.red_team._callback_chat_target.construct_response_from_request" + ) as mock_construct: + mock_memory.get_conversation.return_value = [] + mock_construct.return_value = mock_request + await target.send_prompt_async(message=mock_request) + + sent_messages = mock_callback.call_args.kwargs["messages"] + assert sent_messages[-1]["content"] == "file content for callback" + + @pytest.mark.asyncio + async def test_binary_path_in_conversation_history_resolved(self, tmp_path): + """Conversation history pieces with binary_path are also resolved to file contents.""" + file = tmp_path / "history_context.txt" + file.write_text("history file content", encoding="utf-8") + + mock_callback = AsyncMock( + return_value={ + "messages": [{"role": "assistant", "content": "ok"}], + "stream": False, + "session_state": None, + "context": {}, + } + ) + + target = _CallbackChatTarget(callback=mock_callback, retry_enabled=False) + + # Build a history message with binary_path piece + history_piece = MagicMock() + history_piece.converted_value = str(file) + history_piece.original_value = str(file) + history_piece.converted_value_data_type = "binary_path" + history_piece.api_role = "user" + history_piece.role = "user" + + history_msg = MagicMock() + history_msg.message_pieces = [history_piece] + + # Current request (plain text) + mock_piece = MagicMock() + mock_piece.id = "piece-2" + mock_piece.converted_value = "follow-up question" + mock_piece.original_value = "follow-up question" + mock_piece.converted_value_data_type = "text" + mock_piece.conversation_id = "conv-bp-hist" + mock_piece.api_role = "user" + mock_piece.role = "user" + mock_piece.labels = {} + + mock_request = MagicMock() + mock_request.message_pieces = [mock_piece] + mock_request.get_piece.return_value = mock_piece + + with patch.object(target, "_memory") as mock_memory, patch( + "azure.ai.evaluation.red_team._callback_chat_target.construct_response_from_request" + ) as mock_construct: + mock_memory.get_conversation.return_value = [history_msg] + mock_construct.return_value = mock_request + await target.send_prompt_async(message=mock_request) + + sent_messages = mock_callback.call_args.kwargs["messages"] + # First message is from history — should contain file content, not path + assert sent_messages[0]["content"] == "history file content" + # Second message is the current plain-text request + assert sent_messages[1]["content"] == "follow-up question"