Fix XPIA binary_path incompatibility for model targets (#5058420)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_callback_chat_target.py

@@ -94,6 +94,37 @@ async def _send_prompt_with_retry(self, *, message: Message) -> List[Message]:
         """
         return await self._send_prompt_impl(message=message)
 
+    def _resolve_content(self, piece: Any) -> str:
+        """Resolve the text content for a message piece, reading file content for binary_path pieces.
+
+        XPIA (indirect jailbreak) strategy creates file-based context prompts with
+        binary_path data type, but model targets only support text content. This helper
+        reads the file and returns its contents for binary_path pieces, or returns the
+        converted/original value as-is for other data types.
+
+        Args:
+            piece: A message piece with converted_value, original_value, and
+                converted_value_data_type attributes.
+
+        Returns:
+            The resolved text content string.
+        """
+        value = piece.converted_value or piece.original_value or ""
+        if getattr(piece, "converted_value_data_type", None) == "binary_path" and isinstance(value, str) and value:
+            try:
+                # Synchronous read is intentional here — XPIA context files are small
+                # text files, so the blocking I/O is negligible.
+                with open(value, "r", encoding="utf-8", errors="replace") as f:
+                    return f.read()
+            except (OSError, IOError) as exc:
+                logger.warning(
+                    "Failed to read binary_path file %s: %s. Falling back to file path string.",
+                    value,
+                    exc,
+                )
+                return value
+        return value
+
     async def _send_prompt_impl(self, *, message: Message) -> List[Message]:
         """
         Core implementation of send_prompt_async.
@@ -105,6 +136,8 @@ async def _send_prompt_impl(self, *, message: Message) -> List[Message]:
         self._validate_request(prompt_request=message)
         request = message.get_piece(0)
 
+        request_content = self._resolve_content(request)
+
         # Get conversation history and convert to chat message format
         conversation_history = self._memory.get_conversation(conversation_id=request.conversation_id)
         messages: List[Dict[str, str]] = []
@@ -113,15 +146,15 @@ async def _send_prompt_impl(self, *, message: Message) -> List[Message]:
                 messages.append(
                     {
                         "role": (piece.api_role if hasattr(piece, "api_role") else str(piece.role)),
-                        "content": piece.converted_value or piece.original_value or "",
+                        "content": self._resolve_content(piece),
                     }
                 )
 
         # Add current request
         messages.append(
             {
                 "role": (request.api_role if hasattr(request, "api_role") else str(request.role)),
-                "content": request.converted_value or request.original_value or "",
+                "content": request_content,
             }
         )
 

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_redteam/test_callback_chat_target.py

@@ -609,3 +609,139 @@ async def test_empty_messages_list_raises_valueerror(self, mock_callback):
 
             with pytest.raises(ValueError, match="invalid response"):
                 await target.send_prompt_async(message=mock_request)
+
+
+@pytest.mark.unittest
+class TestCallbackChatTargetBinaryPath:
+    """Tests for binary_path resolution via _resolve_content helper."""
+
+    def test_resolve_content_reads_binary_path_file(self, tmp_path):
+        """binary_path piece with a valid temp file returns file contents."""
+        file = tmp_path / "xpia_context.txt"
+        file.write_text("injected XPIA payload", encoding="utf-8")
+
+        piece = MagicMock()
+        piece.converted_value = str(file)
+        piece.original_value = str(file)
+        piece.converted_value_data_type = "binary_path"
+
+        target = _CallbackChatTarget(callback=AsyncMock())
+        assert target._resolve_content(piece) == "injected XPIA payload"
+
+    def test_resolve_content_falls_back_on_missing_file(self):
+        """Missing binary_path file logs warning and returns the path string."""
+        piece = MagicMock()
+        piece.converted_value = "/nonexistent/path/to/file.txt"
+        piece.original_value = "/nonexistent/path/to/file.txt"
+        piece.converted_value_data_type = "binary_path"
+
+        target = _CallbackChatTarget(callback=AsyncMock())
+        result = target._resolve_content(piece)
+        assert result == "/nonexistent/path/to/file.txt"
+
+    def test_resolve_content_returns_text_as_is(self):
+        """Text data type pieces are returned without file I/O."""
+        piece = MagicMock()
+        piece.converted_value = "plain text prompt"
+        piece.original_value = "plain text prompt"
+        piece.converted_value_data_type = "text"
+
+        target = _CallbackChatTarget(callback=AsyncMock())
+        assert target._resolve_content(piece) == "plain text prompt"
+
+    @pytest.mark.asyncio
+    async def test_binary_path_content_sent_to_callback(self, tmp_path):
+        """Callback receives file *contents* (not the path) for binary_path requests."""
+        file = tmp_path / "context.txt"
+        file.write_text("file content for callback", encoding="utf-8")
+
+        mock_callback = AsyncMock(
+            return_value={
+                "messages": [{"role": "assistant", "content": "ok"}],
+                "stream": False,
+                "session_state": None,
+                "context": {},
+            }
+        )
+
+        target = _CallbackChatTarget(callback=mock_callback, retry_enabled=False)
+
+        mock_piece = MagicMock()
+        mock_piece.id = "piece-1"
+        mock_piece.converted_value = str(file)
+        mock_piece.original_value = str(file)
+        mock_piece.converted_value_data_type = "binary_path"
+        mock_piece.conversation_id = "conv-bp"
+        mock_piece.api_role = "user"
+        mock_piece.role = "user"
+        mock_piece.labels = {}
+
+        mock_request = MagicMock()
+        mock_request.message_pieces = [mock_piece]
+        mock_request.get_piece.return_value = mock_piece
+
+        with patch.object(target, "_memory") as mock_memory, patch(
+            "azure.ai.evaluation.red_team._callback_chat_target.construct_response_from_request"
+        ) as mock_construct:
+            mock_memory.get_conversation.return_value = []
+            mock_construct.return_value = mock_request
+            await target.send_prompt_async(message=mock_request)
+
+        sent_messages = mock_callback.call_args.kwargs["messages"]
+        assert sent_messages[-1]["content"] == "file content for callback"
+
+    @pytest.mark.asyncio
+    async def test_binary_path_in_conversation_history_resolved(self, tmp_path):
+        """Conversation history pieces with binary_path are also resolved to file contents."""
+        file = tmp_path / "history_context.txt"
+        file.write_text("history file content", encoding="utf-8")
+
+        mock_callback = AsyncMock(
+            return_value={
+                "messages": [{"role": "assistant", "content": "ok"}],
+                "stream": False,
+                "session_state": None,
+                "context": {},
+            }
+        )
+
+        target = _CallbackChatTarget(callback=mock_callback, retry_enabled=False)
+
+        # Build a history message with binary_path piece
+        history_piece = MagicMock()
+        history_piece.converted_value = str(file)
+        history_piece.original_value = str(file)
+        history_piece.converted_value_data_type = "binary_path"
+        history_piece.api_role = "user"
+        history_piece.role = "user"
+
+        history_msg = MagicMock()
+        history_msg.message_pieces = [history_piece]
+
+        # Current request (plain text)
+        mock_piece = MagicMock()
+        mock_piece.id = "piece-2"
+        mock_piece.converted_value = "follow-up question"
+        mock_piece.original_value = "follow-up question"
+        mock_piece.converted_value_data_type = "text"
+        mock_piece.conversation_id = "conv-bp-hist"
+        mock_piece.api_role = "user"
+        mock_piece.role = "user"
+        mock_piece.labels = {}
+
+        mock_request = MagicMock()
+        mock_request.message_pieces = [mock_piece]
+        mock_request.get_piece.return_value = mock_piece
+
+        with patch.object(target, "_memory") as mock_memory, patch(
+            "azure.ai.evaluation.red_team._callback_chat_target.construct_response_from_request"
+        ) as mock_construct:
+            mock_memory.get_conversation.return_value = [history_msg]
+            mock_construct.return_value = mock_request
+            await target.send_prompt_async(message=mock_request)
+
+        sent_messages = mock_callback.call_args.kwargs["messages"]
+        # First message is from history — should contain file content, not path
+        assert sent_messages[0]["content"] == "history file content"
+        # Second message is the current plain-text request
+        assert sent_messages[1]["content"] == "follow-up question"