From 258f65e0e7606c678c90ce5dbef6e05da8f3c89d Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Fri, 8 Aug 2025 13:10:23 +0800
Subject: [PATCH 01/12] support multiturn in quickstart_multimodal

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 examples/llm-api/quickstart_multimodal.py | 268 ++++++++++++++++++++++
 1 file changed, 268 insertions(+)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index fc18671ee280..aba46ee621ff 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -1,6 +1,7 @@
 import argparse
 import json
 import os
+from typing import List, Optional, Union
 
 from quickstart_advanced import add_llm_args, setup_llm
 
@@ -77,6 +78,213 @@
     },
 }
 
+# Multiturn conversation examples configuration
+multiturn_conversation_examples = {
+    "image": {
+        "conversations": [{
+            "user":
+            "Describe what you see in this image.",
+            "assistant":
+            "I can see a beautiful coastal scene with a sandy beach, clear blue water, and a rocky shoreline. The image shows a peaceful natural environment with waves gently lapping against the shore."
+        }, {
+            "user":
+            "What colors are most prominent in this scene?",
+            "assistant":
+            "The most prominent colors in this coastal scene are various shades of blue from the ocean water, golden/beige tones from the sandy beach, and natural brown/gray colors from the rocky shoreline."
+        }, {
+            "user":
+            "Based on what you see, what time of day might this be?",
+            "assistant":
+            "Based on the lighting and colors in the image, this appears to be taken during daylight hours, likely in the afternoon when the sun provides good illumination of the coastal landscape."
+        }],
+        "media": [
+            "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/seashore.png"
+        ]
+    },
+    "video": {
+        "conversations": [{
+            "user":
+            "What's happening in this video?",
+            "assistant":
+            "This video shows a dynamic urban scene with people walking and moving through what appears to be a busy city environment, possibly Tokyo based on the setting."
+        }, {
+            "user":
+            "How would you describe the atmosphere of this scene?",
+            "assistant":
+            "The atmosphere appears to be lively and energetic, with people actively moving through the urban space, creating a sense of busy city life and human activity."
+        }],
+        "media": [
+            "https://huggingface.co/datasets/Efficient-Large-Model/VILA-inference-demos/resolve/main/OAI-sora-tokyo-walk.mp4"
+        ]
+    },
+    "audio": {
+        "conversations": [{
+            "user":
+            "Transcribe the audio clip into text, please don't add other text.",
+            "assistant": "What is the traffic sign in the image?"
+        }, {
+            "user":
+            "What type of question is being asked in the audio?",
+            "assistant":
+            "The audio contains a question asking about identifying a traffic sign in an image, which is a visual recognition task."
+        }],
+        "media": [
+            "https://huggingface.co/microsoft/Phi-4-multimodal-instruct/resolve/main/examples/what_is_the_traffic_sign_in_the_image.wav"
+        ]
+    },
+    "mixture_text_image": {
+        "conversations": [{
+            "user":
+            "Who invented the internet?",
+            "assistant":
+            "The internet was not invented by a single person, but rather developed through collaborative efforts over several decades. Key contributors include Vint Cerf and Bob Kahn, who developed TCP/IP protocols in the 1970s, and Tim Berners-Lee, who created the World Wide Web in 1989."
+        }, {
+            "user":
+            "Now look at this image and describe what you see.",
+            "assistant":
+            "I can see an image that appears to show a painting or artistic scene, likely created using digital tools or artistic techniques."
+        }, {
+            "user":
+            "How does this image relate to the previous topic about the internet?",
+            "assistant":
+            "This image appears to be unrelated to the internet topic we discussed earlier. It seems to be an artistic or creative work, while our previous conversation was about the technical development of the internet infrastructure."
+        }],
+        "media":
+        [[],
+         [
+             "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"
+         ],
+         [
+             "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"
+         ]]
+    }
+}
+
+
+class MultimodalConversationManager:
+    """Class for managing multimodal multiturn conversations"""
+
+    def __init__(self,
+                 llm,
+                 tokenizer,
+                 model_type,
+                 sampling_params,
+                 debug_prompt=False):
+        self.llm = llm
+        self.tokenizer = tokenizer
+        self.model_type = model_type
+        self.sampling_params = sampling_params
+        self.debug_prompt = debug_prompt
+        self.conversation_history = []
+        self.current_media = None
+        self.current_modality = None
+
+    def add_message(self,
+                    role: str,
+                    content: str,
+                    media: Optional[Union[str, List[str]]] = None):
+        """Add message to conversation history"""
+        message = {"role": role, "content": content, "media": media}
+        self.conversation_history.append(message)
+
+    def get_conversation_text(self) -> str:
+        """Get formatted conversation text for context"""
+        conversation_text = ""
+        for msg in self.conversation_history:
+            if msg["role"] == "user":
+                conversation_text += f"User: {msg['content']}\n"
+            elif msg["role"] == "assistant":
+                conversation_text += f"Assistant: {msg['content']}\n"
+        return conversation_text
+
+    def build_conversation_prompt(self, user_input: str) -> str:
+        """Build conversation prompt with proper formatting"""
+        conversation_context = self.get_conversation_text()
+
+        if conversation_context:
+            # Add the new user input to the conversation
+            full_prompt = conversation_context + f"User: {user_input}\nAssistant:"
+        else:
+            # First message in conversation
+            full_prompt = f"User: {user_input}\nAssistant:"
+
+        return full_prompt
+
+    def generate_response(self,
+                          user_input: str,
+                          media: Optional[Union[str, List[str]]] = None) -> str:
+        """Generate response to user input with conversation context"""
+        # Build conversation prompt with proper formatting
+        full_prompt = self.build_conversation_prompt(user_input)
+
+        # Debug: Show the full prompt if debug mode is enabled
+        if self.debug_prompt:
+            print(f"\n=== DEBUG: Full Prompt ===")
+            print(f"Prompt: {repr(full_prompt)}")
+            print(f"Media: {media}")
+            print("=" * 40)
+
+        # Use multimodal input loader to process input with conversation context
+        inputs = default_multimodal_input_loader(
+            tokenizer=self.tokenizer,
+            model_dir=self.llm._hf_model_dir,
+            model_type=self.model_type,
+            modality=self._determine_modality(media),
+            prompts=[full_prompt],
+            media=[media] if media else None,
+            image_data_format="pt",
+            num_frames=8,
+            device="cpu")
+
+        # Generate response
+        outputs = self.llm.generate(inputs, self.sampling_params)
+        response = outputs[0].outputs[0].text.strip()
+
+        # Add user message and assistant response to history
+        self.add_message("user", user_input, media)
+        self.add_message("assistant", response)
+
+        return response
+
+    def _determine_modality(self, media) -> str:
+        """Determine modality type based on media content"""
+        if not media:
+            return "text"
+        if isinstance(media, str):
+            if media.endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
+                return "image"
+            elif media.endswith(('.mp4', '.avi', '.mov', '.mkv')):
+                return "video"
+            elif media.endswith(('.wav', '.mp3', '.flac', '.aac')):
+                return "audio"
+        elif isinstance(media, list):
+            if len(media) == 0:
+                return "text"
+            elif len(media) == 1:
+                return self._determine_modality(media[0])
+            else:
+                # Multiple images case
+                return "multiple_image"
+        return "text"
+
+    def reset_conversation(self):
+        """Reset conversation history"""
+        self.conversation_history = []
+        self.current_media = None
+        self.current_modality = None
+
+    def print_conversation_history(self):
+        """Print current conversation history for debugging"""
+        print("\n=== Current Conversation History ===")
+        if not self.conversation_history:
+            print("No conversation history yet.")
+        else:
+            for i, msg in enumerate(self.conversation_history):
+                print(f"{i+1}. {msg['role'].title()}: {msg['content']}")
+                if msg.get('media'):
+                    print(f"   Media: {msg['media']}")
+        print("=" * 40)
+
 
 def add_multimodal_args(parser):
     parser.add_argument("--model_type",
@@ -108,6 +316,19 @@ def add_multimodal_args(parser):
                         type=str,
                         default="cpu",
                         help="The device to have the input on.")
+    # Add multiturn conversation related parameters
+    parser.add_argument("--multiturn",
+                        action="store_true",
+                        help="Enable multi-turn conversation mode.")
+    parser.add_argument(
+        "--conversation_turns",
+        type=int,
+        default=3,
+        help="Number of conversation turns for automated testing.")
+    parser.add_argument(
+        "--debug_prompt",
+        action="store_true",
+        help="Show the full prompt being sent to the model for debugging.")
     return parser
 
 
@@ -138,6 +359,38 @@ def parse_arguments():
     return args
 
 
+def run_multiturn_conversation_example(
+        conversation_manager: MultimodalConversationManager, modality: str):
+    """Run predefined multiturn conversation examples"""
+    if modality not in multiturn_conversation_examples:
+        print(f"No predefined conversation example for modality: {modality}")
+        return
+
+    example = multiturn_conversation_examples[modality]
+    conversations = example["conversations"]
+    media_list = example["media"]
+
+    print(f"\n=== Starting {modality} multiturn conversation example ===")
+
+    for i, (conv, media) in enumerate(zip(conversations, media_list)):
+        print(f"\n--- Turn {i+1} ---")
+        print(f"User: {conv['user']}")
+
+        # Generate response
+        response = conversation_manager.generate_response(conv['user'], media)
+        print(f"Assistant: {response}")
+
+        # Show expected response (for comparison)
+        print(f"Expected response: {conv['assistant']}")
+
+        # Show conversation context after each turn
+        if i < len(conversations) - 1:  # Don't show after the last turn
+            print("\nCurrent conversation context:")
+            conversation_manager.print_conversation_history()
+
+        print("-" * 50)
+
+
 def main():
     args = parse_arguments()
 
@@ -162,6 +415,21 @@ def main():
             open(os.path.join(llm._hf_model_dir, 'config.json')))['model_type']
     assert model_type in ALL_SUPPORTED_MULTIMODAL_MODELS, f"Unsupported model_type: {model_type}"
 
+    # If multiturn mode is enabled
+    if args.multiturn:
+        # Create conversation manager
+        conversation_manager = MultimodalConversationManager(
+            llm=llm,
+            tokenizer=llm.tokenizer,
+            model_type=model_type,
+            sampling_params=sampling_params,
+            debug_prompt=args.debug_prompt)
+
+        # Run predefined multiturn conversation examples
+        run_multiturn_conversation_example(conversation_manager, args.modality)
+        return
+
+    # Original single-turn processing logic
     # set prompts and media to example prompts and images if they are not provided
     if args.prompt is None:
         args.prompt = example_medias_and_prompts[args.modality]["prompt"]

From e8154c0425d251ce122cc4732b4ee4eea463ca08 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Fri, 8 Aug 2025 09:04:16 +0000
Subject: [PATCH 02/12] enable draft

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 examples/llm-api/quickstart_multimodal.py     | 326 ++++--------------
 tests/integration/defs/test_e2e.py            | 102 +++++-
 .../test_lists/qa/llm_function_full.txt       |   3 +
 3 files changed, 177 insertions(+), 254 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index aba46ee621ff..f30e6e5bd874 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -1,7 +1,6 @@
 import argparse
 import json
 import os
-from typing import List, Optional, Union
 
 from quickstart_advanced import add_llm_args, setup_llm
 
@@ -78,213 +77,6 @@
     },
 }
 
-# Multiturn conversation examples configuration
-multiturn_conversation_examples = {
-    "image": {
-        "conversations": [{
-            "user":
-            "Describe what you see in this image.",
-            "assistant":
-            "I can see a beautiful coastal scene with a sandy beach, clear blue water, and a rocky shoreline. The image shows a peaceful natural environment with waves gently lapping against the shore."
-        }, {
-            "user":
-            "What colors are most prominent in this scene?",
-            "assistant":
-            "The most prominent colors in this coastal scene are various shades of blue from the ocean water, golden/beige tones from the sandy beach, and natural brown/gray colors from the rocky shoreline."
-        }, {
-            "user":
-            "Based on what you see, what time of day might this be?",
-            "assistant":
-            "Based on the lighting and colors in the image, this appears to be taken during daylight hours, likely in the afternoon when the sun provides good illumination of the coastal landscape."
-        }],
-        "media": [
-            "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/seashore.png"
-        ]
-    },
-    "video": {
-        "conversations": [{
-            "user":
-            "What's happening in this video?",
-            "assistant":
-            "This video shows a dynamic urban scene with people walking and moving through what appears to be a busy city environment, possibly Tokyo based on the setting."
-        }, {
-            "user":
-            "How would you describe the atmosphere of this scene?",
-            "assistant":
-            "The atmosphere appears to be lively and energetic, with people actively moving through the urban space, creating a sense of busy city life and human activity."
-        }],
-        "media": [
-            "https://huggingface.co/datasets/Efficient-Large-Model/VILA-inference-demos/resolve/main/OAI-sora-tokyo-walk.mp4"
-        ]
-    },
-    "audio": {
-        "conversations": [{
-            "user":
-            "Transcribe the audio clip into text, please don't add other text.",
-            "assistant": "What is the traffic sign in the image?"
-        }, {
-            "user":
-            "What type of question is being asked in the audio?",
-            "assistant":
-            "The audio contains a question asking about identifying a traffic sign in an image, which is a visual recognition task."
-        }],
-        "media": [
-            "https://huggingface.co/microsoft/Phi-4-multimodal-instruct/resolve/main/examples/what_is_the_traffic_sign_in_the_image.wav"
-        ]
-    },
-    "mixture_text_image": {
-        "conversations": [{
-            "user":
-            "Who invented the internet?",
-            "assistant":
-            "The internet was not invented by a single person, but rather developed through collaborative efforts over several decades. Key contributors include Vint Cerf and Bob Kahn, who developed TCP/IP protocols in the 1970s, and Tim Berners-Lee, who created the World Wide Web in 1989."
-        }, {
-            "user":
-            "Now look at this image and describe what you see.",
-            "assistant":
-            "I can see an image that appears to show a painting or artistic scene, likely created using digital tools or artistic techniques."
-        }, {
-            "user":
-            "How does this image relate to the previous topic about the internet?",
-            "assistant":
-            "This image appears to be unrelated to the internet topic we discussed earlier. It seems to be an artistic or creative work, while our previous conversation was about the technical development of the internet infrastructure."
-        }],
-        "media":
-        [[],
-         [
-             "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"
-         ],
-         [
-             "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"
-         ]]
-    }
-}
-
-
-class MultimodalConversationManager:
-    """Class for managing multimodal multiturn conversations"""
-
-    def __init__(self,
-                 llm,
-                 tokenizer,
-                 model_type,
-                 sampling_params,
-                 debug_prompt=False):
-        self.llm = llm
-        self.tokenizer = tokenizer
-        self.model_type = model_type
-        self.sampling_params = sampling_params
-        self.debug_prompt = debug_prompt
-        self.conversation_history = []
-        self.current_media = None
-        self.current_modality = None
-
-    def add_message(self,
-                    role: str,
-                    content: str,
-                    media: Optional[Union[str, List[str]]] = None):
-        """Add message to conversation history"""
-        message = {"role": role, "content": content, "media": media}
-        self.conversation_history.append(message)
-
-    def get_conversation_text(self) -> str:
-        """Get formatted conversation text for context"""
-        conversation_text = ""
-        for msg in self.conversation_history:
-            if msg["role"] == "user":
-                conversation_text += f"User: {msg['content']}\n"
-            elif msg["role"] == "assistant":
-                conversation_text += f"Assistant: {msg['content']}\n"
-        return conversation_text
-
-    def build_conversation_prompt(self, user_input: str) -> str:
-        """Build conversation prompt with proper formatting"""
-        conversation_context = self.get_conversation_text()
-
-        if conversation_context:
-            # Add the new user input to the conversation
-            full_prompt = conversation_context + f"User: {user_input}\nAssistant:"
-        else:
-            # First message in conversation
-            full_prompt = f"User: {user_input}\nAssistant:"
-
-        return full_prompt
-
-    def generate_response(self,
-                          user_input: str,
-                          media: Optional[Union[str, List[str]]] = None) -> str:
-        """Generate response to user input with conversation context"""
-        # Build conversation prompt with proper formatting
-        full_prompt = self.build_conversation_prompt(user_input)
-
-        # Debug: Show the full prompt if debug mode is enabled
-        if self.debug_prompt:
-            print(f"\n=== DEBUG: Full Prompt ===")
-            print(f"Prompt: {repr(full_prompt)}")
-            print(f"Media: {media}")
-            print("=" * 40)
-
-        # Use multimodal input loader to process input with conversation context
-        inputs = default_multimodal_input_loader(
-            tokenizer=self.tokenizer,
-            model_dir=self.llm._hf_model_dir,
-            model_type=self.model_type,
-            modality=self._determine_modality(media),
-            prompts=[full_prompt],
-            media=[media] if media else None,
-            image_data_format="pt",
-            num_frames=8,
-            device="cpu")
-
-        # Generate response
-        outputs = self.llm.generate(inputs, self.sampling_params)
-        response = outputs[0].outputs[0].text.strip()
-
-        # Add user message and assistant response to history
-        self.add_message("user", user_input, media)
-        self.add_message("assistant", response)
-
-        return response
-
-    def _determine_modality(self, media) -> str:
-        """Determine modality type based on media content"""
-        if not media:
-            return "text"
-        if isinstance(media, str):
-            if media.endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
-                return "image"
-            elif media.endswith(('.mp4', '.avi', '.mov', '.mkv')):
-                return "video"
-            elif media.endswith(('.wav', '.mp3', '.flac', '.aac')):
-                return "audio"
-        elif isinstance(media, list):
-            if len(media) == 0:
-                return "text"
-            elif len(media) == 1:
-                return self._determine_modality(media[0])
-            else:
-                # Multiple images case
-                return "multiple_image"
-        return "text"
-
-    def reset_conversation(self):
-        """Reset conversation history"""
-        self.conversation_history = []
-        self.current_media = None
-        self.current_modality = None
-
-    def print_conversation_history(self):
-        """Print current conversation history for debugging"""
-        print("\n=== Current Conversation History ===")
-        if not self.conversation_history:
-            print("No conversation history yet.")
-        else:
-            for i, msg in enumerate(self.conversation_history):
-                print(f"{i+1}. {msg['role'].title()}: {msg['content']}")
-                if msg.get('media'):
-                    print(f"   Media: {msg['media']}")
-        print("=" * 40)
-
 
 def add_multimodal_args(parser):
     parser.add_argument("--model_type",
@@ -323,12 +115,8 @@ def add_multimodal_args(parser):
     parser.add_argument(
         "--conversation_turns",
         type=int,
-        default=3,
+        default=2,
         help="Number of conversation turns for automated testing.")
-    parser.add_argument(
-        "--debug_prompt",
-        action="store_true",
-        help="Show the full prompt being sent to the model for debugging.")
     return parser
 
 
@@ -359,42 +147,21 @@ def parse_arguments():
     return args
 
 
-def run_multiturn_conversation_example(
-        conversation_manager: MultimodalConversationManager, modality: str):
-    """Run predefined multiturn conversation examples"""
-    if modality not in multiturn_conversation_examples:
-        print(f"No predefined conversation example for modality: {modality}")
-        return
-
-    example = multiturn_conversation_examples[modality]
-    conversations = example["conversations"]
-    media_list = example["media"]
-
-    print(f"\n=== Starting {modality} multiturn conversation example ===")
-
-    for i, (conv, media) in enumerate(zip(conversations, media_list)):
-        print(f"\n--- Turn {i+1} ---")
-        print(f"User: {conv['user']}")
-
-        # Generate response
-        response = conversation_manager.generate_response(conv['user'], media)
-        print(f"Assistant: {response}")
-
-        # Show expected response (for comparison)
-        print(f"Expected response: {conv['assistant']}")
-
-        # Show conversation context after each turn
-        if i < len(conversations) - 1:  # Don't show after the last turn
-            print("\nCurrent conversation context:")
-            conversation_manager.print_conversation_history()
-
-        print("-" * 50)
+def clear_cuda_cache():
+    """Clear CUDA cache to prevent memory issues"""
+    try:
+        import torch
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except:
+        pass
 
 
 def main():
     args = parse_arguments()
 
     lora_config = None
+    model_class = None  # Initialize model_class to None
     if args.load_lora:
         assert args.auto_model_name is not None, "Please provide the auto model name to load LoRA config."
         import importlib
@@ -417,16 +184,71 @@ def main():
 
     # If multiturn mode is enabled
     if args.multiturn:
-        # Create conversation manager
-        conversation_manager = MultimodalConversationManager(
-            llm=llm,
-            tokenizer=llm.tokenizer,
-            model_type=model_type,
-            sampling_params=sampling_params,
-            debug_prompt=args.debug_prompt)
-
         # Run predefined multiturn conversation examples
-        run_multiturn_conversation_example(conversation_manager, args.modality)
+        assert args.prompt is not None, "Please provide a prompt for multiturn conversation."
+        assert args.media is not None, "Please provide media for multiturn conversation."
+        # Determine how many turns to run
+        max_turns = min(args.conversation_turns, len(args.prompt))
+        generated_outputs = []  # Store generated outputs for return
+
+        for i in range(max_turns):
+            print(f"\n--- Turn {i+1} ---")
+
+            try:
+                # Use multimodal input loader to process input with conversation context
+                cur_prompt = args.prompt[i]
+                inputs = default_multimodal_input_loader(
+                    tokenizer=llm.tokenizer,
+                    model_dir=llm._hf_model_dir,
+                    model_type=model_type,
+                    modality=args.modality,
+                    prompts=[cur_prompt],
+                    media=args.media,
+                    image_data_format="pt",
+                    num_frames=8,
+                    device="cpu")
+
+                lora_request = None
+                if args.load_lora:
+                    if model_class is None:
+                        raise ValueError(
+                            "model_class must be provided when load_lora is True"
+                        )
+                    lora_request = model_class.lora_request(
+                        len(inputs), args.modality, llm._hf_model_dir)
+
+                # Generate response
+                outputs = llm.generate(inputs,
+                                       sampling_params,
+                                       lora_request=lora_request)
+                assert outputs and len(
+                    outputs) > 0 and outputs[0].outputs and len(
+                        outputs[0].outputs) > 0
+                response = outputs[0].outputs[0].text.strip()
+
+                # Store generated output
+                generated_outputs.append({
+                    "turn": i + 1,
+                    "user_input": args.prompt[i],
+                    "assistant_response": response,
+                    "media": args.media
+                })
+
+                # Add to cur_prompt
+                cur_prompt = cur_prompt + f"assistant: {response}"
+
+            except Exception as e:
+                print(f"Error in turn {i+1}: {e}")
+                import traceback
+                traceback.print_exc()
+                continue
+
+            clear_cuda_cache()
+
+        for i, output in enumerate(generated_outputs):
+            print(
+                f"[{i}] Prompt: {output['user_input']!r}, Generated text: {output['assistant_response']!r}"
+            )
         return
 
     # Original single-turn processing logic
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index d247063a125b..06bc63df2b24 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1584,8 +1584,6 @@ def test_build_time_benchmark_sanity(llm_root, llm_venv):
 
 
 ### PyTorch examples
-
-
 def parse_output(text):
     results = []
     text_lists = re.split(r"\[\d+\] Prompt:", text)
@@ -2521,6 +2519,106 @@ def test_ptp_quickstart_multimodal_2gpu(llm_root, llm_venv, model_name,
     print("All answers are correct!")
 
 
+@pytest.mark.skip_less_device_memory(80000)
+@pytest.mark.parametrize("model_name,model_path", [
+    ("gemma-3-27b-it", "gemma/gemma-3-27b-it"),
+    ("mistral-small-3.1-24b-instruct", "Mistral-Small-3.1-24B-Instruct-2503"),
+    ("Phi-4-multimodal-instruct", "multimodals/Phi-4-multimodal-instruct"),
+])
+def test_ptp_quickstart_multimodal_multiturn(llm_root, llm_venv, model_name,
+                                             model_path):
+    example_root = Path(os.path.join(llm_root, "examples", "llm-api"))
+    test_data_root = Path(
+        os.path.join(llm_models_root(), "multimodals", "test_data"))
+
+    print(f"Accuracy test {model_name} image mode with example inputs.")
+
+    # Define accuracy inputs for image modality
+    accuracy_inputs = {
+        "image": {
+            "prompt": [
+                "user: Describe what you see in this image.",
+                "user: How would you describe the atmosphere of this scene?",
+            ],
+            "media": [
+                str(test_data_root / "inpaint.png"),
+            ],
+        }
+    }
+
+    # Define expected keywords for each model
+    expected_keywords = {
+        "gemma-3-27b-it": {
+            "image": [
+                ["half", "dome", "yosemite", "landmark", "rounded"],
+                ["atmosphere", "serene", "sense", "clear", "calm"],
+            ],
+        },
+        "mistral-small-3.1-24b-instruct": {
+            "image": [
+                ["depicts", "landscape", "rock", "sky", "yosemite"],
+                ["serene", "natural", "beauty", "grand", "tranquility"],
+            ],
+        },
+        "Phi-4-multimodal-instruct": {
+            "image": [
+                ["image", "depicts", "mountain", "half", "rock"],
+                ["atmosphere", "beauty", "mountain", "sheer", "landscape"],
+            ],
+        },
+    }
+    # Build command for image modality
+    cmd = [
+        str(example_root / "quickstart_multimodal.py"),
+        "--model_dir",
+        f"{llm_models_root()}/{model_path}",
+        "--modality",
+        "image",
+        "--multiturn",
+        "--prompt",
+        *accuracy_inputs["image"]["prompt"],
+        "--media",
+        *accuracy_inputs["image"]["media"],
+    ]
+
+    # Add model-specific configurations
+    if model_name == "gemma-3-27b-it":
+        # Gemma3 VLM needs a custom mask which is only supported by flashinfer backend currently.
+        # Custom mask involves bidirectional masking of image tokens in context phase. To get this
+        # correct, chunked prefill and kv cache reuse need to be turned off.
+        cmd.append("--image_format=pil")
+        cmd.append("--attention_backend=FLASHINFER")
+        cmd.append("--disable_kv_cache_reuse")
+    elif model_name == "Phi-4-multimodal-instruct":
+        # Set max_seq_len to 4096 to use short rope factor.
+        cmd.append("--max_seq_len=4096")
+        cmd.append("--load_lora")
+        cmd.append("--auto_model_name")
+        cmd.append("Phi4MMForCausalLM")
+
+    output = llm_venv.run_cmd(cmd, caller=check_output)
+    print("output:", output)
+    # Set match ratio based on model
+    match_ratio = 4.0 / 5
+    if model_name == "Phi-4-multimodal-instruct":
+        match_ratio = 0.6
+
+    # Check output accuracy
+    for prompt_output, prompt_keywords in zip(
+            parse_output(output), expected_keywords[model_name]["image"]):
+        matches = [
+            keyword in prompt_output.lower() for keyword in prompt_keywords
+        ]
+        obs_match_ratio = 1. * sum(matches) / len(matches)
+        print("prompt_output:", prompt_output)
+        print("prompt_keywords:", prompt_keywords)
+        print("matches:", matches)
+        print("obs_match_ratio:", obs_match_ratio)
+        assert obs_match_ratio >= match_ratio, f"Incorrect output!\nGenerated \"{prompt_output}\"\nExpected keywords \"{prompt_keywords}\"\n Matched keywords: {matches}\n Observed match ratio {obs_match_ratio} below threshold {match_ratio}"
+
+    print("All answers are correct!")
+
+
 @pytest.mark.parametrize("model_name,model_path", [
     ("BertForSequenceClassification", "bert/bert-base-uncased-yelp-polarity"),
 ])
diff --git a/tests/integration/test_lists/qa/llm_function_full.txt b/tests/integration/test_lists/qa/llm_function_full.txt
index 5c0a9585ffcd..42e04be7e78f 100644
--- a/tests/integration/test_lists/qa/llm_function_full.txt
+++ b/tests/integration/test_lists/qa/llm_function_full.txt
@@ -602,6 +602,9 @@ test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[image_audio]
 test_e2e.py::test_ptp_quickstart_multimodal_2gpu[gemma-3-27b-it-gemma/gemma-3-27b-it]
 test_e2e.py::test_ptp_quickstart_multimodal_2gpu[mistral-small-3.1-24b-instruct-Mistral-Small-3.1-24B-Instruct-2503]
 test_e2e.py::test_ptp_quickstart_multimodal_2gpu[Phi-4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct]
+test_e2e.py::test_ptp_quickstart_multimodal_multiturn[gemma-3-27b-it-gemma/gemma-3-27b-it]
+test_e2e.py::test_ptp_quickstart_multimodal_multiturn[mistral-small-3.1-24b-instruct-Mistral-Small-3.1-24B-Instruct-2503]
+test_e2e.py::test_ptp_quickstart_multimodal_multiturn[Phi-4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct]
 test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]
 test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]
 test_e2e.py::test_ptp_star_attention_example[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]

From 8c93aab64fa2491768b1560c7072211dff724a26 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Fri, 8 Aug 2025 09:41:42 +0000
Subject: [PATCH 03/12] fix error

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 examples/llm-api/quickstart_multimodal.py | 1 -
 tests/integration/defs/test_e2e.py        | 6 +++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index f30e6e5bd874..026426fa49b3 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -161,7 +161,6 @@ def main():
     args = parse_arguments()
 
     lora_config = None
-    model_class = None  # Initialize model_class to None
     if args.load_lora:
         assert args.auto_model_name is not None, "Please provide the auto model name to load LoRA config."
         import importlib
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index 06bc63df2b24..f0a8e84ffe79 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1590,12 +1590,12 @@ def parse_output(text):
     for item in text_lists:
         item = item.replace(os.linesep, "")
         while True:
-            match = re.search(r"(Generated text: \'(.*?)\')", item,
+            match = re.search(r"Generated text: [\"'](.*?)[\"']", item,
                               re.MULTILINE)
             if match is None:
                 break
             _, end = match.span(1)
-            results.append(match.group(2))
+            results.append(match.group(1))
             item = item[end:]
     return results
 
@@ -2562,7 +2562,7 @@ def test_ptp_quickstart_multimodal_multiturn(llm_root, llm_venv, model_name,
         },
         "Phi-4-multimodal-instruct": {
             "image": [
-                ["image", "depicts", "mountain", "half", "rock"],
+                ["depicts", "landscape", "mountain", "half", "rock"],
                 ["atmosphere", "beauty", "mountain", "sheer", "landscape"],
             ],
         },

From 2c4e90df447eedf738cf952a75896c40b828e37a Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Fri, 8 Aug 2025 14:08:14 +0000
Subject: [PATCH 04/12] update

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 examples/llm-api/quickstart_multimodal.py |  4 ++--
 tests/integration/defs/test_e2e.py        | 10 ++++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index 026426fa49b3..f000d216e0ca 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -228,13 +228,13 @@ def main():
                 # Store generated output
                 generated_outputs.append({
                     "turn": i + 1,
-                    "user_input": args.prompt[i],
+                    "user_input": cur_prompt,
                     "assistant_response": response,
                     "media": args.media
                 })
 
                 # Add to cur_prompt
-                cur_prompt = cur_prompt + f"assistant: {response}"
+                cur_prompt = cur_prompt + response
 
             except Exception as e:
                 print(f"Error in turn {i+1}: {e}")
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index f0a8e84ffe79..68d0625ba142 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1584,18 +1584,20 @@ def test_build_time_benchmark_sanity(llm_root, llm_venv):
 
 
 ### PyTorch examples
+
+
 def parse_output(text):
     results = []
     text_lists = re.split(r"\[\d+\] Prompt:", text)
     for item in text_lists:
         item = item.replace(os.linesep, "")
         while True:
-            match = re.search(r"Generated text: [\"'](.*?)[\"']", item,
+            match = re.search(r"(Generated text: \'(.*?)\')", item,
                               re.MULTILINE)
             if match is None:
                 break
             _, end = match.span(1)
-            results.append(match.group(1))
+            results.append(match.group(2))
             item = item[end:]
     return results
 
@@ -2537,8 +2539,8 @@ def test_ptp_quickstart_multimodal_multiturn(llm_root, llm_venv, model_name,
     accuracy_inputs = {
         "image": {
             "prompt": [
-                "user: Describe what you see in this image.",
-                "user: How would you describe the atmosphere of this scene?",
+                "Describe what you see in this image.",
+                "How would you describe the atmosphere of this scene?",
             ],
             "media": [
                 str(test_data_root / "inpaint.png"),

From a2788ef48abe9bac29d258d914e1cac10ee9281f Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Sun, 10 Aug 2025 23:55:50 +0800
Subject: [PATCH 05/12] update cuda_graph api

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/accuracy/test_llm_api_pytorch.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index ecb8a1980bfe..3e30ef9ee0fc 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -1033,7 +1033,7 @@ def test_cute_dsl_fp8_block_scales(
             max_num_streams=3) if torch_compile else None)
         pytorch_config = dict(
             disable_overlap_scheduler=not overlap_scheduler,
-            use_cuda_graph=cuda_graph,
+            cuda_graph_config=CudaGraphConfig() if cuda_graph else None,
             torch_compile_config=torch_compile_config,
             moe_config=MoeConfig(backend="CUTEDSL"),
         )
@@ -1191,7 +1191,7 @@ def test_cute_dsl_fp8_block_scales_4gpus(
             max_num_streams=3) if torch_compile else None)
         pytorch_config = dict(
             disable_overlap_scheduler=not overlap_scheduler,
-            use_cuda_graph=cuda_graph,
+            cuda_graph_config=CudaGraphConfig() if cuda_graph else None,
             torch_compile_config=torch_compile_config,
             moe_config=MoeConfig(backend="CUTEDSL"),
         )

From 9e4ef2b6da26cd90207412ad904c41c0de23b7f0 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Mon, 11 Aug 2025 12:42:15 +0800
Subject: [PATCH 06/12] skip pre hopper for qwen3

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/accuracy/test_disaggregated_serving.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py
index 4ee258280c7c..0e9212ba5df7 100644
--- a/tests/integration/defs/accuracy/test_disaggregated_serving.py
+++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py
@@ -671,6 +671,7 @@ def test_nixl_backend(self):
             task.evaluate(llm)
 
     @pytest.mark.parametrize("overlap_scheduler", [False, True])
+    @skip_pre_hopper
     def test_auto_dtype(self, overlap_scheduler):
         ctx_server_config = {
             "disable_overlap_scheduler": True,

From f6a3c50833eddc0125b709c919011bdf47f81e03 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Mon, 11 Aug 2025 14:23:47 +0800
Subject: [PATCH 07/12] update script path

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/test_e2e.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index 68d0625ba142..4a6a4ab18bce 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -2051,7 +2051,7 @@ def test_ptp_quickstart_advanced_8gpus(llm_root, llm_venv, model_name,
 def test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k(
         llm_root, llm_venv, model_name, model_path, cuda_graph):
     print(f"Testing {model_name} on 8 GPUs.")
-    example_root = Path(os.path.join(llm_root, "examples", "pytorch"))
+    example_root = Path(os.path.join(llm_root, "examples", "llm-api"))
     cmd = [
         str(example_root / "quickstart_advanced.py"),
         "--enable_chunked_prefill",
@@ -2076,10 +2076,12 @@ def test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k(
 @pytest.mark.skip_less_device_memory(80000)
 @pytest.mark.skip_less_device(2)
 @pytest.mark.parametrize("model_name,model_path", [
-    ("Llama3.1-70B-BF16", "llama-3.1-model/Meta-Llama-3.1-70B"),
     ('Nemotron-Super-49B-v1-BF16',
      'nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1'),
     ("Mixtral-8x7B-BF16", "Mixtral-8x7B-Instruct-v0.1"),
+    pytest.param('Llama3.1-70B-BF16',
+                 'llama-3.1-model/Meta-Llama-3.1-70B',
+                 marks=pytest.mark.skip_less_device_memory(95000)),
 ])
 def test_ptp_quickstart_advanced_2gpus_sm120(llm_root, llm_venv, model_name,
                                              model_path):

From 3539bf4787df6d7eb0856fa45f3f0a190bef4479 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Mon, 11 Aug 2025 15:27:02 +0800
Subject: [PATCH 08/12] update memory limitation

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/accuracy/test_llm_api_pytorch.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 3e30ef9ee0fc..c667bbaa10ac 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -695,6 +695,7 @@ class TestMistralSmall24B(LlmapiAccuracyTestHarness):
     MODEL_NAME = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
     MODEL_PATH = f"{llm_models_root()}/Mistral-Small-3.1-24B-Instruct-2503"
 
+    @pytest.mark.skip_less_device_memory(80000)
     def test_auto_dtype(self):
         with LLM(self.MODEL_PATH) as llm:
             task = CnnDailymail(self.MODEL_NAME)

From 42e03b6a2bfe26dfda07fad9834a1214884f4552 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Mon, 11 Aug 2025 16:40:19 +0800
Subject: [PATCH 09/12] update skip context

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 .../integration/defs/accuracy/test_disaggregated_serving.py  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py
index 0e9212ba5df7..8fd7508b075c 100644
--- a/tests/integration/defs/accuracy/test_disaggregated_serving.py
+++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py
@@ -246,7 +246,7 @@ def run_parallel_test(model_name: str, model_path: str, ctx_pp: int,
     total_ctx_gpus = ctx_tp * ctx_pp * ctx_instances
     total_gen_gpus = gen_tp * gen_pp * gen_instances
     if total_ctx_gpus + total_gen_gpus > get_device_count():
-        pytest.fail(
+        pytest.skip(
             f"Not enough devices for {ctx_instances} ctx instances (ctx_pp={ctx_pp}*ctx_tp={ctx_tp}) + {gen_instances} gen instances (gen_pp={gen_pp}*gen_tp={gen_tp}), total: {total_ctx_gpus + total_gen_gpus}"
         )
 
@@ -378,6 +378,7 @@ def test_ngram(self):
             task = GSM8K(self.MODEL_NAME)
             task.evaluate(llm)
 
+    @skip_pre_hopper
     @parametrize_with_ids("overlap_scheduler", [True, False])
     @parametrize_with_ids("eagle3_one_model", [True, False])
     def test_eagle3(self, overlap_scheduler, eagle3_one_model):
@@ -461,6 +462,7 @@ def test_multi_instance(self, testset):
 
 @pytest.mark.skip_less_device_memory(140000)
 @pytest.mark.timeout(3600)
+@pytest.mark.skip_less_device(4)
 class TestLlama4ScoutInstruct(LlmapiAccuracyTestHarness):
     MODEL_NAME = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
     MODEL_PATH = f"{llm_models_root()}/llama4-models/Llama-4-Scout-17B-16E-Instruct"
@@ -540,6 +542,7 @@ def test_nixl_backend(self):
     @parametrize_with_ids("overlap_scheduler", [True, False])
     @parametrize_with_ids("mtp_nextn",
                           [0, pytest.param(2, marks=skip_pre_hopper)])
+    @pytest.mark.skip_less_device(4)
     def test_auto_dtype(self, overlap_scheduler, mtp_nextn):
         ctx_server_config = {"disable_overlap_scheduler": True}
         gen_server_config = {"disable_overlap_scheduler": not overlap_scheduler}

From 40abe0e1e86e22f2d2a813684d14b5ba742e33cf Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Tue, 12 Aug 2025 10:18:40 +0800
Subject: [PATCH 10/12] update multiturn prompt

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 examples/llm-api/quickstart_multimodal.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index f000d216e0ca..97c9ec4bf6ac 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -190,12 +190,16 @@ def main():
         max_turns = min(args.conversation_turns, len(args.prompt))
         generated_outputs = []  # Store generated outputs for return
 
+        # Initialize conversation history with the first prompt
+        conversation_history = args.prompt[0] if args.prompt else ""
+
         for i in range(max_turns):
             print(f"\n--- Turn {i+1} ---")
 
             try:
                 # Use multimodal input loader to process input with conversation context
-                cur_prompt = args.prompt[i]
+                # Use accumulated conversation history instead of just the current prompt
+                cur_prompt = conversation_history
                 inputs = default_multimodal_input_loader(
                     tokenizer=llm.tokenizer,
                     model_dir=llm._hf_model_dir,
@@ -233,8 +237,10 @@ def main():
                     "media": args.media
                 })
 
-                # Add to cur_prompt
-                cur_prompt = cur_prompt + response
+                conversation_history = conversation_history + "\n" + response
+                if i + 1 < len(args.prompt):
+                    conversation_history = conversation_history + "\n" + args.prompt[
+                        i + 1]
 
             except Exception as e:
                 print(f"Error in turn {i+1}: {e}")

From 2b589503b979e02c38ef9586f2a1062a54ccf3dd Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Tue, 12 Aug 2025 10:18:57 +0800
Subject: [PATCH 11/12] update acc config

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/accuracy/references/gsm8k.yaml | 7 +++++++
 tests/integration/defs/accuracy/references/mmlu.yaml  | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml
index 5e72cc225e1d..ba6ad70265bb 100644
--- a/tests/integration/defs/accuracy/references/gsm8k.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k.yaml
@@ -19,6 +19,13 @@ meta-llama/Llama-3.3-70B-Instruct:
     accuracy: 84.08
 meta-llama/Llama-4-Maverick-17B-128E-Instruct:
   - accuracy: 92.20
+  - quant_algo: FP8
+    kv_cache_quant_algo: FP8
+    accuracy: 92.20
+  - quant_algo: FP8
+    kv_cache_quant_algo: FP8
+    spec_dec_algo: Eagle
+    accuracy: 92.20
 meta-llama/Llama-4-Scout-17B-16E-Instruct:
   - accuracy: 89.70
   - quant_algo: NVFP4
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index 70cfb64bfbef..01516a0b7036 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -73,6 +73,9 @@ meta-llama/Llama-4-Maverick-17B-128E-Instruct:
     kv_cache_quant_algo: FP8
     spec_dec_algo: Eagle
     accuracy: 86.40
+  - quant_algo: FP8
+    kv_cache_quant_algo: FP8
+    accuracy: 86.40
 meta-llama/Llama-4-Scout-17B-16E-Instruct:
   - accuracy: 80.00
   - quant_algo: NVFP4

From 9a298af5759c8b313717e1e7f1d251a9ee679b8f Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Tue, 12 Aug 2025 14:12:49 +0800
Subject: [PATCH 12/12] fix comments

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 examples/llm-api/quickstart_multimodal.py | 12 ------------
 tests/integration/defs/test_e2e.py        | 10 +++++-----
 2 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index 97c9ec4bf6ac..dda7ceeadcd0 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -147,16 +147,6 @@ def parse_arguments():
     return args
 
 
-def clear_cuda_cache():
-    """Clear CUDA cache to prevent memory issues"""
-    try:
-        import torch
-        if torch.cuda.is_available():
-            torch.cuda.empty_cache()
-    except:
-        pass
-
-
 def main():
     args = parse_arguments()
 
@@ -248,8 +238,6 @@ def main():
                 traceback.print_exc()
                 continue
 
-            clear_cuda_cache()
-
         for i, output in enumerate(generated_outputs):
             print(
                 f"[{i}] Prompt: {output['user_input']!r}, Generated text: {output['assistant_response']!r}"
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index 4a6a4ab18bce..2921b4c2d042 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -2555,19 +2555,19 @@ def test_ptp_quickstart_multimodal_multiturn(llm_root, llm_venv, model_name,
         "gemma-3-27b-it": {
             "image": [
                 ["half", "dome", "yosemite", "landmark", "rounded"],
-                ["atmosphere", "serene", "sense", "clear", "calm"],
+                ["atmosphere", "peaceful", "majestic", "calm", "quiet"],
             ],
         },
         "mistral-small-3.1-24b-instruct": {
             "image": [
-                ["depicts", "landscape", "rock", "sky", "yosemite"],
-                ["serene", "natural", "beauty", "grand", "tranquility"],
+                ["depicts", "landscape", "rock", "sky", "high", "altitude"],
+                ["atmosphere", "serene", "majestic", "sense", "tranquility"],
             ],
         },
         "Phi-4-multimodal-instruct": {
             "image": [
-                ["depicts", "landscape", "mountain", "half", "rock"],
-                ["atmosphere", "beauty", "mountain", "sheer", "landscape"],
+                ["depicts", "landscape", "mountain", "half", "dome"],
+                ["atmosphere", "serene", "sense", "tranquility", "peace."],
             ],
         },
     }