From 0a5fa4476ae1a2a11c56c5123a5cf220ab0e9b96 Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:35:22 -0700 Subject: [PATCH] [nvbugs/6402058][fix] Remove dead per-forward torch.cuda.Event allocation `PyTorchModelEngine._forward_step_intra_graph` allocated a fresh `torch.cuda.Event` every forward pass into `self._prepare_inputs_event` and recorded it, but its only reader (`wait_for_input_copy()`) has zero callers anywhere in the codebase. On SM120 + MTP nextn=2 + attention_dp + overlap scheduler + torch.compile with piecewise CUDA graph, the per-iteration event churn (allocate, record, drop previous for GC) can serialize `cudaEventDestroy` in the Python GC thread against in-flight MoE all-to-all streams, manifesting as a silent IPC-poll hang. Delete the allocation, the class-level attribute, and the unused `wait_for_input_copy` method; unwaive the affected test. Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Signed-off-by: handongl --- tensorrt_llm/_torch/pyexecutor/model_engine.py | 12 ------------ tests/integration/test_lists/waives.txt | 1 - 2 files changed, 13 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 50aa317e9e43..f4875bc0d105 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -811,8 +811,6 @@ def __init__( self.kv_cache_dtype_byte_size = self.get_kv_cache_dtype_byte_size() - self._prepare_inputs_event: Optional[torch.cuda.Event] = None - # Cache for enc-dec cross-attention stable generation steps. # Populated on the first CUDA-graph generation step; cleared whenever # the batch composition changes (new encoder request arrives). @@ -6154,8 +6152,6 @@ def forward(self, new_tensors_device, cache_indirection_buffer, num_accepted_tokens_device, req_id_to_old_request, resource_manager, can_run_graph) - self._prepare_inputs_event = torch.cuda.Event() - self._prepare_inputs_event.record() with with_shared_pool(self.cuda_graph_runner.get_graph_pool()): if not can_run_graph: @@ -6657,11 +6653,3 @@ def _execute_logit_post_processors(self, logits_tensor, beam_width, token_ids, logits_row_offset) logits_row_offset += beam_width - - def wait_for_input_copy(self): - """ - Wait for input preparation and H2D copy of previous iteration before modifying host input, - otherwise the input of previous iteration will be overwritten. - """ - if self._prepare_inputs_event is not None: - self._prepare_inputs_event.synchronize() diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6dc4776bb633..a77d9245715b 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -32,7 +32,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[baseline] accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[mtp3_fp8kv_chunked] SKIP (https://nvbugs/5989920) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6305404) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6426847) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6402058) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6428057) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337)