From 701d5eb83446fa37f90dde83628cc7ccf825cc67 Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:17:33 -0700 Subject: [PATCH] [nvbugs/6426852][fix] Force GC before ipc_collect per partial-update RPC Partial-update loops on MoE models (Qwen3-30B-A3B) issue many per-suffix update_weights RPCs. Each RPC's worker side reconstructs CUDA IPC tensors via rebuild_cuda_tensor (cudaIpcOpenMemHandle) into 'weights', then drops the dict. Without an explicit gc.collect(), the underlying storages (backed by IPC mappings) can linger in unreachable cycles until the next full GC, so the following torch.cuda.ipc_collect() finds nothing to reclaim and the CUDA driver's IPC handle table saturates before the finalize step, producing cudaErrorMapBufferObjectFailed on later RPCs. Add gc.collect() between 'del weights' and 'torch.cuda.ipc_collect()' in WorkerExtension.update_weights so IPC-mapped storages are actually released before the CUDA IPC pool sweep in each iteration. Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- tensorrt_llm/llmapi/rlhf_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorrt_llm/llmapi/rlhf_utils.py b/tensorrt_llm/llmapi/rlhf_utils.py index 47135c765678..958529827025 100644 --- a/tensorrt_llm/llmapi/rlhf_utils.py +++ b/tensorrt_llm/llmapi/rlhf_utils.py @@ -124,6 +124,11 @@ def update_weights(self, ipc_handles: Optional[dict] = None): self.engine.model_engine.model, weights, allow_partial_loading=True ) del weights + # ipc_collect only reclaims mappings whose Python refs are + # already gone; force GC so cudaIpcOpenMemHandle storages are + # released before the sweep, or the IPC handle table saturates + # across repeated RPCs. + gc.collect() torch.cuda.ipc_collect() else: logger.info("Finalize update weights")