From ed6cf9a1c8c2387d50134bf9d119d169658eb235 Mon Sep 17 00:00:00 2001 From: Yuewei Na Date: Wed, 18 Mar 2026 01:13:17 -0700 Subject: [PATCH] [NVBug 5969206][fix] Break circular dependency in disagg KV transfer timeout cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In disaggregated serving, cancelled requests stuck in state 21 (kDISAGG_CONTEXT_TRANS_IN_PROGRESS) leak KV cache blocks permanently. The kv_transfer_timeout_ms mechanism has a two-phase design: - Phase 1 (_check_kv_transfer_timeout): flags timed-out requests every iteration - Phase 2 (_check_disagg_ctx_cache_transfer_status): cleans up flagged requests Phase 2 only ran inside _send_kv_async, which requires new context requests to be scheduled. Once the KV pool is exhausted by leaked blocks, no new requests can be scheduled, so Phase 2 never runs — creating an unrecoverable circular dependency. Move the Phase 2 call to the main executor loop so it runs every iteration regardless of whether new requests are being scheduled. This allows timed-out requests to be cleaned up even when the pool is fully exhausted. Reproducer: 1P1D disagg with Qwen/Qwen3-0.6B on L40 PCIe, 4+ cancel cycles with concurrency=300. The prefill worker enters a permanent timeout loop with "Timed out waiting for context KV cache transfer after 1000 milliseconds" and GPU utilization drops to 0%. Also adds diagnostic logging to the cancel request path for debuggability. Signed-off-by: Yuewei Na --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index e23b9a8a9448..c0d489907a22 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -2197,6 +2197,13 @@ def _executor_loop_overlap(self): if self.kv_cache_transceiver and self.async_transfer_manager.has_any_inflight_requests( ): self._check_kv_transfer_timeout() + # Check for timed-out context transfers every iteration, + # not just inside _send_kv_async. This breaks the circular + # dependency where Phase 2 cleanup requires new requests + # to be scheduled, but scheduling is blocked by exhausted + # KV cache pool held by timed-out requests. + # See: NVBugs 5969206 + self._check_disagg_ctx_cache_transfer_status(0) self._kv_connector_terminate_requests() @@ -3206,15 +3213,26 @@ def _try_cancel_request(self, request) -> bool: return True if not self._is_request_in_transmission(request): + logger.info( + f"[CANCEL-DIAG] Request {request.py_request_id} not in transmission " + f"(state={request.state}), cancel OK") return True - return self.kv_cache_transceiver.cancel_request(request) + result = self.kv_cache_transceiver.cancel_request(request) + logger.warning( + f"[CANCEL-DIAG] Request {request.py_request_id} in transmission " + f"(state={request.state}), cancel_request returned {result}") + return result @nvtx_range("_handle_canceled_requests") def _handle_canceled_requests(self): if len(self.canceled_req_ids) == 0: return + logger.info( + f"[CANCEL-DIAG] Processing {len(self.canceled_req_ids)} cancel requests: " + f"{self.canceled_req_ids[:5]}...") + # Create set from list of canceled request ids to speed up canceled test canceled_req_ids_set = set(self.canceled_req_ids)