Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion tensorrt_llm/_torch/pyexecutor/py_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2197,6 +2197,13 @@ def _executor_loop_overlap(self):
if self.kv_cache_transceiver and self.async_transfer_manager.has_any_inflight_requests(
):
self._check_kv_transfer_timeout()
# Check for timed-out context transfers every iteration,
# not just inside _send_kv_async. This breaks the circular
# dependency where Phase 2 cleanup requires new requests
# to be scheduled, but scheduling is blocked by exhausted
# KV cache pool held by timed-out requests.
# See: NVBugs 5969206
self._check_disagg_ctx_cache_transfer_status(0)

self._kv_connector_terminate_requests()

Expand Down Expand Up @@ -3206,15 +3213,26 @@ def _try_cancel_request(self, request) -> bool:
return True

if not self._is_request_in_transmission(request):
logger.info(
f"[CANCEL-DIAG] Request {request.py_request_id} not in transmission "
f"(state={request.state}), cancel OK")
return True

return self.kv_cache_transceiver.cancel_request(request)
result = self.kv_cache_transceiver.cancel_request(request)
logger.warning(
f"[CANCEL-DIAG] Request {request.py_request_id} in transmission "
f"(state={request.state}), cancel_request returned {result}")
return result

@nvtx_range("_handle_canceled_requests")
def _handle_canceled_requests(self):
if len(self.canceled_req_ids) == 0:
return

logger.info(
f"[CANCEL-DIAG] Processing {len(self.canceled_req_ids)} cancel requests: "
f"{self.canceled_req_ids[:5]}...")

# Create set from list of canceled request ids to speed up canceled test
canceled_req_ids_set = set(self.canceled_req_ids)

Expand Down
Loading