[None][feat] DSv4 follow-up: runtime KV and cache foundations

cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h

@@ -327,19 +327,32 @@ struct KvCacheStats
     std::size_t allocatedBytes{};
 };
 
-/// @brief Per-iteration KV cache statistics. All delta counters represent changes since the last call to
-/// getIterationStats(). Gauges are instantaneous snapshots.
+/// @brief Per-iteration KV cache statistics. All delta counters and peak gauges represent values since the last call
+/// to getIterationStats(). Snapshot gauges are instantaneous.
 struct KvCacheIterationStats
 {
     // --- Instantaneous gauges ---
     // Primary (GPU) pool
     SizeType32 primaryMaxNumBlocks{0};
     SizeType32 primaryFreeNumBlocks{0};
     SizeType32 primaryUsedNumBlocks{0};
+    // Cached-but-unpinned blocks in the primary pool. Distinct from primaryUsedNumBlocks,
+    // which also counts blocks pinned during onboard memcpy windows.
+    SizeType32 primaryEvictableNumBlocks{0};
+    SizeType32 primaryPeakFreeNumBlocks{0};
+    SizeType32 primaryPeakUsedNumBlocks{0};
+    SizeType32 primaryPeakEvictableNumBlocks{0};
     // Secondary (host) pool
     SizeType32 secondaryMaxNumBlocks{0};
     SizeType32 secondaryFreeNumBlocks{0};
     SizeType32 secondaryUsedNumBlocks{0};
+    // Cached-but-unpinned blocks in the secondary pool. Useful to gauge "how full is the
+    // host cache"; secondaryUsedNumBlocks only counts pinned blocks during the sub-ms
+    // onboard memcpy window so it cannot answer that question on its own.
+    SizeType32 secondaryEvictableNumBlocks{0};
+    SizeType32 secondaryPeakFreeNumBlocks{0};
+    SizeType32 secondaryPeakUsedNumBlocks{0};
+    SizeType32 secondaryPeakEvictableNumBlocks{0};
 
     // --- Per-iteration deltas (reset on each read) ---
     // Context phase: block allocation and reuse
@@ -361,6 +374,11 @@ struct KvCacheIterationStats
     // Intra-device (GPU → GPU) block copies (e.g. partial reuse when source block has refs)
     SizeType32 iterIntraDeviceCopyBlocks{0};
     std::size_t iterIntraDeviceCopyBytes{0};
+
+    // Pages released by LRU from the last cache tier without ever being onboarded back
+    // to GPU during their stay at that tier (i.e. fully dropped from the hierarchy).
+    SizeType32 iterHostDroppedBlocks{0};
+    std::size_t iterHostDroppedBytes{0};
 };
 
 // Basic building block of a paged KV cache - a single

cpp/include/tensorrt_llm/executor/executor.h

@@ -1556,6 +1556,9 @@ class ExecutorConfig
     // Per request stats may have additional overhead due to going through all requests. Turned off by default.
     static constexpr SizeType32 kDefaultRequestStatsMaxIterations = 0;
 
+    // A value of -1 keeps all iteration/request stats until they are fetched.
+    static constexpr SizeType32 kUnlimitedStatsMaxIterations = -1;
+
     explicit ExecutorConfig(SizeType32 maxBeamWidth = 1, SchedulerConfig schedulerConfig = SchedulerConfig(),
         KvCacheConfig kvCacheConfig = KvCacheConfig(), bool enableChunkedContext = true, bool normalizeLogProbs = false,
         SizeType32 iterStatsMaxIterations = kDefaultIterStatsMaxIterations,
@@ -1661,9 +1664,11 @@ class ExecutorConfig
     bool mNormalizeLogProbs;
 
     /// @brief Controls the maximum number of iterations for which to keep statistics.
+    ///        Set to -1 to keep all iteration statistics. Set to 0 to disable iteration statistics.
     SizeType32 mIterStatsMaxIterations;
 
     /// @brief Controls the maximum number of iterations for which to keep per-request statistics.
+    ///        Set to -1 to keep all per-request statistics. Set to 0 to disable per-request statistics.
     SizeType32 mRequestStatsMaxIterations;
 
     /// @brief The type of batching strategy to use. See BatchingType.
@@ -1944,12 +1949,12 @@ class Executor
     void shutdown();
 
     /// @brief  Returns the per-iterations statistics computed since last call to getLatestIterationStats.
-    ///         Contains at most iterStatsMaxIterations iterations.
+    ///         Contains at most iterStatsMaxIterations iterations, or all iterations when set to -1.
     /// @return Iteration stats
     std::deque<IterationStats> getLatestIterationStats();
 
     /// @brief  Returns the request stats of each iteration computed since last call to getLatestRequestStats.
-    ///         Contains at most requestStatsMaxIterations iterations.
+    ///         Contains at most requestStatsMaxIterations iterations, or all iterations when set to -1.
     /// @return Request stats grouped by iterations
     std::deque<RequestStatsPerIteration> getLatestRequestStats();
 

cpp/tensorrt_llm/executor/cacheTransceiverConfig.cpp

@@ -28,8 +28,8 @@ CacheTransceiverConfig::CacheTransceiverConfig(std::optional<BackendType> backen
     , mMaxTokensInBuffer(maxNumTokens)
     , mKvTransferTimeoutMs(kvTransferTimeoutMs)
     , mKvTransferSenderFutureTimeoutMs(kvTransferSenderFutureTimeoutMs)
-    , mKvTransferPollIntervalMs(kvTransferPollIntervalMs)
 {
+    setKvTransferPollIntervalMs(kvTransferPollIntervalMs);
 }
 
 bool CacheTransceiverConfig::operator==(CacheTransceiverConfig const& other) const

cpp/tensorrt_llm/executor/executorConfig.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -65,8 +65,8 @@ ExecutorConfig::ExecutorConfig(SizeType32 maxBeamWidth, SchedulerConfig schedule
     , mEnableTrtOverlap(enableTrtOverlap)
     , mFailFastOnAttentionWindowTooLarge(failFastOnAttentionWindowTooLarge)
 {
-    TLLM_CHECK(iterStatsMaxIterations >= 0);
-    TLLM_CHECK(requestStatsMaxIterations >= 0);
+    TLLM_CHECK(iterStatsMaxIterations >= kUnlimitedStatsMaxIterations);
+    TLLM_CHECK(requestStatsMaxIterations >= kUnlimitedStatsMaxIterations);
     TLLM_CHECK(mMaxBeamWidth > 0);
     TLLM_CHECK(maxSeqIdleMicroseconds > 0);
 }
@@ -271,13 +271,13 @@ void ExecutorConfig::setNormalizeLogProbs(bool normalizeLogProbs)
 void ExecutorConfig::setIterStatsMaxIterations(SizeType32 iterStatsMaxIterations)
 {
     mIterStatsMaxIterations = iterStatsMaxIterations;
-    TLLM_CHECK(mIterStatsMaxIterations >= 0);
+    TLLM_CHECK(mIterStatsMaxIterations >= kUnlimitedStatsMaxIterations);
 }
 
 void ExecutorConfig::setRequestStatsMaxIterations(SizeType32 requestStatsMaxIterations)
 {
     mRequestStatsMaxIterations = requestStatsMaxIterations;
-    TLLM_CHECK(mRequestStatsMaxIterations >= 0);
+    TLLM_CHECK(mRequestStatsMaxIterations >= kUnlimitedStatsMaxIterations);
 }
 
 void ExecutorConfig::setBatchingType(BatchingType batchingType)

cpp/tensorrt_llm/executor/executorImpl.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -86,6 +86,16 @@ namespace
     return fixedExecutorConfig;
 }
 
+[[nodiscard]] bool statsBufferIsEnabled(SizeType32 maxIterations)
+{
+    return maxIterations != 0;
+}
+
+[[nodiscard]] bool statsBufferIsBounded(SizeType32 maxIterations)
+{
+    return maxIterations > 0;
+}
+
 SizeType32 getNumChildRequests(Request const& request)
 {
     auto samplingConfig = request.getSamplingConfig();
@@ -1908,34 +1918,38 @@ RequestStatsPerIteration Executor::Impl::getCurrentRequestStats(
 void Executor::Impl::appendCurrentIterStats(IterationStats&& currentIterStats)
 {
     std::scoped_lock<std::mutex> lck(mIterStatsMtx);
-    if (mIterationStats.size() >= mIterStatsMaxIterations)
+    if (statsBufferIsBounded(mIterStatsMaxIterations))
     {
-        mIterationStats.pop_front();
+        auto const maxIterStats = static_cast<std::size_t>(mIterStatsMaxIterations);
+        if (mIterationStats.size() >= maxIterStats)
+        {
+            mIterationStats.pop_front();
+        }
     }
     mIterationStats.emplace_back(std::move(currentIterStats));
 }
 
 void Executor::Impl::appendMultipleIterStats(std::vector<IterationStats>&& currentIterStatsVec)
 {
     std::scoped_lock<std::mutex> lck(mIterStatsMtx);
-    if (mIterationStats.size() + currentIterStatsVec.size() > mIterStatsMaxIterations)
+    mIterationStats.insert(mIterationStats.end(), std::make_move_iterator(currentIterStatsVec.begin()),
+        std::make_move_iterator(currentIterStatsVec.end()));
+    if (statsBufferIsBounded(mIterStatsMaxIterations))
     {
-        size_t removeCount = mIterationStats.size() + currentIterStatsVec.size() - mIterStatsMaxIterations;
-        for (size_t i = 0; i < removeCount; i++)
+        auto const maxIterStats = static_cast<std::size_t>(mIterStatsMaxIterations);
+        while (mIterationStats.size() > maxIterStats)
         {
             mIterationStats.pop_front();
         }
     }
-    mIterationStats.insert(mIterationStats.end(), std::make_move_iterator(currentIterStatsVec.begin()),
-        std::make_move_iterator(currentIterStatsVec.end()));
 }
 
 void Executor::Impl::updateIterationStats(RequestList const& activeRequests, double iterLatencyMS,
     SizeType32 numNewActiveRequests, double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests,
     bool flushToOrchestrator)
 {
     NVTX3_SCOPED_RANGE(updateIterationStats);
-    if (mIterStatsMaxIterations > 0 && mIsLeader)
+    if (statsBufferIsEnabled(mIterStatsMaxIterations) && mIsLeader)
     {
         auto currentIterStats = getCurrentIterationStats(
             activeRequests, iterLatencyMS, numNewActiveRequests, newActiveRequestsQueueLatencyMS, numCompletedRequests);
@@ -1972,33 +1986,37 @@ void Executor::Impl::updateIterationStats(RequestList const& activeRequests, dou
 void Executor::Impl::appendCurrentRequestStats(RequestStatsPerIteration&& currentRequestStats)
 {
     std::scoped_lock<std::mutex> lck(mRequestStatsMtx);
-    if (mRequestStats.size() >= mRequestStatsMaxIterations)
+    if (statsBufferIsBounded(mRequestStatsMaxIterations))
     {
-        mRequestStats.pop_front();
+        auto const maxRequestStats = static_cast<std::size_t>(mRequestStatsMaxIterations);
+        if (mRequestStats.size() >= maxRequestStats)
+        {
+            mRequestStats.pop_front();
+        }
     }
     mRequestStats.emplace_back(std::move(currentRequestStats));
 }
 
 void Executor::Impl::appendMultipleRequestStats(std::vector<RequestStatsPerIteration>&& currentRequestStatsVec)
 {
     std::scoped_lock<std::mutex> lck(mRequestStatsMtx);
-    if (mRequestStats.size() + currentRequestStatsVec.size() > mRequestStatsMaxIterations)
+    mRequestStats.insert(mRequestStats.end(), std::make_move_iterator(currentRequestStatsVec.begin()),
+        std::make_move_iterator(currentRequestStatsVec.end()));
+    if (statsBufferIsBounded(mRequestStatsMaxIterations))
     {
-        size_t removeCount = mRequestStats.size() + currentRequestStatsVec.size() - mRequestStatsMaxIterations;
-        for (size_t i = 0; i < removeCount; i++)
+        auto const maxRequestStats = static_cast<std::size_t>(mRequestStatsMaxIterations);
+        while (mRequestStats.size() > maxRequestStats)
         {
             mRequestStats.pop_front();
         }
     }
-    mRequestStats.insert(mRequestStats.end(), std::make_move_iterator(currentRequestStatsVec.begin()),
-        std::make_move_iterator(currentRequestStatsVec.end()));
 }
 
 void Executor::Impl::updateRequestStats(
     RequestList const& activeRequests, RequestList const& finishedRequests, bool flushToOrchestrator)
 {
     NVTX3_SCOPED_RANGE(updateRequestStats);
-    if (mRequestStatsMaxIterations > 0 && mIsLeader)
+    if (statsBufferIsEnabled(mRequestStatsMaxIterations) && mIsLeader)
     {
         // Add current iteration request stats
         auto currentRequestStats = getCurrentRequestStats(activeRequests, finishedRequests);

cpp/tensorrt_llm/executor/executorImpl.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -310,12 +310,12 @@ class Executor::Impl
     std::unordered_map<IdType, std::vector<IdType>> mChildReqIdsMap;
 
     // Iteration stats
-    IterationType mIterStatsMaxIterations;
+    SizeType32 mIterStatsMaxIterations;
     std::mutex mIterStatsMtx;
     std::deque<IterationStats> mIterationStats;
 
     // Request stats
-    IterationType mRequestStatsMaxIterations;
+    SizeType32 mRequestStatsMaxIterations;
     std::mutex mRequestStatsMtx;
     std::deque<RequestStatsPerIteration> mRequestStats;
 

cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp

@@ -379,9 +379,17 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m)
         .def_rw("primary_max_num_blocks", &tbk::KvCacheIterationStats::primaryMaxNumBlocks)
         .def_rw("primary_free_num_blocks", &tbk::KvCacheIterationStats::primaryFreeNumBlocks)
         .def_rw("primary_used_num_blocks", &tbk::KvCacheIterationStats::primaryUsedNumBlocks)
+        .def_rw("primary_evictable_num_blocks", &tbk::KvCacheIterationStats::primaryEvictableNumBlocks)
+        .def_rw("primary_peak_free_num_blocks", &tbk::KvCacheIterationStats::primaryPeakFreeNumBlocks)
+        .def_rw("primary_peak_used_num_blocks", &tbk::KvCacheIterationStats::primaryPeakUsedNumBlocks)
+        .def_rw("primary_peak_evictable_num_blocks", &tbk::KvCacheIterationStats::primaryPeakEvictableNumBlocks)
         .def_rw("secondary_max_num_blocks", &tbk::KvCacheIterationStats::secondaryMaxNumBlocks)
         .def_rw("secondary_free_num_blocks", &tbk::KvCacheIterationStats::secondaryFreeNumBlocks)
         .def_rw("secondary_used_num_blocks", &tbk::KvCacheIterationStats::secondaryUsedNumBlocks)
+        .def_rw("secondary_evictable_num_blocks", &tbk::KvCacheIterationStats::secondaryEvictableNumBlocks)
+        .def_rw("secondary_peak_free_num_blocks", &tbk::KvCacheIterationStats::secondaryPeakFreeNumBlocks)
+        .def_rw("secondary_peak_used_num_blocks", &tbk::KvCacheIterationStats::secondaryPeakUsedNumBlocks)
+        .def_rw("secondary_peak_evictable_num_blocks", &tbk::KvCacheIterationStats::secondaryPeakEvictableNumBlocks)
         .def_rw("iter_alloc_total_blocks", &tbk::KvCacheIterationStats::iterAllocTotalBlocks)
         .def_rw("iter_alloc_new_blocks", &tbk::KvCacheIterationStats::iterAllocNewBlocks)
         .def_rw("iter_reused_blocks", &tbk::KvCacheIterationStats::iterReusedBlocks)
@@ -395,7 +403,9 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m)
         .def_rw("iter_offload_blocks", &tbk::KvCacheIterationStats::iterOffloadBlocks)
         .def_rw("iter_offload_bytes", &tbk::KvCacheIterationStats::iterOffloadBytes)
         .def_rw("iter_intra_device_copy_blocks", &tbk::KvCacheIterationStats::iterIntraDeviceCopyBlocks)
-        .def_rw("iter_intra_device_copy_bytes", &tbk::KvCacheIterationStats::iterIntraDeviceCopyBytes);
+        .def_rw("iter_intra_device_copy_bytes", &tbk::KvCacheIterationStats::iterIntraDeviceCopyBytes)
+        .def_rw("iter_host_dropped_blocks", &tbk::KvCacheIterationStats::iterHostDroppedBlocks)
+        .def_rw("iter_host_dropped_bytes", &tbk::KvCacheIterationStats::iterHostDroppedBytes);
 
     nb::class_<tbk::BlockKey>(m, "BlockKey")
         .def(nb::init<>())