From 434660586853295c406b70149fe2eee8ec93154d Mon Sep 17 00:00:00 2001 From: Xianjie <5410381+qiaoxj07@users.noreply.github.com> Date: Mon, 22 Sep 2025 18:32:06 -0700 Subject: [PATCH 1/4] update disagg genonly benchmark Signed-off-by: Xianjie <5410381+qiaoxj07@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 40dd9f9b071d..de97b961351e 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -1143,17 +1143,10 @@ def _executor_loop_overlap(self): torch.cuda.set_device(self.device_id) # ensure the context is created, otherwise, some MPI calls will fail. CUASSERT(cudart.cudaSetDevice(self.device_id)) - if self.dist.rank == 0 and not self.is_warmup and self.benchmark_req_queues_size > 0 and self.kv_cache_transceiver: - while self.executor_request_queue.get_request_queue_size( - ) < self.benchmark_req_queues_size: - logger.info( - f"sleep 5 seconds, num_request_queue: {self.executor_request_queue.get_request_queue_size()}" - ) - time.sleep(5) - with self._profiler() as profile_step: iter_start_time = time.time() iter_stats = None + can_forward = False if self.benchmark_req_queues_size > 0 and self.kv_cache_transceiver else True while True: profile_step() if self.enable_iter_perf_stats: @@ -1162,6 +1155,32 @@ def _executor_loop_overlap(self): scheduled_batch, iter_stats = self._prepare_and_schedule_batch() if scheduled_batch is None: break + if not self.is_warmup and not can_forward: + if self.enable_attention_dp: + local_can_forward = self.executor_request_queue.num_fetch_requests + \ + len(scheduled_batch.generation_requests) >= self.benchmark_req_queues_size + all_can_forward = self.dist.allgather(local_can_forward) + if all(all_can_forward): + time.sleep(30) + can_forward = True + else: + if self.dist.rank == 0: + logger.info( + f"sleep 10 seconds, num_fetched_requests: {self.executor_request_queue.num_fetch_requests}, scheduled_gen_batch: {len(scheduled_batch.generation_requests)}" + ) + time.sleep(10) + continue + else: + if len(scheduled_batch.generation_requests + ) < self.benchmark_req_queues_size: + if self.dist.rank == 0: + logger.info( + f"sleep 10 seconds, num_fetched_requests: {len(scheduled_batch.generation_requests)}, scheduled_gen_batch: {len(scheduled_batch.generation_requests)}" + ) + time.sleep(10) + continue + else: + can_forward = True self._pause_requests(scheduled_batch.paused_requests) From a0e58b575bbd9ec7e9c6a706b39be5bb23354bb0 Mon Sep 17 00:00:00 2001 From: Xianjie <5410381+qiaoxj07@users.noreply.github.com> Date: Mon, 22 Sep 2025 19:45:54 -0700 Subject: [PATCH 2/4] update Signed-off-by: Xianjie <5410381+qiaoxj07@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index de97b961351e..1371728846af 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -1159,9 +1159,10 @@ def _executor_loop_overlap(self): if self.enable_attention_dp: local_can_forward = self.executor_request_queue.num_fetch_requests + \ len(scheduled_batch.generation_requests) >= self.benchmark_req_queues_size - all_can_forward = self.dist.allgather(local_can_forward) + all_can_forward = self.dist.tp_allgather( + local_can_forward) if all(all_can_forward): - time.sleep(30) + time.sleep(10) can_forward = True else: if self.dist.rank == 0: From 7006c71f74ceb503fb445dd9767a382b04e6403a Mon Sep 17 00:00:00 2001 From: Xianjie <5410381+qiaoxj07@users.noreply.github.com> Date: Mon, 22 Sep 2025 19:55:00 -0700 Subject: [PATCH 3/4] update Signed-off-by: Xianjie <5410381+qiaoxj07@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 1371728846af..696b124f68a6 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -1162,8 +1162,8 @@ def _executor_loop_overlap(self): all_can_forward = self.dist.tp_allgather( local_can_forward) if all(all_can_forward): - time.sleep(10) can_forward = True + time.sleep(10) else: if self.dist.rank == 0: logger.info( From abb209cd50748a0ce34e3b515ce26690b353ad8f Mon Sep 17 00:00:00 2001 From: Xianjie <5410381+qiaoxj07@users.noreply.github.com> Date: Thu, 25 Sep 2025 02:55:22 -0700 Subject: [PATCH 4/4] update Signed-off-by: Xianjie <5410381+qiaoxj07@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 696b124f68a6..d0243177f9ed 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -1155,6 +1155,9 @@ def _executor_loop_overlap(self): scheduled_batch, iter_stats = self._prepare_and_schedule_batch() if scheduled_batch is None: break + # In gen-only benchmarking mode, wait until the number of scheduled generation + # requests reaches the required threshold before starting forward pass, + # to ensure consistent batch sizes for accurate performance measurement. if not self.is_warmup and not can_forward: if self.enable_attention_dp: local_can_forward = self.executor_request_queue.num_fetch_requests + \ @@ -1176,7 +1179,7 @@ def _executor_loop_overlap(self): ) < self.benchmark_req_queues_size: if self.dist.rank == 0: logger.info( - f"sleep 10 seconds, num_fetched_requests: {len(scheduled_batch.generation_requests)}, scheduled_gen_batch: {len(scheduled_batch.generation_requests)}" + f"sleep 10 seconds, scheduled_gen_batch: {len(scheduled_batch.generation_requests)}" ) time.sleep(10) continue