From 9118b8e473d1b36a22600a439eff5b8c47274b09 Mon Sep 17 00:00:00 2001 From: Tailing Yuan Date: Thu, 25 Sep 2025 10:49:35 +0000 Subject: [PATCH 1/2] Optionally disable server GC and worker GC Signed-off-by: Tailing Yuan --- tensorrt_llm/commands/serve.py | 4 ++++ tensorrt_llm/executor/worker.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 6689bac1e27c..7d5b5ff52a60 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -182,6 +182,10 @@ def launch_server(host: str, server_role=server_role, metadata_server_cfg=metadata_server_cfg) + # Optionally disable GC (default: not disabled) + if int(os.getenv("TRTLLM_SERVER_DISABLE_GC", "0")): + gc.disable() + asyncio.run(server(host, port)) diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index f653587d20f7..fca346475ed8 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -1,3 +1,4 @@ +import gc import json import os import time @@ -413,6 +414,10 @@ def notify_proxy_threads_to_quit(): logger.error("Failed to deliver error message to proxy") return + # Optionally disable GC (default: not disabled) + if int(os.getenv("TRTLLM_WORKER_DISABLE_GC", "0")): + gc.disable() + with worker: try: worker.block_subordinates() From 203cefbf1cc3aa60abff5c84cdea7ab6dbf93666 Mon Sep 17 00:00:00 2001 From: Tailing Yuan Date: Fri, 26 Sep 2025 02:34:57 +0000 Subject: [PATCH 2/2] Cancel int conversion Signed-off-by: Tailing Yuan --- tensorrt_llm/commands/serve.py | 4 ++-- tensorrt_llm/executor/worker.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 7d5b5ff52a60..eba2403797fe 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -183,7 +183,7 @@ def launch_server(host: str, metadata_server_cfg=metadata_server_cfg) # Optionally disable GC (default: not disabled) - if int(os.getenv("TRTLLM_SERVER_DISABLE_GC", "0")): + if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": gc.disable() asyncio.run(server(host, port)) @@ -522,7 +522,7 @@ def disaggregated(config_file: Optional[str], # increment, and observed that `count0` (obtained by `gc.get_count()`) # increases by fewer than 1,000 after every 200,000 requests, while the # maximum value of `count0` exceeded 3,000,000 during the test. - if int(os.getenv("TRTLLM_DISAGG_SERVER_DISABLE_GC", "1")): + if os.getenv("TRTLLM_DISAGG_SERVER_DISABLE_GC", "1") == "1": gc.disable() asyncio.run(server(disagg_cfg.hostname, disagg_cfg.port)) diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index fca346475ed8..148cdcf038c4 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -415,7 +415,7 @@ def notify_proxy_threads_to_quit(): return # Optionally disable GC (default: not disabled) - if int(os.getenv("TRTLLM_WORKER_DISABLE_GC", "0")): + if os.getenv("TRTLLM_WORKER_DISABLE_GC", "0") == "1": gc.disable() with worker: