diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 6689bac1e27c..eba2403797fe 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -182,6 +182,10 @@ def launch_server(host: str, server_role=server_role, metadata_server_cfg=metadata_server_cfg) + # Optionally disable GC (default: not disabled) + if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": + gc.disable() + asyncio.run(server(host, port)) @@ -518,7 +522,7 @@ def disaggregated(config_file: Optional[str], # increment, and observed that `count0` (obtained by `gc.get_count()`) # increases by fewer than 1,000 after every 200,000 requests, while the # maximum value of `count0` exceeded 3,000,000 during the test. - if int(os.getenv("TRTLLM_DISAGG_SERVER_DISABLE_GC", "1")): + if os.getenv("TRTLLM_DISAGG_SERVER_DISABLE_GC", "1") == "1": gc.disable() asyncio.run(server(disagg_cfg.hostname, disagg_cfg.port)) diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index f653587d20f7..148cdcf038c4 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -1,3 +1,4 @@ +import gc import json import os import time @@ -413,6 +414,10 @@ def notify_proxy_threads_to_quit(): logger.error("Failed to deliver error message to proxy") return + # Optionally disable GC (default: not disabled) + if os.getenv("TRTLLM_WORKER_DISABLE_GC", "0") == "1": + gc.disable() + with worker: try: worker.block_subordinates()