Skip to content

[TRTLLM-4926][feat] Reimplement metrics endpoint with stats about requests #5560

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pathorn wants to merge 2 commits into from
Closed

[TRTLLM-4926][feat] Reimplement metrics endpoint with stats about requests #5560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ae55aa
Use decorator for request cancelation and handle CancelledError
pathorn Jun 27, 2025
f78d0c1
Reimplement metrics endpoint with stats about requests
pathorn Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion tensorrt_llm/_torch/pyexecutor/py_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,15 @@
LlmResponse, executor_request_to_llm_request)
from .model_engine import ModelEngine
from .sampler import Sampler, SampleState, SampleStateTensors, TorchSampler
from .scheduler import RequestScheduler, ScheduledRequests
from .scheduler import ScheduledRequests
from collections import defaultdict
import array
import json

PROM_METRICS_FILENAME = '/dev/shm/prom_metrics.json'

prom_metrics = defaultdict(float)
prom_metrics_file = None

# Environment variable to specify iteration ranges for profiling start/stop.
# Format: "start1-stop1,start2-stop2,..." or single iterations "iter1,iter2,..."
Expand Down Expand Up @@ -487,6 +495,7 @@ def profile_step():
f"trace saved to {torch_trace_path}")
torch.cuda.cudart().cudaProfilerStop()
enabled = False
last_start_time = start_time

if start_time is not None and self.print_log and self.dist.rank == 0:
end_time = time.time()
Expand Down Expand Up @@ -514,6 +523,33 @@ def profile_step():
enabled = True
start_time = time.time()

if last_start_time is not None and self.dist.rank == 0:
iter_states = self.model_engine.iter_states
total_running = iter_states['num_ctx_requests'] + iter_states['num_generation_tokens'] / (1 + self.model_engine.max_draft_len)
prom_metrics["num_requests_running"] = total_running
prom_metrics["num_requests_swapped"] = total_running - len(self.active_requests)
prom_metrics["iteration_tokens_total_sum"] += iter_states['num_ctx_tokens'] + iter_states['num_generation_tokens']
prom_metrics["iteration_tokens_total_count"] += 1
prom_metrics["time_per_output_token_seconds_sum"] += (start_time - last_start_time)
prom_metrics["time_per_output_token_seconds_count"] += 1
prom_metrics["prompt_tokens_total"] += iter_states['num_ctx_tokens']
prom_metrics["request_prompt_tokens_total_sum"] += iter_states['num_ctx_tokens']
prom_metrics["request_prompt_tokens_total_count"] += 1
prom_metrics["generation_tokens_total"] += iter_states['num_generation_tokens']
prom_metrics["request_generation_tokens_total_sum"] += iter_states['num_generation_tokens']
prom_metrics["request_generation_tokens_total_count"] += 1
global prom_metrics_file
try:
if prom_metrics_file is None:
prom_metrics_file = os.open(PROM_METRICS_FILENAME,
os.O_RDWR|os.O_CREAT)
os.pwrite(prom_metrics_file, (
json.dumps(list(prom_metrics.keys())).encode('UTF-8') +
b'\0' +
array.array('d',prom_metrics.values()).tobytes()), 0)
except:
traceback.print_exc()

try:
yield profile_step
finally:
Expand Down
Loading