From 3454b2ab1e84c253326b38f5ff3df2ca0e0d24d5 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:25:41 +0800 Subject: [PATCH 1/2] immprove gpt-oss stress tests Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/defs/.test_durations | 1591 ++++++++--------- ...ig_ctxtp2_gentp2_gptoss_eagle_triton.yaml} | 4 +- ...fig_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml | 67 + ...g_config_ctxtp2_gentp2_gptoss_triton.yaml} | 4 +- .../defs/disaggregated/test_disaggregated.py | 34 +- .../test_lists/qa/llm_function_core.txt | 1 - .../test_lists/qa/llm_function_rtx6k.txt | 1 - .../test_lists/qa/llm_function_stress.txt | 7 +- .../test_lists/test-db/l0_l40s.yml | 1 - tests/integration/test_lists/waives.txt | 4 - 10 files changed, 803 insertions(+), 911 deletions(-) rename tests/integration/defs/disaggregated/test_configs/{disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml => disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml} (97%) create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml rename tests/integration/defs/disaggregated/test_configs/{disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml => disagg_config_ctxtp2_gentp2_gptoss_triton.yaml} (96%) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 9b2e9851b370..f01220bcc5d7 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -1,889 +1,706 @@ { - "cpp/test_e2e.py::test_model[-eagle-86]": 467.31710943579674, - "cpp/test_e2e.py::test_model[-medusa-86]": 559.4042182117701, - "test_fmha.py::test_fmha": 1843.8030557027087, - "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 371.68835608288646, - "llmapi/test_llm_examples.py::test_llmapi_example_inference": 390.4545612037182, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 28.57412961218506, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 71.92784738913178, - "llmapi/test_llm_examples.py::test_llmapi_example_logits_processor": 28.32273621391505, - "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 46.27545684855431, - "llmapi/test_llm_examples.py::test_llmapi_quickstart": 29.23038684949279, - "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mha]]": 16.939650654792786, - "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mla]]": 16.761926997452974, - "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[NIXL-mha-ctx_fp16_gen_fp16]]": 12.963495340198278, - "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[UCX-mha-ctx_fp16_gen_fp16]]": 12.825301811099052, - "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 75.91953653842211, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 36.62238800525665, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 47.69775189831853, - "llmapi/test_llm_api_connector.py::test_connector_async_onboard[False]": 20.00915778055787, - "llmapi/test_llm_api_connector.py::test_connector_async_save[False]": 13.049194630235434, - "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[False]": 27.849122423678637, - "llmapi/test_llm_api_connector.py::test_connector_e2e_persistent_cache": 24.956512186676264, - "llmapi/test_llm_api_connector.py::test_connector_priorities": 15.430590681731701, - "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[attention_dp]": 5.342560388147831, - "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[host_offloading]": 12.192664381116629, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[True]": 17.182394333183765, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[True]": 11.133101604878902, - "llmapi/test_llm_api_connector.py::test_connector_simple[True]": 12.597462255507708, - "test_e2e.py::test_openai_completions_example[pytorch]": 256.56871559098363, - "test_e2e.py::test_openai_mmencoder_example": 128.8613565787673, - "test_e2e.py::test_openai_prometheus": 165.67273128777742, - "test_e2e.py::test_openai_responses_entrypoint": 100.25372431054711, - "test_e2e.py::test_trtllm_bench_invalid_token_pytorch[TinyLlama-1.1B-Chat-v1.0-TinyLlama-1.1B-Chat-v1.0]": 70.06262697279453, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 255.5959302908741, - "test_e2e.py::test_trtllm_serve_multimodal_example": 185.39177925512195, - "thirdparty/test_git_modules.py::test_gitmodules": 0.0012150108814239502, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_async_transfer_manager.py]": 24.65272542834282, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_kv_cache_budget_split.py]": 24.599674943834543, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_scheduler_serializable_output.py]": 23.298357471823692, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_mistral.py]": 31.256194911897182, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_pixtral.py]": 30.9562314376235, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_nemotron_nano_preprocessing.py]": 56.75650619342923, - "test_unittests.py::test_unittests_v2[unittest/_torch/models/checkpoints/hf/test_weight_loader.py]": 23.097423560917377, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_torch_sampler.py]": 261.76642061397433, - "test_unittests.py::test_unittests_v2[unittest/_torch/test_model_config.py]": 22.64773726090789, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_block.py]": 19.638685578946024, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_cluster_storage.py]": 41.06863783299923, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_openai_client.py]": 26.17624157294631, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_extractor.py]": 19.388083309866488, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_messenger.py]": 21.500893251039088, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_openai_disagg_service.py]": 27.999925918877125, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_remoteDictionary.py]": 49.35567116411403, - "test_unittests.py::test_unittests_v2[unittest/executor/test_fatal_error_health_check.py]": 25.49836080148816, - "test_unittests.py::test_unittests_v2[unittest/executor/test_rpc.py]": 497.4198516495526, - "test_unittests.py::test_unittests_v2[unittest/inputs/test_content_format.py]": 23.498350095003843, - "test_unittests.py::test_unittests_v2[unittest/inputs/test_url_validation.py]": 24.147701989859343, - "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_harmony_channel_validation.py]": 36.30120064690709, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_additional_model_outputs.py -m \"gpu1\"]": 48.403994899243116, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_kv_cache_dtype_override.py]": 24.848896216601133, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryArchitectureExtraction]": 57.554038252681494, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_utils.py]": 87.86072044074535, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_reasoning_parser.py]": 23.850911132991314, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_serialization.py]": 23.949409436434507, - "test_unittests.py::test_unittests_v2[unittest/media/test_encoding.py]": 23.400118369609118, - "test_unittests.py::test_unittests_v2[unittest/others/test_lora_manager.py]": 33.45008610934019, - "test_unittests.py::test_unittests_v2[unittest/others/test_time_breakdown.py]": 25.352738097310066, - "test_unittests.py::test_unittests_v2[unittest/scripts]": 29.149033349007368, - "test_unittests.py::test_unittests_v2[unittest/usage/test_config.py]": 19.241729954257607, - "test_unittests.py::test_unittests_v2[unittest/usage/test_opt_out.py]": 19.389309899881482, - "test_unittests.py::test_unittests_v2[unittest/usage/test_schema.py]": 18.941728390287608, - "test_unittests.py::test_unittests_v2[unittest/utils/test_logger.py]": 23.649205207824707, - "test_unittests.py::test_unittests_v2[unittest/visual_gen/test_output.py]": 24.097700644284487, - "disaggregated/test_disaggregated.py::test_disaggregated_benchmark_gen_only_insufficient_kv[TinyLlama-1.1B-Chat-v1.0]": 83.51802018284798, - "disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0]": 99.42717270739377, - "disaggregated/test_disaggregated.py::test_disaggregated_ngram[TinyLlama-1.1B-Chat-v1.0]": 91.52266521565616, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 20.080937860999256, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 60.11040577664971, - "llmapi/test_llm_api_connector.py::test_connector_async_onboard[True]": 17.928842270746827, - "llmapi/test_llm_api_connector.py::test_connector_async_save[True]": 12.977049142122269, - "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[True]": 25.513779915869236, - "llmapi/test_llm_api_connector.py::test_connector_multi_request": 9.052204478532076, - "llmapi/test_llm_api_connector.py::test_connector_priorities_default": 12.085481505841017, - "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[beam_search]": 7.714497704058886, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[False]": 16.96005078405142, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[False]": 9.707948675379157, - "llmapi/test_llm_api_connector.py::test_connector_simple[False]": 13.015593154355884, - "test_e2e.py::test_get_ci_container_port": 0.0005284938961267471, - "test_e2e.py::test_openai_chat_example[pytorch]": 561.0242523606867, - "test_e2e.py::test_openai_misc_example[pytorch]": 274.1224825847894, - "test_e2e.py::test_openai_perf_metrics": 79.67633541114628, - "test_e2e.py::test_openai_reasoning[pytorch]": 240.12692171894014, - "test_e2e.py::test_openai_tool_call": 88.49165302142501, - "test_e2e.py::test_trtllm_serve_top_logprobs[pytorch]": 87.6148243919015, - "thirdparty/test_cmake_third_party.py::test_cmake_listfiles": 0.10550227761268616, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_disagg_index_mapper_early_release.py]": 25.352552112191916, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_kv_cache_estimation.py]": 25.74726054444909, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_cohere2.py]": 48.46714654006064, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_parakeet.py]": 42.047723326832056, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_radio.py]": 25.22459696419537, - "test_unittests.py::test_unittests_v2[unittest/_torch/models/checkpoints/hf/test_checkpoint_loader.py]": 56.12043555267155, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_torch_multi_arange.py]": 41.8165452927351, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_trtllm_sampler.py]": 173.59781158529222, - "test_unittests.py::test_unittests_v2[unittest/_torch/visual_gen/test_visual_gen_params.py]": 24.5291297249496, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_agent_multi_backends.py]": 55.291444917209446, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_cluster_manager_worker.py]": 50.3197379950434, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_utils.py]": 18.99160304525867, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_mamba_transfer.py]": 32.886980537325144, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_openai_disagg_server.py]": 24.754129583016038, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_peer.py]": 18.989873898215592, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_router.py]": 22.49254883499816, - "test_unittests.py::test_unittests_v2[unittest/executor/test_ipc.py]": 31.610522534698248, - "test_unittests.py::test_unittests_v2[unittest/inputs/test_chat_template_dispatch.py]": 25.169238595291972, - "test_unittests.py::test_unittests_v2[unittest/inputs/test_multimodal.py]": 25.204775847494602, - "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_chat_utils.py]": 24.531344229355454, - "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_tool_parsers.py]": 28.049183627590537, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_gc_utils.py]": 31.62360288016498, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_args.py]": 166.04291549883783, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryPyTorchBackend]": 59.91823088936508, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_request_priority.py]": 25.325514804571867, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_utils.py]": 26.306996766477823, - "test_unittests.py::test_unittests_v2[unittest/others/test_convert_utils.py]": 23.858180426061153, - "test_unittests.py::test_unittests_v2[unittest/others/test_lora_module_count.py]": 25.352086970582604, - "test_unittests.py::test_unittests_v2[unittest/others/test_tracing.py]": 134.26669367216527, - "test_unittests.py::test_unittests_v2[unittest/usage/test_collectors.py]": 21.589319362770766, - "test_unittests.py::test_unittests_v2[unittest/usage/test_e2e_capture.py]": 21.25535298185423, - "test_unittests.py::test_unittests_v2[unittest/usage/test_reporter.py]": 21.342137079220265, - "test_unittests.py::test_unittests_v2[unittest/usage/test_transport.py]": 23.60382394399494, - "test_unittests.py::test_unittests_v2[unittest/utils/test_util.py]": 26.621632516384125, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 85.27059441700112, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_cpp_runtime]": 193.6268605251098, - "test_unittests.py::test_unittests_v2[unittest/api_stability]": 25.643251301022246, - "test_unittests.py::test_unittests_v2[unittest/bindings]": 714.101646464318, - "test_unittests.py::test_unittests_v2[unittest/dynamo]": 26.950018653995357, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryTRTBackend]": 166.76355233497452, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command_with_lora[llama-llama-models-v2/llama-v2-7b-hf]": 185.91652243910357, - "llmapi/test_llm_examples.py::test_llmapi_chat_example": 78.83368532638997, - "llmapi/test_llm_examples.py::test_llmapi_server_example": 83.81749124731869, - "test_e2e.py::test_openai_misc_example[trt]": 193.8965124920942, - "test_e2e.py::test_trtllm_bench_sanity[--non-streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 352.73563918611035, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition0\"]": 73.73264672234654, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_no_cache.py]": 54.35324762901291, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_llama.py]": 1274.6575344190933, - "test_e2e.py::test_openai_reasoning[trt]": 112.70626465464011, - "test_e2e.py::test_trtllm_serve_example": 209.98838954232633, - "test_e2e.py::test_trtllm_serve_top_logprobs[trt]": 140.92444621305913, - "test_unittests.py::test_unittests_v2[unittest/test_model_runner_cpp.py]": 476.4086992479861, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition1\"]": 73.379359818995, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition2\"]": 72.02512616757303, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_IFB.py]": 126.87880356237292, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mistral.py]": 328.5478213331662, - "llmapi/test_llm_e2e.py::test_llmapi_exit": 18.432179793715477, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-llama-models/llama-7b-hf]": 232.97426022961736, - "llmapi/test_llm_examples.py::test_llmapi_kv_cache_connector[Qwen2-0.5B]": 82.59823241084814, - "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 114.59312928467989, - "test_e2e.py::test_openai_health": 60.943585466593504, - "test_e2e.py::test_trtllm_bench_latency_sanity[FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 246.85433532483876, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_build_cache.py]": 23.103414099663496, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition3\"]": 68.93004452623427, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"xqa_generic\"]": 801.098383044824, - "test_unittests.py::test_unittests_v2[unittest/trt/functional]": 196.5955226458609, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization]": 487.10545333847404, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[gemma-3-1b]": 17.166659578680992, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[phi-4-mini]": 14.305965591222048, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[qwen2-7b]": 13.09041291847825, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[qwen3-0.6b]": 5.315278757363558, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[starcoder2-3b]": 17.601711992174387, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[tinyllama-1.1b]": 6.3743207305669785, - "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encode_matches_huggingface_classification[bert-yelp]": 3.595465302467346, - "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encode_matches_huggingface_per_token_reward[qwen2.5-prm-7b]": 11.411922980099916, - "test_e2e.py::test_openai_lora": 173.81692308560014, - "test_e2e.py::test_trtllm_serve_lora_example": 98.36129532754421, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_encode.py]": 58.21390492469072, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part0\"]": 598.7036472780164, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part1\"]": 126.27206432109233, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part2\"]": 484.83895123400725, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part3\"]": 208.49013988906518, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_memory_profiling.py::test_pyexecutor_and_kvcache_share_execution_stream]": 40.03293885290623, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_quantization.py]": 23.93977590277791, - "triton_server/test_triton.py::test_medusa[medusa]": 569.5001850500703, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/compile]": 37.790911596268415, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/custom_ops]": 623.6737900990993, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/models]": 115.3445010241121, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/shim]": 22.633222276344895, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/smoke]": 670.1154091879725, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/transformations]": 300.3593855574727, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/utils]": 24.93424261920154, - "cpp/test_e2e.py::test_model[-gpt_tests-80]": 615.122148225084, - "cpp/test_unit_tests.py::test_unit_tests[common-80]": 12.664204696193337, - "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 687.9856592249125, - "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 1.2464399840682745, - "cpp/test_unit_tests.py::test_unit_tests[utils-80]": 1.8540223143063486, - "cpp/test_e2e.py::test_model[-gpt_executor-80]": 814.0748630631715, - "cpp/test_unit_tests.py::test_unit_tests[batch_manager-80]": 292.39521062839776, - "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 278.46642002556473, - "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 856.879348281771, - "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 548.5782862240449, - "test_e2e.py::test_openai_chat_with_logit_bias[torch_sampler]": 73.78876239433885, - "test_e2e.py::test_openai_chat_with_logit_bias[trtllm_sampler]": 73.2108123600483, - "test_e2e.py::test_openai_completions_with_logit_bias[trtllm_sampler]": 72.54250191617757, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen\"]": 754.3435163404793, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_beam_search.py]": 789.6666672006249, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_logits_logprobs.py]": 392.0245815804228, - "test_e2e.py::test_openai_completions_with_logit_bias[torch_sampler]": 75.10106470668688, - "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 28.074005853384733, - "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 24.315578892827034, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nas\"]": 32.59954630583525, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_out_of_tree\"]": 138.8535058517009, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_phi3\"]": 26.83701508399099, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen_moe\"]": 281.13565809698775, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_beam_search_speculative_d2h.py]": 213.0422897520475, - "triton_server/test_triton.py::test_fill_template[fill-template]": 1.6639938056468964, - "triton_server/test_triton.py::test_llama[llama]": 179.10436917375773, - "triton_server/test_triton.py::test_llmapi_unit_tests[llmapi-unit-tests]": 4.315617096610367, - "triton_server/test_triton.py::test_mistral[mistral]": 175.61003254074603, - "triton_server/test_triton.py::test_mistral_ib_streaming[mistral-ib-streaming]": 342.24799779988825, - "triton_server/test_triton.py::test_python_preproc_unit_tests[python-preproc-unit-tests]": 9.682531874626875, - "triton_server/test_triton_llm.py::test_benchmark_core_model[gptj_6b-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization-4096--1-1-1-False]": 425.68877257127315, - "triton_server/test_triton_llm.py::test_gpt_350m_ifb[test_basic-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 103.82896339520812, - "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[e2e]": 106.43146257847548, - "triton_server/test_triton_llm.py::test_gpt_disaggregated_serving_bls[test_basic-False-1-top_k_top_p--False-True-True-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-True-tensorrt_llm_bls]": 98.34344942122698, - "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[False-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 268.7699463283643, - "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-False-ensemble]": 13.90028147213161, - "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[batched_inputs-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 181.1615214087069, - "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[test_stop_words-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 16.108110164292157, - "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-enableDecoupleMode-tensorrt_llm]": 78.61408129800111, - "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-python-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 114.62468159664422, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 122.2498293183744, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 33.313044152222574, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 17.912641955539584, - "triton_server/test_triton_llm.py::test_whisper_large_v3_ifb[True-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-0.5-guaranteed_no_evict--24000-1-1-1-False-ensemble]": 137.97018379811198, - "triton_server/test_triton.py::test_gpt[gpt]": 182.20508953463286, - "triton_server/test_triton.py::test_gptj[gptj]": 223.35692731942981, - "triton_server/test_triton.py::test_python_multimodal_encoders_unit_tests[python-multimodal-encoders-unit-tests]": 28.360558917745948, - "triton_server/test_triton.py::test_whisper[whisper]": 204.72570630162954, - "triton_server/test_triton_llm.py::test_benchmark_core_model[llama_v2_7b-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization-4096--1-1-1-False]": 389.1086704917252, - "triton_server/test_triton_llm.py::test_eagle_vicuna_7b_ifb[False-1-eagle--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 189.3712582429871, - "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[accuracy]": 149.95370694343, - "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 278.40986166801304, - "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[batched_inputs-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-enableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 161.46982649620622, - "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-disableDecoupleMode-tensorrt_llm]": 90.04027471225709, - "triton_server/test_triton_llm.py::test_llmapi_lora[1-tensorrt_llm]": 63.79091303702444, - "triton_server/test_triton_llm.py::test_medusa_vicuna_7b_ifb[False-1-medusa--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 161.17319475393742, - "triton_server/test_triton_llm.py::test_t5_small_enc_dec_ifb[test_basic-False-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap---guaranteed_no_evict--4096-1-1-1-False-tensorrt_llm_bls]": 125.47026043757796, - "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-tensorrtllm-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 119.73666133265942, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 114.50461501348764, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 18.529517172835767, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 18.59724544081837, - "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp": 251.41180250793695, - "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_ngram": 98.84480216354132, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False]": 228.78805011400254, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True]": 189.88735772500513, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True]": 294.4758595749736, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 246.82779098799801, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 119.71112632699078, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 222.06951643901994, - "test_unittests.py::test_unittests_v2[unittest/_torch/attention]": 307.9887097366154, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor]": 557.2529671951197, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/mamba]": 221.1838954249397, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"]": 151.7901495929982, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"]": 340.960238839034, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"]": 47.828515230008634, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]]": 48.728463740990264, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]]": 24.82597585298936, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 21.67635251401225, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 25.978178004006622, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]]": 20.424516523984494, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 19.677430056006415, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 24.725896997988457, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 23.124266630999045, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 20.727860749000683, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]]": 42.2266049849859, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_awq_quantization.py]": 19.188348875846714, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_activation_quant.py]": 20.014674406964332, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_add_rms_norm_quant.py]": 19.493550593033433, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_moe.py]": 19.816725137876347, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_group_rmn_norm.py]": 23.089757694862783, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_load_balancer.py]": 19.26597039285116, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_routing.py]": 28.89103313582018, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_rotary_embedding.py]": 23.866294936975464, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_triton_linear.py]": 23.79145319806412, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/tests_lora_modules]": 659.0716862599365, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel]": 4351.49280942278, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/serial]": 58.17412027902901, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 217.26752274786122, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 222.7607175433077, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 302.9856714049997, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 223.62056540699996, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 200.86565831400003, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 189.0692499190004, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 1031.1528510910002, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 461.9252251499993, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=False]": 267.6105636049997, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]]": 71.63301057300032, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]]": 48.08178372900056, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 58.63044721000006, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]]": 85.04990262999945, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 50.014132944000266, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]]": 27.723852323000756, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 20.322831061000215, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 28.123089660000005, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 48.727174462999756, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 50.479853326999546, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 48.27790872099922, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 17.62364664799952, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 27.62391243500042, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 18.274859787999958, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_host_sharer.py]": 20.292041711043566, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python": 163.95312987302896, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 287.43463359299994, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 196.23940314585343, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 216.26651366100032, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 249.59442985600026, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 261.57555265400015, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 207.06534601002932, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 155.4231541629415, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 79.62999053788371, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True]": 419.2233988319995, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_cutlass-torch_compile=True]": 275.25382337999963, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_llama\"]": 119.63326477212831, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_mla_helix.py]": 976.3970404448919, - "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling -k \"deepseek\"]": 354.90453113700005, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 178.26867335097631, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 229.5246064819803, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 172.92849959799787, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 147.37839249300305, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 266.96483929408714, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 206.03644429100677, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 188.69488530093804, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 164.00336496997625, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 172.82999583697529, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 251.55259347101673, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False]": 139.83139117099927, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=False]": 135.59422479500063, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 111.3068211893551, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=False]": 183.35118844200042, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True]": 201.5803480189934, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_gpt_oss\"]": 107.69878626195714, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mixtral\"]": 41.068741515045986, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-one_model-overlap_scheduler]": 2229.254062668013, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-two_model-overlap_scheduler]": 2873.1229173520114, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 1908.5162332250038, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler]": 2359.458676673996, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False]": 222.66332006501034, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 210.72954666399164, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-True]": 235.43180733395275, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-False]": 186.75015839299886, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-True]": 215.9784758510068, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-True]": 225.26497576397378, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-True]": 181.96771040401654, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 79.8115957059781, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 92.91837230202509, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram": 451.23835995496484, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True]": 393.2114061889588, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True-True]": 371.7093208580045, - "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu -m \"not post_merge\"]": 2881.170321949001, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_additional_model_outputs.py -m \"gpu2\"]": 45.69425537396455, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_mpi_session.py::test_llmapi_launch_multiple_tasks]": 88.89764528500382, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync": 275.9294940570253, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 280.02051854197634, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-False]": 226.95165822003037, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-True]": 176.63856029702583, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False]": 213.76671027002158, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-False]": 221.16686225897865, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-False]": 180.42018839297816, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-False]": 198.51033656601794, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 188.66204771603225, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 293.0652321569505, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 179.5286056349869, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 93.58281131199328, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False]": 393.84327019704506, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill": 393.43859653401887, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_pixtral.py::test_tensor_parallelism]": 44.9982833910035, - "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_disagg_serving_perf_metrics.py]": 71.39680112700444, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu2\"]": 272.88288813398685, - "examples/test_ray.py::test_llm_inference_distributed_ray[pp2]": 138.11034928797744, - "examples/test_ray.py::test_llm_inference_distributed_ray[tep2]": 109.29040575103136, - "examples/test_ray.py::test_llm_inference_distributed_ray[tp2]": 72.43276466900716, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu2\"]": 235.5710942610167, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part0\"]": 341.03334485401865, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part1\"]": 341.3838682859787, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part2\"]": 179.14875201904215, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part3\"]": 180.01359257102013, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part4\"]": 215.3152888190234, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"gpu2\"]": 71.13692150497809, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer]": 191.26553119625896, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[Qwen_QwQ-32B-False]": 440.4416093239561, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[google_gemma-3-1b-it-False]": 61.59581129811704, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.1-8B-Instruct-False]": 46.71345825213939, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.3-70B-Instruct-False]": 171.76501162629575, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Codestral-22B-v0.1-False]": 240.30174564756453, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Ministral-8B-Instruct-2410-False]": 75.44280633982271, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True]": 127.60458957124501, - "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm]": 399.2673514550552, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-mpi_kvcache-90]": 566.998665202409, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-nixl_kvcache-90]": 9.834118995815516, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-ucx_kvcache-90]": 10.331790680065751, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-mpi_kvcache-90]": 23.657401267439127, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-nixl_kvcache-90]": 23.906753124669194, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-ucx_kvcache-90]": 23.80666296184063, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-mpi_kvcache-90]": 12.536427475512028, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-nixl_kvcache-90]": 12.840721806511283, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-ucx_kvcache-90]": 12.540847543627024, - "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-mpi_kvcache-90]": 18.39725266955793, - "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-nixl_kvcache-90]": 16.89188546873629, - "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-ucx_kvcache-90]": 16.693953413516283, - "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-nixl_kvcache-90]": 30.972941564396024, - "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-ucx_kvcache-90]": 29.168602090328932, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-mpi_kvcache-90]": 4.774993252009153, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-nixl_kvcache-90]": 4.874061470851302, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-ucx_kvcache-90]": 4.673807211220264, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-mpi_kvcache-90]": 9.332997119054198, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-nixl_kvcache-90]": 9.682227460667491, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-ucx_kvcache-90]": 9.431488024070859, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-mpi_kvcache-90]": 11.735960626974702, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-nixl_kvcache-90]": 11.684558764100075, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-ucx_kvcache-90]": 11.736474914476275, - "cpp/test_multi_gpu.py::test_enc_dec[t5-90]": 6.025917749851942, - "cpp/test_multi_gpu.py::test_llama_executor[llama-leader-90]": 84.81573174893856, - "cpp/test_multi_gpu.py::test_llama_executor[llama-orchestrator-90]": 162.71292873471975, - "cpp/test_multi_gpu.py::test_llama_executor_guided_decoding[llama-90]": 13.140320116654038, - "cpp/test_multi_gpu.py::test_llama_executor_logits_proc[llama-90]": 44.34581693634391, - "cpp/test_multi_gpu.py::test_trt_gpt_real_decoder[llama-90]": 113.51448027789593, - "cpp/test_multi_gpu.py::test_mpi_utils[90]": 45.64661745913327, - "cpp/test_multi_gpu.py::test_user_buffer[2proc-90]": 2.11952923797071, - "cpp/test_multi_gpu.py::test_fused_gemm_allreduce[4proc-90]": 19.49912805482745, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 134.54751868266612, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 154.7853800514713, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 135.73957254644483, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 174.52036147098988, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 152.75700911972672, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 211.04387662932277, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 129.80014774948359, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_transceiver_runtime_python[DeepSeek-V3-Lite-fp8]": 159.481474686414, - "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp4-bf16-trtllm-deepseekv3_lite]]": 102.35796185210347, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 152.4078388279304, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 119.35970550915226, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 130.15719587402418, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 200.01216824213043, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus_static_eplb": 124.69750702101737, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_empty_batch[DeepSeek-V3-Lite-bf16]": 202.9623187012039, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 139.4163328241557, - "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp1-bf16-trtllm-deepseekv3_lite]]": 93.75344497803599, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 1437.00570011104, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 541.5369369090186, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 471.5579449380166, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 866.0648840669892, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 594.6447642149869, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache_no_reuse-tp4-cutlass-auto]": 419.3978291969979, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 376.8179885330028, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2]": 154.61260438500904, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2]": 100.9895223529893, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 161.34927469299873, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 88.99735977302771, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 99.42438653996214, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[noadp-ctx_tp2pp1-gen_tp1pp1]": 151.80676046095323, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 104.44514593598433, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 54.41597771999659, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache]": 374.00853590102633, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache]": 210.51360351400217, - "disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin]": 100.25905621499987, - "disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin]": 98.90331856295234, - "disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin]": 49.15359141398221, - "disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin]": 49.10201428201981, - "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin]": 206.90428009995958, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0]": 64.96553604403744, - "disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python[TinyLlama-1.1B-Chat-v1.0]": 85.28914290998364, - "test_e2e.py::test_ptp_quickstart_advanced_deepseek_v3_lite_4gpus_adp_balance[DeepSeek-V3-Lite-FP8-DeepSeek-V3-Lite/fp8]": 111.94489605998388, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and FP8 and not MXFP8\"]": 1379.5952013240312, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_AWQ\"]": 669.7905237200321, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb]": 109.54814359301236, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu4\"]": 234.50615247199312, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2]": 167.242011513561, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2]": 98.52402048092335, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2]": 199.50387456268072, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 146.5718833785504, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2]": 94.77812515385449, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[adp-ctx_tp2pp1-gen_tp2pp1]": 156.16143955662847, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 80.73761726729572, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 76.1688071032986, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 74.1406664410606, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache]": 372.94147814717144, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache]": 213.26513709686697, - "disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin]": 139.31395231373608, - "disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin]": 98.91171756386757, - "disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware]": 49.066187175922096, - "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing]": 213.4513011770323, - "disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing]": 207.4394314819947, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0]": 70.3319569742307, - "test_e2e.py::test_ptp_quickstart_advanced_bs1": 112.39259800966829, - "test_e2e.py::test_trtllm_bench_llmapi_launch[pytorch_backend-llama-v3-llama3-8b]": 128.85917745530605, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A16_MXFP4\"]": 1016.8723992183805, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W8A16\"]": 691.5421204390004, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_py_cache_transceiver_mp.py]": 2267.29654201027, - "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp1_4instances]": 112.71113630395848, - "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp2_2instances]": 116.105620782997, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu4\"]": 469.73704576998716, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp1_4instances]]": 290.26655738794943, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp2_2instances]]": 318.72108415700495, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"gpu4\"]": 77.99721742799738, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 132.69184266900993, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 158.83748530130833, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse": 153.4440853409469, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse": 82.38394147483632, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler": 71.19869335694239, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=False-overlap_scheduler=False]": 1218.607248812914, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 80.05104345735162, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 107.66202485794201, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format": 62.41016404423863, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash": 621.2223827303387, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8": 152.73669020738453, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 110.01207454688847, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[False-TinyLlama-1.1B-Chat-v1.0]": 35.032475878018886, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 101.00177443493158, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 83.89112774282694, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 63.88218005700037, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[no-overlap-chunked]": 168.74234423786402, - "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 61.069874382112175, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 250.20505770668387, - "test_unittests.py::test_unittests_v2[unittest/_torch/compilation]": 25.54568713111803, - "test_unittests.py::test_unittests_v2[unittest/_torch/debugger]": 19.291544676292688, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal]": 1385.4457194129936, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_aux.py]": 18.838812944013625, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_request_id.py]": 19.28899485990405, - "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mla-ctx_fp16_gen_fp16]]": 31.09049210883677, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 129.92204322700854, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True]": 235.29396068817005, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 190.64960246894043, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=2]": 127.35095311468467, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 220.07175700203516, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler": 75.0205802610144, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse": 72.13469798583537, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format": 34.00468918774277, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_fp8": 300.7987077347934, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 309.6223061759956, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 247.2587843281217, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[True-TinyLlama-1.1B-Chat-v1.0]": 25.400752289104275, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 85.75992965977639, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 83.36212210077792, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 53.837384417653084, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[max-util-chunked]": 158.87441884400323, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[python-scheduler]": 172.01433536224067, - "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 148.76338731311262, - "test_e2e.py::test_openai_responses": 352.3726217681542, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 92.3254858199507, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 99.50422196800355, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_gemma3\"]": 243.37643587309867, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTLASS\"]": 228.12391290487722, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel_hw_agnostic]": 585.0550106270239, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_region.py]": 20.752864133100957, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disaggregated_params.py]": 21.85614148201421, - "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mha-ctx_fp16_gen_fp16]]": 33.907329918816686, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True]": 233.35579176666215, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format": 96.40042386110872, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=0]": 120.64296809816733, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=fp8-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True]": 131.03063145186752, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse": 82.81574466405436, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse": 72.67876957170665, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=True-eagle3_one_model=True-overlap_scheduler=True]": 1075.0943210227415, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 48.7322347429581, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True]": 170.77184127038345, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16": 511.3651969926432, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 126.62458716612309, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=True-enable_draft_len_schedule=False]": 109.76810259604827, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 113.73702853103168, - "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8]": 848.9961165660061, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 148.54536304995418, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_cancel_gen_requests[TinyLlama-1.1B-Chat-v1.0]": 36.3188966489397, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[False-TinyLlama-1.1B-Chat-v1.0]": 28.20248524507042, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-True-DeepSeek-V3-Lite-fp8/fp8]": 127.0615609427914, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 45.532342946040444, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[guaranteed-chunked]": 155.79833736736327, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-no-chunked]": 161.94301183428615, - "test_e2e.py::test_openai_chat_harmony": 256.37394262477756, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 94.71067717205733, - "test_e2e.py::test_trtllm_benchmark_serving[llama-3.1-model/Meta-Llama-3.1-8B]": 125.41995501983911, - "test_unittests.py::test_unittests_v2[unittest/_torch/lora]": 20.80229289084673, - "test_unittests.py::test_unittests_v2[unittest/_torch/misc]": 344.32345556328073, - "test_unittests.py::test_unittests_v2[unittest/_torch/speculative/test_eagle3.py]": 1019.0683842310682, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_page.py]": 20.143666464835405, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_kv_transfer_mp.py]": 123.84979211073369, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_rank_info.py]": 18.993583410046995, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py]": 550.8318111081608, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse": 162.39044524217024, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse": 82.52972493879497, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available": 81.73615213809535, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=TRTLLM]": 149.4546632859856, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=True-overlap_scheduler=True]": 1072.4692367021926, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 77.7370796087198, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 100.55706254392862, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False]": 120.40908256312832, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dflash": 184.22292581619695, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dummy_load_format": 59.57457195688039, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=True]": 110.55068488512188, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8]": 111.89929933100939, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0]": 23.80086028506048, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 70.33974885300267, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[True-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 41.7932134908624, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-chunked]": 163.14287832612172, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix_smoke": 130.64739427994937, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 70.83530108863488, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_sanity]": 253.82326687918976, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler]": 1191.2096670051105, - "test_unittests.py::test_unittests_v2[unittest/_torch/speculative/hw_agnostic]": 1831.2627570661716, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_cache_transceiver_single_process.py]": 3324.7812275849283, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_kv_transfer.py]": 928.6214334205724, - "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_perf_logger.py]": 21.391628997866064, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 152.97960573100136, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 153.88833729398903, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 121.66436125000473, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 134.81132309199893, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 171.62797562000924, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 127.72052242897917, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 154.6956266699999, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 129.47093768097693, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 134.80305856201448, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 156.33859626899357, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 127.20496735500637, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 154.19827068597078, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 116.73455706000095, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 125.1877486439771, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 147.13845685200067, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 123.54514406202361, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 127.23275308904704, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 81.83421968104085, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False]": 279.30084877600893, - "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype": 957.9583136999863, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 89.2490907079773, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 135.65254550502868, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 89.36297566298163, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=FLASHINFER]": 243.73127793997992, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 75.11799066798994, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 93.10347031999845, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 45.586120243999176, - "accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill]": 215.60936970298644, - "accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill]": 344.82283938402543, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 137.0733454680012, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 143.5359488390095, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 116.0004205229925, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 123.336180874001, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 166.7623178389913, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 118.65510969999013, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 145.0461449699942, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 124.612122652994, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 122.85179915899062, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 140.71412417298416, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 114.79413927401765, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 131.09561426599976, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 119.48417889501434, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 176.2980228930246, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 122.85712360299658, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 121.1533630049671, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 121.68564532196615, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 124.12095914199017, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 141.94041900796583, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 129.43495701398933, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True]": 317.6746448139893, - "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 447.11634525697445, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 127.43863378401147, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=FLASHINFER]": 187.84411286801333, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 124.13719056698028, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 111.11976641096408, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 140.97133619396482, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 168.74814063502708, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 233.80381705239415, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 77.1988166347146, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 79.65342007577419, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 69.40490325912833, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 76.35176690667868, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 81.16481507569551, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 72.47008088231087, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 83.34465102478862, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 67.5397269576788, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 89.43130768090487, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 76.1686408482492, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 72.9958230741322, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 78.06165297329426, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 75.27438813075423, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 80.74985931068659, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 86.10687870532274, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 82.69737496972084, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 87.65647116675973, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 82.57926809042692, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 75.8943274691701, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 75.05017046257854, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 122.5022951066494, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 72.29567366093397, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 83.35832703486085, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 77.41014160588384, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 104.96686546504498, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 75.51850606128573, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 85.44486574828625, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 71.4350159689784, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 83.38494031503797, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 153.37503067031503, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 99.90067131072283, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 94.13767189905047, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 104.74171372875571, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 94.56583361327648, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 115.35695255175233, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 103.06190882995725, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 95.00528683885932, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 106.75661451742053, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 115.989338260144, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 99.97725444287062, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 123.66422721371055, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 103.99547399580479, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 103.63337088748813, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 96.55534584447742, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 98.63648730143905, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 95.0242466814816, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 96.21232785284519, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 98.30433483421803, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False]": 153.95823365077376, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=False]": 63.16813847795129, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=True]": 76.7680161446333, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=False]": 63.61583887413144, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=True]": 72.02205647155643, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=True]": 88.9733154848218, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 112.0710693411529, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 60.62598153203726, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 61.945570688694715, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 91.45242443680763, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 62.574849516153336, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=False]": 992.8535194210708, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True]": 620.6253603063524, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False]": 808.6647146046162, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=True]": 600.0486651174724, - "test_e2e.py::test_trtllm_bench_mgmn": 91.79827606678009, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_235b_a22b_fp8_hopper-qwen3_235b_a22b_fp8_tp4_ep4_cutlass_8k1k]": 910.4367664121091, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_32b_fp8_hopper-qwen3_32b_fp8_tp2_6k1k]": 1242.9062750637531, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 292.60953721031547, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 175.63857275992632, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 186.52553553506732, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 161.51636945083737, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4]": 190.38547903299332, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4]": 131.49605344608426, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4]": 104.33410734310746, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4]": 99.25327169150114, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2]": 152.5729262419045, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2]": 89.9921805858612, - "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[False]": 661.3829264938831, - "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[True]": 322.62455509230494, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[latency]": 2162.4878128543496, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_off]": 231.63562587695196, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0]": 89.56677887961268, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0]": 67.95142366364598, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora]": 304.1238304525614, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 16.223822474479675, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.float16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 16.374391281977296, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency]": 246.29444654472172, - "test_unittests.py::test_unittests_v2[unittest/_torch/attention/test_attention_mla.py]": 231.48642779141665, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_kv_cache_aware_routing[mtp_nextn=0]": 163.56748677929863, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=CUTLASS]": 252.98956363508478, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP]": 251.82381451921538, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=2]": 211.0549951121211, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=2]": 225.89990004291758, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 157.48752847802825, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 204.017874710029, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 284.236327491235, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 166.91178651992232, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 174.30949333589524, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=TRTLLM]": 228.28387101413682, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-overlap_scheduler]": 439.3828970207833, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-two_model-overlap_scheduler]": 385.81541442498565, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 548.5454997923225, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus_online_eplb[fp8]": 396.00846621999517, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False]": 688.351709254086, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 691.9655529111624, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP]": 410.2264897967689, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTLASS]": 369.08950016694143, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[no_cuda_graph_overlap-cutlass]": 343.0884657520801, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep1-cutlass]": 422.295906488318, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_off-trtllm]": 407.53346419893205, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_boundary]": 353.82153057679534, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 558.5696469410323, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_8k1k]": 813.6045300079277, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k]": 560.361906999955, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k]": 468.6927351569757, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 295.37768525700085, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 391.1908592169639, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 417.4842175709782, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_8k1k]": 733.6566050739493, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tep2_1k8k]": 787.2090990659781, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 511.71572106005624, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 2908.4839029698633, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 333.19444385915995, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k]": 452.1892409371212, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 411.64431313984096, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 337.6239333390258, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k]": 945.6405130829662, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k]": 708.4553801259026, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 473.57998897507787, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 178.40271015278995, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k8k]": 2171.321392316837, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_1k1k]": 233.66355675505474, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k]": 405.2331366450526, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 546.0889781760052, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 168.79803335713223, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_8k1k]": 266.5968868341297, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k]": 203.08553342614323, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 463.4555850980105, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 450.5373125029728, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k8k]": 521.3741318379762, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep2_1k1k]": 439.9891236410476, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp4_eagle3_1k1k]": 229.3437453189399, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2190.739206559956, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL]": 5787.14215046796, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 667.5444930130616, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 456.85187179397326, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 603.7066158190137, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 2627.6349721120205, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 2105.2029822249897, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 238.50409000902437, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 206.6761712988373, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 219.68855385808274, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 144.7110821590759, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 283.6205424251966, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 256.5663748360239, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 85.24103148793802, - "examples/test_visual_gen.py::test_visual_gen_quickstart": 244.85972903110087, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_quant.py]": 341.68943272158504, - "test_unittests.py::test_unittests_v2[unittest/trt/model/eagle]": 35.844667091965675, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt_e2e.py]": 375.8710653409362, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_api_multi_gpu.py]": 16.811496037989855, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_level_api.py]": 16.826461140066385, - "accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False]": 880.4145672637969, - "accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma3n_e2b_it": 845.1492025442421, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton_paged-False-1]": 365.7970229499042, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-1]": 83.0432817041874, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm]": 185.77484482899308, - "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-trtllm]": 528.6534720119089, - "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True]": 659.9385115206242, - "examples/test_ad_guided_decoding.py::test_autodeploy_guided_decoding_main_json": 105.7096260804683, - "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[flashinfer-torch-simple]": 64.08092588745058, - "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[trtllm-torch-cudagraph]": 45.57437503896654, - "examples/test_ad_speculative_decoding.py::test_eagle_model_with_weights": 0.8169050384312868, - "examples/test_ad_speculative_decoding.py::test_nemotron_mtp_model_with_weights": 891.9159074425697, - "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]": 245.15667858114466, - "examples/test_ray.py::test_llm_inference_async_ray": 98.74219689599704, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_overlap_scheduler.py]": 552.1893509819638, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py]": 34.445088005042635, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_sleep.py]": 79.70088898506947, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part0\"]": 555.2823958380613, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part1\"]": 546.5310950400308, - "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part2\"]": 401.5109062109841, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"not (gpu2 or gpu4)\"]": 140.57188469800167, - "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8": 419.2079811580479, - "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8_gemm_plugin": 211.37097293697298, - "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8_gemm_swiglu_plugin": 213.21514989249408, - "accuracy/test_cli_flow.py::TestLlama3_1_8BInstruct::test_fp8_prequantized": 162.781315760687, - "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-disable_fp8]": 332.239777687937, - "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-9b-it]": 277.80719018913805, - "examples/test_gemma.py::test_llm_gemma_1gpu_summary_vswa[gemma-3-1b-it-other-bfloat16-8]": 93.52066849544644, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.1-8b]": 149.5570853278041, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_1.5b_instruct]": 159.4304439369589, - "test_e2e.py::test_mistral_large_hidden_vocab_size": 65.75925209745765, - "test_e2e.py::test_trtllm_bench_sanity[--streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 223.21146443486214, - "test_e2e.py::test_trtllm_bench_sanity[-extra_config-streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 222.07093558087945, - "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_moe.py]": 280.5644433479756, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py]": 50.19224583171308, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_quant_matmul.py]": 57.290618147701025, - "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]": 428.9558831639588, - "accuracy/test_llm_api_pytorch_multimodal.py::TestVILA1_5_3B::test_auto_dtype": 407.1404729729984, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_siglip\"]": 223.38703144699684, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_llava_next.py::test_llava_next_expand_prompt_token_ids_for_mm]": 23.44378970999969, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen3vl.py::TestQwen3VL::test_all]": 138.33508067900038, - "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/_test_openai_chat_multimodal.py::test_single_chat_session_image_embeds -m needs_l40s]": 158.215519313002, - "accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype": 856.7462824780014, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nano_v2_vl\"]": 511.051033497999, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_vila\"]": 36.8462807270007, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_llava_next.py::TestLlavaNext::test_all]": 36.84895586700077, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen2_5vl.py::TestQwen2_5_VL::test_all]": 101.95980946499913, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen3vl_moe.py::TestQwen3VLMoe::test_all]": 116.45382200300082, - "examples/test_llama.py::test_llm_llama_1gpu[llama-3.1-8b-instruct-hf-fp8-enable_fp8-float16-summarization-nb:1]": 678.0718930736184, - "examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b]": 798.493692047894, - "test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b]": 113.39240002073348, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 169.85340281389654, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 109.8938184119761, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 67.28465028293431, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 967.6696398910135, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_fp8_hf-Qwen3/saved_models_Qwen3-30B-A3B_fp8_hf]": 1818.6560711394995, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf]": 566.2712351139635, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py]": 18.0341038107872, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py]": 24.46495814807713, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_w4a16_linear.py]": 19.433096984401345, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_w4a8_linear.py]": 18.131888270378113, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_weight_only_quant_gemm.py]": 44.08381740376353, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_weight_only_quant_linear.py]": 20.011995047330856 + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 229.88366167014465, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 1291.2246230191085, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 226.58150382409804, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 230.04569808510132, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 153.0148511910811, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 151.06352188810706, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 157.06875282898545, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 153.20611042692326, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 71.2399792142678, + "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False]": 286.7775873204227537, + "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True]": 286.6778334858827293, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 781.7928658421151, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 270.3750694899354, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2]": 195.4896494857967, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4]": 205.93911361903884, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2]": 188.56422709790058, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4]": 199.29050170327537, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2]": 127.93316596397199, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4]": 129.7617962991353, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2]": 127.73340241820551, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4]": 129.19083517300896, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 329.7597716320306, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 232.4293970640283, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 198.69031671597622, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 199.68801344232634, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False]": 161.8193401999306, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 107.20401050220244, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 163.33450849773362, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 137.72523731505498, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 209.07444706000388, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 149.8242256443482, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram": 286.3013918437063694, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2]": 199.44922504294664, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 173.89489603298716, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2]": 172.84839022206143, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2]": 128.10282056825235, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 121.90447079204023, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2]": 117.0786016730126, + "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[False]": 64428.639228201006, + "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[True]": 572.5455802679062, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False]": 472.62511800276116, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True]": 273.7770717362873, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 56.07656032079831, + "accuracy/test_llm_api.py::TestLlama3_1_8B::test_fp8_rowwise": 361.5573864541948, + "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph": 95.2069768682122, + "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 92.18154831900029, + "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 92.545280149206519, + "accuracy/test_llm_api.py::TestLlama3_2_1B::test_auto_dtype": 48.46169294399442, + "accuracy/test_llm_api.py::TestLlama3_2_1B::test_fp8_rowwise": 46.47006619500462, + "accuracy/test_llm_api.py::TestMixtral8x7B::test_tp2": 226.48782553203637, + "accuracy/test_llm_api.py::TestQwen2_5_0_5BInstruct::test_fp8": 93.85909735393943, + "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_auto_dtype": 173.21916160301771, + "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_fp8": 176.52263558004051, + "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_weight_only": 137.54228180646896, + "accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8": 223.2249169460265, + "accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8_kvcache": 293.131719612516463, + "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_auto_dtype": 146.72803921002196, + "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_fp8": 149.11297382606426, + "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_weight_only": 594.9357111975551, + "accuracy/test_llm_api_pytorch.py::TestBielik11BInstruct::test_auto_dtype": 3600.001755183912, + "accuracy/test_llm_api_pytorch.py::TestBielik11BInstruct::test_fp8": 3600.0018868579646, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput]": 7291.4317992888391, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale_chunked_prefill[latency]": 7291.9177003782242537, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale_chunked_prefill[throughput]": 7291.818120779618621, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency]": 7292.3961801091209054, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp]": 7292.2807201944456277, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen]": 7292.6643901705286214, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen_adp_lmtp]": 7292.5567501965034455, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput]": 7292.690330147743225, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4]": 7292.468819195404649, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp8]": 7292.699600299820304, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_chunked_prefill[latency]": 7292.5356449983082712, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_chunked_prefill[throughput_tp4]": 7292.413349622860551, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_corner_case": 7292.248919965699315, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 591.2785023800097, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 107.58471493399702, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 143.84012729604729, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 295.3527018489549, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 306.84709841990843, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 220.57452515885234, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 165.08514453098178, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 202.22269394202158, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 113.82226522010751, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 205.7252635700861, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True]": 213.78996226208983, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=0-moe_backend=WIDEEP]": 292.7267158059985377, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP]": 292.5756296711042523, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[-attention_dp-cuda_graph-overlap_scheduler-torch_compile=False]": 326.1317654890008, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 647.6109309499152, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 184.20976317999884, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 226.01353620411828, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 506.1045090719126, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-attention_dp-cuda_graph-overlap_scheduler-torch_compile=False]": 336.02580665098503, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 413.903915906325, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 143.841789112892, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 246.64391099987552, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 202.37037238897756, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus_static_eplb": 3600.0020909640007, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 3600.441698686045129, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 3600.591623628977686, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 3600.642337311059237, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 3600.572194146050606, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=0]": 3600.003432472993, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=2]": 3600.21731633093441, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0]": 3600.002661294944, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2]": 3600.0022540579666, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 89.92349556891713, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 85.24235329206567, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 252.70569713797886, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 90.21807348495349, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 175.661773331929, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 360.0003233290044590831, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=False-moe_backend=WIDEEP]": 360.9275938109494746, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=WIDEEP]": 360.0002855450729839504, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=0.75-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 360.0003064870252273977, + "accuracy/test_llm_api_pytorch.py::TestEXAONE4::test_auto_dtype": 3600.0004039629711769521, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 360.00032637204276397824, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-fp8]": 360.0003586999955587089, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto]": 360.6586053780047223, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-auto]": 360.0003633099840953946, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-fp8]": 360.00036422599805518985, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 360.00032637204276397824, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-fp8]": 360.0003586999955587089, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-triton-auto]": 360.6586053780047223, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-auto]": 360.0003633099840953946, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-fp8]": 360.00036422599805518985, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-cutlass-auto]": 360.0003378289984539151, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-triton-auto]": 360.9436147869564593, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-trtllm-auto]": 360.0003398499684408307, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-trtllm-fp8]": 360.0002922280109487474, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-cutlass-auto]": 360.0003666180418804288, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-triton-auto]": 360.9300670439261012, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-trtllm-auto]": 360.0002812399761751294, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-trtllm-fp8]": 360.0008064290159381926, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-cutlass-auto]": 360.0003697940264828503, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-triton-auto]": 360.8670774899655953, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-trtllm-auto]": 360.00040231598541140556, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-trtllm-fp8]": 360.0003254589391872287, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-cutlass-auto]": 745.8583740849863, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 745.9345730679342523, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-auto]": 745.0004936959594488144, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-fp8]": 745.00031642295653000474, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-cutlass-auto]": 658.1757711600512, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 745.9436021829606034, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 745.0004371170070953667, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-fp8]": 745.0004142870311625302, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-auto]": 676.3980704760179, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 745.0292645250447094, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-auto]": 745.0003769229515455663, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-fp8]": 677.000331886054482311, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-cutlass-auto]": 745.8583740849863, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 745.9345730679342523, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-auto]": 745.0004936959594488144, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-fp8]": 745.00031642295653000474, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutlass-auto]": 658.1757711600512, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 745.9436021829606034, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 745.0004371170070953667, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-fp8]": 745.0004142870311625302, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto]": 676.3980704760179, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-triton-auto]": 745.0292645250447094, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 745.0003769229515455663, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-fp8]": 677.000331886054482311, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto]": 643.3513998010312, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto]": 764.9216735750087537, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-auto]": 764.0002969659981317818, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-fp8]": 764.0008383550448343158, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 764.8800516680348665, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8]": 764.00035094103077426553, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 14400, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler]": 14400, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 88.30407958402066, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype": 892.3242024959764, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 451.8630696780165, + "accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype": 86.96273394301534, + "accuracy/test_llm_api_pytorch.py::TestKimiK2::test_fp8_blockscale[latency]": 0.19557904999237508, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 56.31924073398113, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False]": 115.23303954396397, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True]": 115.0261728389305, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False]": 98.29266697796993, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=True]": 96.56704166403506, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False]": 98.47511212801328, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True]": 96.76115251897136, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 307.12596721109, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 443.91388061689213, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 191.10617867391557, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 166.85348949534819, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[attn_backend=FLASHINFER]": 167.15153613401344, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[attn_backend=TRTLLM]": 90.12104846700095, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 1112.0988524899585, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 979.2759481471148, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 237.24446990108117, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 226.39608797896653, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 174.38962662010454, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 313.69273760309443, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 409.8932851999998, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 344.8807112099603, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 103.82129427790642, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 164.91815144987777, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False]": 124.62386814301135, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True]": 121.9821372089209, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False]": 77.16783500701422, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=True]": 75.89485901995795, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False]": 77.46995012206025, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True]": 75.93042165302904, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_llm_sampler": 61.986203632026445, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 44.978786278981715, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 46.60325947904494, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[llguidance]": 52.44408063602168, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 52.84089746000245, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False]": 36.739327706047334, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 67.41741452802671, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 36.758924948982894, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 32.73584783205297, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[llguidance]": 31.598825128981844, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[xgrammar]": 31.997449714050163, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_ngram": 404.38593446696177, + "accuracy/test_llm_api_pytorch.py::TestLlama3_2_1B::test_auto_dtype": 49.4644276680192, + "accuracy/test_llm_api_pytorch.py::TestLlama3_2_1B::test_fp8_prequantized": 40.83171262202086, + "accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype": 60.00025596999330446124, + "accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_fp8_prequantized": 67.63213626696961, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False]": 3770.7388199700508, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False]": 4010.9139676889754, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False]": 2616.262340236979, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 3600.00026297004660591483, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4-cuda_graph=False]": 7200.00030252401484176517, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4ep2-cuda_graph=True]": 7200.0002791329752653837, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4ep4-cuda_graph=True]": 7200.00028608099091798067, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False]": 7200.0002498350222595036, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True]": 7200.00023661594605073333, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True]": 7200.00023339298786595464, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_chunked_prefill[attn_backend=FLASHINFER]": 7200.0002561529981903732, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_chunked_prefill[attn_backend=TRTLLM]": 7200.0052206520340405405, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4-cuda_graph=True]": 7200.2161163400160149, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4ep2-cuda_graph=True]": 7200.2132116109714843, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4ep4-cuda_graph=True]": 7200.3660773960873485, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8-cuda_graph=True]": 7200.00025232898769900203, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8ep4-cuda_graph=True]": 7200.00025386200286448, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8ep8-cuda_graph=True]": 7200.00023917207727208734, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_chunked_prefill[tp8ep8-cuda_graph=False]": 7200.5301868109382, + "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_chunked_prefill[tp8ep8-cuda_graph=True]": 72600.1488124400494, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4-cuda_graph=False]": 3600.0010551271309959702, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4ep2-cuda_graph=True]": 3600.0009890546519891359, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4ep4-cuda_graph=True]": 3600.000870058874017559, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-cuda_graph=False]": 3600.0022709049517, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep4-cuda_graph=True]": 3600.0008703919593, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep8-cuda_graph=True]": 3600.001674739062, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4[tp4-cuda_graph=True]": 3600.0003222679952159524, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4[tp8ep8-cuda_graph=True]": 3600.0004189839819446206, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4_chunked_prefill[tp4ep4-cuda_graph=True]": 3600.0009446179610677063, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp4-cuda_graph=True]": 3600.0600651470012963, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp8ep8-cuda_graph=True]": 3600.0020443379763, + "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8_chunked_prefill[tp4ep4-cuda_graph=True]": 3600.0016280449927, + "accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_auto_dtype": 300.0017418859643, + "accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8": 300.001715709921, + "accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized": 48.064747432945296, + "accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype": 96.73990149900783, + "accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype": 3600.0012007239857, + "accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype_tp2": 3600.001961252012, + "accuracy/test_llm_api_pytorch.py::TestMistralSmall24B::test_auto_dtype": 3600.0003251209855079651, + "accuracy/test_llm_api_pytorch.py::TestMistralSmall24B::test_fp8": 196.57955891895108, + "accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2": 3600.0022730380297, + "accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2": 3600.7994798690197058, + "accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8": 3600.5389363930444, + "accuracy/test_llm_api_pytorch.py::TestPhi4::test_auto_dtype": 3600.0017098310054, + "accuracy/test_llm_api_pytorch.py::TestPhi4::test_fp8": 3600.0018840720295, + "accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype": 3600.000989172084, + "accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype_long_rope": 3600.001809718029, + "accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype": 3600.0018334789784, + "accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype": 49.38216367200948, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency]": 7576.4847942629713, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency]": 75934.1885519769276, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass]": 7570.00041150598553940654, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm]": 7570.0005082660354673862, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp]": 7570.0003929820377379656, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_trtllm_eagle3]": 7570.000994006055407226, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 3240.3035402488895, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 3240.3568000392988324, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False]": 3240.326488903677091, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True]": 3240.3251879825256765, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False]": 3240.3220480284653604, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]": 3240.12236930197104812, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM]": 3240.3637070371955633, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]": 3240.5122077039559372, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRITON]": 3240.46138638898264617, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]": 3240.4374056719825603, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]": 3240.4651110970880836, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]": 3240.4263977239606902, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.5-fp8kv=False]": 1202.0, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.9-fp8kv=False]": 1127.0, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS]": 1091.0, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-TRTLLM]": 1091.0, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False]": 313.0, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=True]": 313.0, + "accuracy/test_llm_api_pytorch.py::TestQwen3_6_27B::test_fp8": 569.04, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency-torch_compile=False]": 149.19146074401215, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 104.32479889906244, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 240.30756398336961865, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[mxfp8-latency]": 240.27633493300527334, + "cpp/test_e2e.py::test_benchmarks[bart-90]": 271.95234084688127, + "cpp/test_e2e.py::test_benchmarks[gpt-80]": 1376.0404928650241, + "cpp/test_e2e.py::test_benchmarks[t5-90]": 523.07, + "cpp/test_e2e.py::test_model[-bart-90]": 391.84748707409017, + "cpp/test_e2e.py::test_model[-eagle-86]": 850.5158995762467, + "cpp/test_e2e.py::test_model[-enc_dec_language_adapter-90]": 416.06, + "cpp/test_e2e.py::test_model[-gpt-80]": 1568.98, + "cpp/test_e2e.py::test_model[-gpt_executor-80]": 1495.14, + "cpp/test_e2e.py::test_model[-gpt_tests-80]": 1206.79, + "cpp/test_e2e.py::test_model[-mamba-86]": 893.8684413917363, + "cpp/test_e2e.py::test_model[-medusa-86]": 577.0913726426661, + "cpp/test_e2e.py::test_model[-redrafter-86]": 356.56682327389717, + "cpp/test_e2e.py::test_model[-t5-90]": 170.26, + "cpp/test_e2e.py::test_model[fp8-llama-90]": 385.98, + "cpp/test_unit_tests.py::test_unit_tests[batch_manager-80]": 1005.24, + "cpp/test_unit_tests.py::test_unit_tests[common-80]": 38.98, + "cpp/test_unit_tests.py::test_unit_tests[common-90]": 25.06, + "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 425.16, + "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 2009.96, + "cpp/test_unit_tests.py::test_unit_tests[kernels-90]": 1333.18, + "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 2209.11, + "cpp/test_unit_tests.py::test_unit_tests[layers-90]": 1627.07, + "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 1671.42, + "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 6.76, + "cpp/test_unit_tests.py::test_unit_tests[thop-90]": 4.16, + "cpp/test_unit_tests.py::test_unit_tests[utils-80]": 8.53, + "cpp/test_unit_tests.py::test_unit_tests[utils-90]": 5.28, + "disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0]": 54.26733888499439, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0]": 64.01942083099857, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0]": 67.32925470802002, + "disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0]": 50.57874152995646, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8]": 105.12023228011094, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one[DeepSeek-V3-Lite-fp8]": 98.13158084987663, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one_mtp[DeepSeek-V3-Lite-fp8]": 104.78005758393556, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_mpi[DeepSeek-V3-Lite-fp8]": 680.20395052596, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 102.03138376423158, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 90.40784636512399, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8]": 124.17078560194932, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 102.67233599093743, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 224.28071974776685, + "disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0]": 52.78952780482359, + "disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0]": 73.48997121001594, + "disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0]": 67.3897166326642, + "disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0]": 54.22262764698826, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 98.97588296607137, + "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0]": 67.9668476767838, + "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0]": 82.28277984517626, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8]": 238.76137515995651, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 78.98068026197143, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 77.51831256924197, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 99.81417108187452, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-True-DeepSeek-V3-Lite-fp8/fp8]": 67.32832619687542, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 48.16434509307146, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 36.88020430901088, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 46.302398771978915, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 38.81214914191514, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 31.12580855889246, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 29.8858498937916, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 55.79476427496411, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 55.29072010796517, + "disaggregated/test_workers.py::test_workers_conditional_disaggregation[TinyLlama-1.1B-Chat-v1.0]": 99.67464494984597, + "disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0]": 113.69421282503754, + "disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0]": 195.86076739802957, + "disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0]": 76.3156570668798, + "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:1-pp:1-float32-BertModel-bert/bert-base-uncased]": 111.17977902293205, + "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:1-pp:1-float32-RobertaModel-bert/roberta-base]": 115.47540166974068, + "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-BertForQuestionAnswering-bert/bert-base-cased-squad2]": 64.32757595699513, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 55.807815481035504, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-BertModel-bert/bert-base-uncased]": 60.69481396203628, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertForQuestionAnswering-bert/bert-base-cased-squad2]": 73.41519981494639, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 73.00019321503350511193, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 121.36917966976762, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaForQuestionAnswering-bert/roberta-base-squad2]": 74.99331583303865, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaForSequenceClassification-bert/twitter-roberta-base-emotion]": 73.000189481012057513, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaModel-bert/roberta-base]": 122.99858937039971, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha_fp32_acc-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 55.26069303997792, + "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-chat-disable_weight_only]": 3640.0045928549953, + "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-chat-enable_weight_only]": 3235.5094731519348, + "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-disable_weight_only]": 3273.7859199331142, + "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-enable_weight_only]": 3602.5807401749771, + "examples/test_commandr.py::test_llm_commandr_v01_single_gpu_summary[disable_weight_only]": 1856.5704392530024, + "examples/test_commandr.py::test_llm_commandr_v01_single_gpu_summary[enable_weight_only]": 654.6126579149859, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 244.9744301661849, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_logits-draft_len_8-float16-bs1]": 398.78198734589387, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 241.73137632384896, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-llama_v2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 1099.3959164519329, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 249.52418848499656, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 257.3995385244489, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_tokens-draft_len_8-float16-bs1]": 397.20915103994776, + "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-llama_v2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 582.6556157508167, + "examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle1]": 254.24225717037916, + "examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle2]": 246.98607586231083, + "examples/test_eagle.py::test_llm_eagle_1gpu_modelopt_ckpt[llama3.1-eagle-8b-hf_v0.5-float16-bs8]": 489.4540088879876, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-bfloat16-enable_gemm_plugin-enable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 232.2934926636517, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-bfloat16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 281.1201816312969, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 2954.5586752621457, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:2-disable_fp8]": 276.10329104214907, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 233.94542215764523, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 283.8865255280398, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-disable_gemm_plugin-disable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 220.32321695238352, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-disable_gemm_plugin-enable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 218.02495155483484, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 96.25796012202045, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 568.2032693652436, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:2-nb:1-enable_fp8]": 354.8090338760521, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-mbart-large-50-many-to-one-mmt-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 160.50506488600513, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-mbart-large-50-many-to-one-mmt-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:2-nb:1-enable_fp8]": 300.20793505245819688, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-disable_gemm_plugin-disable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 218.7171499580145, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 217.96836187317967, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 498.1236152825877, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:2-disable_fp8]": 205.83720442652702, + "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:1-nb:1-enable_fp8]": 295.6078515049885, + "examples/test_enc_dec.py::test_llm_enc_dec_general[no_compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 252.0442810030654, + "examples/test_enc_dec.py::test_llm_enc_dec_general[no_compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:1-nb:1-disable_fp8]": 95.48429084802046, + "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-disable_fp8]": 422.4394793640822, + "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-enable_fp8]": 1074.875556848012, + "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_3.0_7.8b_instruct-float16-nb:1]": 3002.19654186069965, + "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_3.0_7.8b_instruct-float16-nb:4]": 3074.5337073504925, + "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_deep_2.4b-float16-nb:4]": 243.4259528592229, + "examples/test_exaone.py::test_llm_exaone_1gpu[enable_weight_only-exaone_deep_2.4b-float16-nb:1]": 212.50885355699575, + "examples/test_exaone.py::test_llm_exaone_2gpu[exaone_3.0_7.8b_instruct-float16-nb:1]": 7155.35844087804435, + "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-27b-it]": 317.7816583644599, + "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-9b-it]": 317.7816583644599, + "examples/test_gemma.py::test_llm_gemma_1gpu_summary_vswa[gemma-3-1b-it-other-bfloat16-8]": 195.3050664511975, + "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[non_streaming-use_cpp_session-enable_gemm_plugin]": 114.20040711760521, + "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[non_streaming-use_py_session-disable_gemm_plugin]": 105.38906556204893, + "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[streaming-use_cpp_session-enable_gemm_plugin]": 113.51056583970785, + "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[non_streaming-use_cpp_session]": 194.89961875230074, + "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[non_streaming-use_py_session]": 200.52475621178746, + "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[streaming-use_cpp_session]": 195.00627667084336, + "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[non_streaming-use_cpp_session]": 194.90547297894955, + "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[non_streaming-use_py_session]": 194.89357279613614, + "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[streaming-use_cpp_session]": 194.72326660901308, + "examples/test_gpt.py::test_llm_gpt2_next_prompt_tuning[use_cpp_session-tp1]": 460.1370678450912, + "examples/test_gpt.py::test_llm_gpt2_next_prompt_tuning[use_py_session-tp1]": 403.39630596572533, + "examples/test_gpt.py::test_llm_minitron_fp8_with_pseudo_loras[4b]": 259.4826051471755, + "examples/test_granite.py::test_granite_bf16_lora[granite-3.0-1b-a400m-instruct]": 140.08338637300767, + "examples/test_granite.py::test_granite_bf16_lora[granite-3.0-2b-instruct]": 146.46410073013976, + "examples/test_granite.py::test_llm_granite[granite-3.0-1b-a400m-instruct-bfloat16]": 141.05149138718843, + "examples/test_granite.py::test_llm_granite[granite-3.0-2b-instruct-bfloat16]": 1055.801738537848, + "examples/test_internlm.py::test_llm_internlm2_7b_1node_1gpu[bfloat16-enable_context_fmha-enable_gemm_plugin-enable_attention_plugin-nb:2]": 2690.7412294782698, + "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.1-8b]": 160.33107751235366, + "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.2-1b]": 117.10959041584283, + "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.2-3b]": 215.69512976403348, + "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-v2-7b-hf]": 376.96383721905295, + "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-v3-8b-instruct-hf]": 317.46144750900567, + "examples/test_llama.py::test_llm_llama_1gpu[llama-3.1-8b-instruct-hf-fp8-enable_fp8-float16-summarization-nb:1]": 853.2910006027669, + "examples/test_llama.py::test_llm_llama_1gpu_batched_beam_search[llama-7b]": 182.20104870200157, + "examples/test_llama.py::test_llm_llama_1gpu_fp8_kv_cache[llama-v2-7b-hf-bfloat16]": 313.6555140609853, + "examples/test_llama.py::test_llm_llama_long_alpaca_8gpu_summary[pg64317-tp8pp1-nb:1]": 441.2820012559532, + "examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b]": 167.92376559507102, + "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_awq]": 420.1779588930076, + "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp16]": 895.7611340929288, + "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp8]": 314.3205590210273, + "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_int8_wo]": 329.1954380639363, + "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_sq_ootb]": 216.2645359209855, + "examples/test_llama.py::test_llm_llama_v3_1_1node_single_gpu[llama-3.2-1b-disable_fp8]": 382.12588274572045, + "examples/test_llama.py::test_llm_llama_v3_dora_1gpu[commonsense-llama-v3-8b-dora-r32-llama-v3-8b-hf-base_fp16]": 517.2770831151865, + "examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0]": 64.8964971601963, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-1.4b-float16-enable_gemm_plugin]": 181.03255189501215, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-130m-float16-disable_gemm_plugin]": 136.8141469657421, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-130m-float16-enable_gemm_plugin]": 112.04011878371239, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-2.8b-float16-disable_gemm_plugin]": 336.9864642599714, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-370m-float16-enable_gemm_plugin]": 87.74965502799023, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-790m-float16-disable_gemm_plugin]": 148.56857219303492, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-codestral-7B-v0.1-float16-disable_gemm_plugin]": 405.1586506664753, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-codestral-7B-v0.1-float16-enable_gemm_plugin]": 384.9690850973129, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-1.3b-float16-enable_gemm_plugin]": 126.15442710905336, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-130m-float16-disable_gemm_plugin]": 129.8332964628935, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-130m-float16-enable_gemm_plugin]": 118.37521690130234, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-2.7b-float16-disable_gemm_plugin]": 212.40601160994265, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-370m-float16-enable_gemm_plugin]": 75.9140890170238, + "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-780m-float16-disable_gemm_plugin]": 114.81139776791679, + "examples/test_medusa.py::test_llm_medusa_1gpu[use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs1]": 217.8724013082683, + "examples/test_medusa.py::test_llm_medusa_1gpu[use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs8]": 460.24718615040183, + "examples/test_medusa.py::test_llm_medusa_1gpu[use_py_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs1]": 129.02341048512608, + "examples/test_medusa.py::test_llm_medusa_1gpu[use_py_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs8]": 524.8282293006778, + "examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1]": 383.3182801879011, + "examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_py_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1]": 293.03625723719597, + "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-chunked_summarization_long]": 429.02448211982846, + "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-summarization]": 603.6547773182392, + "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-summarization_long]": 391.7267559207976, + "examples/test_mixtral.py::test_llm_mixtral_moe_plugin_lora_4gpus[Mixtral-8x7B-v0.1-chinese-mixtral-lora]": 3600.00015190104022622108, + "examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-cnn_dailymail-Qwen2-VL-7B-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False]": 332.0248579243198, + "examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-scienceqa-Llama-3.2-11B-Vision-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 411.7690062429756, + "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 385.0684349639341, + "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:2-bfloat16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3-vision-128k-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 282.8564471802674, + "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3-vision-128k-instruct-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 90.19939221709501, + "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3.5-vision-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 275.5947739640251, + "examples/test_multimodal.py::test_llm_multimodal_general[Phi-4-multimodal-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 644.3520091949031, + "examples/test_multimodal.py::test_llm_multimodal_general[Qwen2-VL-7B-Instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:4]": 656.4784073680639, + "examples/test_multimodal.py::test_llm_multimodal_general[deplot-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 249.87254932150245, + "examples/test_multimodal.py::test_llm_multimodal_general[deplot-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 100.08572303387336, + "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1]": 492.22362083010375, + "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1]": 333.81485258904286, + "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 459.2980541479774, + "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 176.8547668098472, + "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, + "examples/test_multimodal.py::test_llm_multimodal_general[llava-onevision-qwen2-7b-ov-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 533.2010767317843, + "examples/test_multimodal.py::test_llm_multimodal_general[llava-onevision-qwen2-7b-ov-hf-video-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 200.52463799191173, + "examples/test_multimodal.py::test_llm_multimodal_general[llava-v1.6-mistral-7b-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 274.8723033480346, + "examples/test_multimodal.py::test_llm_multimodal_general[llava-v1.6-mistral-7b-hf-vision-trtllm-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 306.38610201328993, + "examples/test_multimodal.py::test_llm_multimodal_general[nougat-base-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 226.57772368215956, + "examples/test_multimodal.py::test_llm_multimodal_general[nougat-base-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 198.46779074892402, + "examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 260.0002240890171378851, + "examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 260.00014175008982419968, + "examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B]": 335.41048416192643, + "examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs1]": 200.00026965001598000526, + "examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2]": 196.1214354224503, + "examples/test_ngram.py::test_llm_ngram_1gpu[streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs1]": 200.0001331439707428217, + "examples/test_ngram.py::test_llm_ngram_1gpu[streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2]": 195.90045699477196, + "examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp8]": 220.0002483620774000883, + "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3-mini-128k-instruct-fp8-float16]": 360.0002299160696566105, + "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3.5-MoE-instruct-fp8-bfloat16]": 360.0001645770389586687, + "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3.5-mini-instruct-fp8-float16]": 360.0001653869403526187, + "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-4-mini-instruct-fp8-bfloat16]": 360.00032463017851114273, + "examples/test_phi.py::test_llm_phi_quantization_1gpu[phi-2-fp8-bfloat16]": 154.69453522900585, + "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_0.5b_instruct]": 99.85199454193935, + "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_1.5b_instruct]": 193.38715927954763, + "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2_0.5b_instruct]": 128.65416727401316, + "examples/test_qwen.py::test_llm_qwen1_5_7b_single_gpu_lora[qwen1.5_7b_chat-Qwen1.5-7B-Chat-750Mb-lora]": 338.59182655182667, + "examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct]": 423.2417808100581, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.0001445889938622713, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-fp8-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00012858898844569921, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-int4_awq-float16-enable_attn_plugin-enable_gemm_plugin]": 648.7579195387661, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-int8_sq-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00033479504054412246, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-flax-no_paged_cache-disable_quant-float16-enable_attn_plugin-disable_gemm_plugin]": 286.15992603078485, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-no_paged_cache-disable_quant-float16-disable_attn_plugin-enable_gemm_plugin]": 680.0001984360278584063, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-no_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00013055797899141908, + "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.0001333290128968656, + "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_cpp_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb5-bs8]": 386.68252966180444, + "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_cpp_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb8-bs8]": 204.7229775050655, + "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_py_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb5-bs8]": 411.88197461143136, + "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_py_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb8-bs8]": 429.239758990705, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 327.95307156071067, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_cpp_runtime]": 249.98457504063845, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 225.60136043280363, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-int4-float16-nb:1-use_cpp_runtime]": 114.39965145103633, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-int8-float16-nb:1-use_cpp_runtime]": 109.14321990706958, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-enable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 110.00021485507022589445, + "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_logging": 458.4059731345624, + "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_default": 40.70597969903611, + "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_tensorrt": 99.26517963293009, + "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-codellama/CodeLlama-7b-Instruct-hf]": 383.1450063039083, + "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-llama-models/llama-7b-hf]": 101.96415167767555, + "llmapi/test_llm_examples.py::test_llmapi_server_example": 107.44753806525841, + "perf/test_perf.py::test_perf[bert_base-cpp-ootb-float16-bs:32-input_len:32]": 111.37450777366757, + "perf/test_perf.py::test_perf[bert_base-cpp-plugin-float16-bs:32-input_len:32]": 95.00738414749503, + "perf/test_perf.py::test_perf[roberta_base-cpp-plugin-float16-bs:32-input_len:128+512]": 140.2516261599958, + "test_e2e.py::test_build_time_benchmark_sanity": 165.71592589840293, + "test_e2e.py::test_gpt3_175b_1layers_build_only": 131.34366285055876, + "test_e2e.py::test_llama_e2e[use_cpp_session-remove_input_padding-]": 90.12235352798598, + "test_e2e.py::test_llama_e2e[use_py_session--]": 92.68217968731187, + "test_e2e.py::test_llama_e2e[use_py_session-remove_input_padding-]": 93.41917416104116, + "llmapi/test_llm_examples.py::test_llmapi_chat_example": 105.19824166595936, + "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 73.23708964884281, + "llmapi/test_llm_examples.py::test_llmapi_example_inference": 66.82718145102262, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 65.93082024902105, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 67.49109892174602, + "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 72.87169548124075, + "llmapi/test_llm_e2e.py::test_llmapi_exit": 32.64902823418379, + "llmapi/test_llm_e2e.py::test_llmapi_load_ckpt_from_convert_command": 180.59318951144814, + "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command_with_lora[llama-llama-models-v2/llama-v2-7b-hf]": 225.2778383679688, + "llmapi/test_llm_examples.py::test_llmapi_quickstart": 66.53727849572897, + "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 110.45052940770984, + "test_e2e.py::test_mistral_e2e[use_cpp_session-remove_input_padding--]": 192.74050169810653, + "test_e2e.py::test_mistral_e2e[use_py_session---]": 160.08483010903, + "test_e2e.py::test_mistral_e2e[use_py_session-remove_input_padding--]": 157.39577213302255, + "test_e2e.py::test_mistral_large_hidden_vocab_size": 81.36711680702865, + "test_e2e.py::test_openai_chat_example": 876.1966922096908, + "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 55.12449237401597, + "test_e2e.py::test_openai_chat_harmony": 1162.7252594940364, + "test_e2e.py::test_openai_chat_multimodal_example": 215.8254322744906, + "test_e2e.py::test_openai_misc_example": 256.0453990884125, + "test_e2e.py::test_openai_multi_chat_example": 684.1194127409253, + "test_e2e.py::test_ptp_quickstart": 45.67732956283726, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 37.10870039300062, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 81.43792725296225, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 56.05445321695879, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.2-11B-BF16-llama-3.2-models/Llama-3.2-11B-Vision]": 458.2611172522884, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 1198.8076391071081, + "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Maverick-17B-128E-Instruct-FP8-llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8-False]": 7750.113898515003, + "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Maverick-17B-128E-Instruct-FP8-llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8-True]": 7404.8092977448832, + "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Scout-17B-16E-Instruct-FP4-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4-True]": 3600.00027390988543629646, + "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Scout-17B-16E-Instruct-FP8-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8-True]": 2183.7999707763083, + "test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]": 8346.51794013707, + "test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8]": 11215.495469792979, + "test_e2e.py::test_ptp_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B]": 109.26379436196294, + "test_e2e.py::test_ptp_quickstart_advanced_mixed_precision": 80.88908524392173, + "test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]": 99.42739840806462, + "test_e2e.py::test_ptp_quickstart_advanced_ngram[Llama-3.1-8B-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct]": 71.96910276100971, + "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 21.019993914989755, + "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 18.753523574909195, + "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[llava-v1.6-mistral-7b-llava-v1.6-mistral-7b-hf-0.8-image]": 354.18758003995754, + "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[phi4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct-0.8-image]": 313.79139575595036, + "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[qwen2.5-vl-7b-instruct-Qwen2.5-VL-7B-Instruct-0.8-image]": 47.84053734713234, + "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[qwen2.5-vl-7b-instruct-Qwen2.5-VL-7B-Instruct-0.8-video]": 58.646747548133135, + "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[Phi-4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct]": 0.00018897396512329578, + "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[gemma-3-27b-it-gemma/gemma-3-27b-it]": 81.0151858178433, + "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[mistral-small-3.1-24b-instruct-Mistral-Small-3.1-24B-Instruct-2503]": 60.082822071621194, + "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[audio]": 89.50578387826681, + "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[image]": 66.199569062097, + "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[image_audio]": 62.389084175927565, + "test_e2e.py::test_ptp_scaffolding[DeepSeek-R1-Distill-Qwen-7B-DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B]": 7200.0001350759994238615, + "test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B]": 137.7278483910486, + "test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]": 12134.278186964104, + "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 109.25386995915323, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 163.86223009089008, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 114.50899445591494, + "test_e2e.py::test_trtllm_bench_iteration_log[TRT-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 285.3362849447876, + "test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 310.9046222809702, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 114.17938271397725, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 115.74023819994181, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 276.64185731019825, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 252.97791706770658, + "test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b]": 482.31134233786725, + "test_e2e.py::test_trtllm_benchmark_serving[llama-3.1-model/Meta-Llama-3.1-8B]": 80.71726501686499, + "test_e2e.py::test_trtllm_multimodal_benchmark_serving": 360.00022564898245036602, + "test_e2e.py::test_trtllm_serve_example": 200.09309104084969, + "test_e2e.py::test_trtllm_serve_multimodal_example": 130.2214687075466, + "test_unittests.py::test_unittests_v2[unittest/_torch/attention/test_attention_mla.py]": 26.32902159006335, + "test_unittests.py::test_unittests_v2[unittest/_torch/attention]": 588.56, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu]": 539.3006387590431, + "test_unittests.py::test_unittests_v2[unittest/_torch/compilation]": 31.94, + "test_unittests.py::test_unittests_v2[unittest/_torch/debugger]": 36.69, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor]": 170.86, + "test_unittests.py::test_unittests_v2[unittest/_torch/flashinfer/test_trtllm_flashinfer_symbol_collision.py]": 1004.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/misc]": 600.5, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_llama\"]": 718.749935634085, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mixtral\"]": 208.1838396479725, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mllama\"]": 749.5508671940188, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron\"]": 1952.3731448464096, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nas\"]": 498.8839871880482, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_out_of_tree\"]": 55.078535287990235, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen\"]": 551.1881373599754, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen_moe\"]": 401.2630233000382, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_vila\"]": 79.90315388399176, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules]": 158.5, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"]": 68.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"]": 1598.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"]": 22.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DENSEGEMM\"]": 149.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"TRTLLM\"]": 998.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling -k \"deepseek\"]": 393.0210295501165, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_external_embedding.py]": 30.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_find_num_image_tokens.py]": 240.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_fuse_input_embeds.py]": 60.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_mm_encoder_standalone.py]": 1800.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_multimodal_runtime.py]": 180.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_share_multiparams.py]": 30.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler]": 1020.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/speculative]": 1850.16, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel]": 1463.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/serial]": 18.96, + "test_unittests.py::test_unittests_v2[unittest/api_stability]": 33.137137457728386, + "test_unittests.py::test_unittests_v2[unittest/bindings]": 1119.2564616799355, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_build_cache.py]": 34.61376368254423, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_executor.py]": 378.7100401185453, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm.py -m \"not part0\"]": 1883.5484512336552, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm.py -m \"part0\"]": 1601.0243577323854, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"not (part0 or part1)\"]": 825.9972547292709, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"part0\"]": 163.72848848626018, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"part1\"]": 538.573951125145, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_perf_evaluator.py]": 118.36046380549669, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py::test_gemma3_1b_instruct_multi_lora]": 101.58543362899218, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py]": 539.5857984796166, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_quant.py]": 477.989566125907, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_utils.py]": 125.15857975929976, + "test_unittests.py::test_unittests_v2[unittest/test_model_runner_cpp.py]": 973.1355891097337, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_bert_attention.py]": 99.96196278184652, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition0\"]": 77.31474154582247, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition1\"]": 84.67568279313855, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition2\"]": 75.39135546097532, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition3\"]": 78.77339706313796, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"trtllm_gen\"]": 376.012343961047, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"xqa_generic\"]": 267.40264504775405, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_IFB.py]": 85.18935105204582, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_no_cache.py]": 49.3486054521054, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_sage_attention.py unittest/llmapi/test_llm_download.py unittest/llmapi/test_llm_kv_cache_events.py unittest/llmapi/test_mpi_session.py unittest/trt/model/redrafter unittest/trt/model/test_phi.py unittest/trt/model/test_unet.py unittest/trt/python_plugin unittest/tools unittest/utils unittest/others]": 940.7867036014795, + "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_fp4_gemm.py]": 302.49857676401734, + "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_moe.py]": 220.60184395778924, + "test_unittests.py::test_unittests_v2[unittest/trt/functional]": 778.6451135131065, + "test_unittests.py::test_unittests_v2[unittest/trt/model/eagle]": 212.3223411180079, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"other\"]": 125.12117889150977, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition0\"]": 300.0489609502256, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition1\"]": 265.11671224981546, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition2\"]": 357.6496359631419, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition3\"]": 371.381394200027, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt_e2e.py]": 537.5006402550498, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_llama.py]": 1494.1103300452232, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mamba.py]": 76.84791256207973, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mistral.py]": 366.95385985821486, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_nemotron_nas.py -k \"not fp8\"]": 1041.1297603696585, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_api_multi_gpu.py]": 27.33125525712967, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_level_api.py]": 33.818626184016466, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_quantization.py]": 493.8186915554106, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py]": 214.35422350093722, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_quant_matmul.py]": 100.81762219779193, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization]": 673.2582192085683 } diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml similarity index 97% rename from tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml rename to tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml index d5fa9d45a931..b7e04b714d00 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml @@ -22,7 +22,7 @@ context_servers: enable_block_reuse: false free_gpu_memory_fraction: 0.8 moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: null print_iter_log: true cache_transceiver_config: @@ -45,7 +45,7 @@ generation_servers: enable_block_reuse: false free_gpu_memory_fraction: 0.8 moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: enable_padding: true batch_sizes: diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml new file mode 100644 index 000000000000..1ebbe92ac159 --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml @@ -0,0 +1,67 @@ +model: gpt_oss/gpt-oss-120b +hostname: localhost +backend: pytorch +context_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 16640 + max_seq_len: 8232 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: &eagle_spec + decoding_type: Eagle + max_draft_len: 3 + eagle3_one_model: true + speculative_model: gpt_oss/gpt-oss-120b-Eagle3 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: TRTLLM + cuda_graph_config: null + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 +generation_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 10240 + max_seq_len: 10240 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: *eagle_spec + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: TRTLLM + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml similarity index 96% rename from tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml rename to tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml index 47b619faf141..1d6f4d336e01 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml @@ -17,7 +17,7 @@ context_servers: free_gpu_memory_fraction: 0.8 disable_overlap_scheduler: true moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: null print_iter_log: true cache_transceiver_config: @@ -39,7 +39,7 @@ generation_servers: free_gpu_memory_fraction: 0.8 disable_overlap_scheduler: true moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: enable_padding: true batch_sizes: diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index f320690ad787..b8136359f972 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -298,12 +298,14 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml", "deepseek_r1_v2_fp4_mtp_stress": f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml", - "gpt_oss_120b_stress": + "gpt_oss_120b_trtllm_stress": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml", - "gpt_oss_120b_eagle_stress": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml", - "gpt_oss_120b_cutlass_stress": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml", + "gpt_oss_120b_eagle_triton_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml", + "gpt_oss_120b_eagle_trtllm_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml", + "gpt_oss_120b_triton_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml", "qwen3_5_4b_fp8_stress": f"{test_configs_root}/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml", "gpt_oss_120b_harmony": @@ -2297,7 +2299,7 @@ def test_disaggregated_gpt_oss_120b_harmony(disaggregated_test_root, cancellation_delay=0.5), marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell)), pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_stress', + test_desc='gpt_oss_120b_trtllm_stress', request_count=60000, accuracy_threshold=0.42, cancellation_rate=10, @@ -2306,22 +2308,34 @@ def test_disaggregated_gpt_oss_120b_harmony(disaggregated_test_root, pytest.param( TestConfig( model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_eagle_stress', + test_desc='gpt_oss_120b_eagle_triton_stress', request_count=60000, accuracy_threshold=0.42, speculative_model_path='gpt_oss/gpt-oss-120b-Eagle3', cancellation_rate=10, cancellation_delay=0.5, ), - marks=(pytest.mark.skip_less_device(8), skip_pre_hopper), + marks=(pytest.mark.skip_less_device(8), skip_no_hopper), + ), + pytest.param( + TestConfig( + model_path='gpt_oss/gpt-oss-120b', + test_desc='gpt_oss_120b_eagle_trtllm_stress', + request_count=60000, + accuracy_threshold=0.42, + speculative_model_path='gpt_oss/gpt-oss-120b-Eagle3', + cancellation_rate=10, + cancellation_delay=0.5, + ), + marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell), ), pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_cutlass_stress', + test_desc='gpt_oss_120b_triton_stress', request_count=60000, accuracy_threshold=0.42, cancellation_rate=10, cancellation_delay=0.5), - marks=(pytest.mark.skip_less_device(4), skip_pre_hopper)), + marks=(pytest.mark.skip_less_device(4), skip_no_hopper)), pytest.param(TestConfig(model_path='Qwen3.5-4B-FP8', test_desc='qwen3_5_4b_fp8_stress', request_count=3000, diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 63d144aeac69..373677cf3880 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -659,7 +659,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torc accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False] diff --git a/tests/integration/test_lists/qa/llm_function_rtx6k.txt b/tests/integration/test_lists/qa/llm_function_rtx6k.txt index 25e7eabd5da7..43ec270512d8 100644 --- a/tests/integration/test_lists/qa/llm_function_rtx6k.txt +++ b/tests/integration/test_lists/qa/llm_function_rtx6k.txt @@ -123,7 +123,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] diff --git a/tests/integration/test_lists/qa/llm_function_stress.txt b/tests/integration/test_lists/qa/llm_function_stress.txt index fa8a710359f8..1bf7c4f3f77b 100644 --- a/tests/integration/test_lists/qa/llm_function_stress.txt +++ b/tests/integration/test_lists/qa/llm_function_stress.txt @@ -1,8 +1,9 @@ disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_mtp_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_trtllm_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_triton_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_trtllm_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_triton_stress] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_5_4b_fp8_stress] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_fp8_8gpus accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_nvfp4_4gpus diff --git a/tests/integration/test_lists/test-db/l0_l40s.yml b/tests/integration/test_lists/test-db/l0_l40s.yml index 9c72f9dccb86..b55e66129386 100644 --- a/tests/integration/test_lists/test-db/l0_l40s.yml +++ b/tests/integration/test_lists/test-db/l0_l40s.yml @@ -60,7 +60,6 @@ l0_l40s: - examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp16] - examples/test_llama.py::test_llm_llama_v3_dora_1gpu[commonsense-llama-v3-8b-dora-r32-llama-v3-8b-hf-base_fp16] - examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b] - - examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp16] - examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] - llmapi/test_llm_examples.py::test_llmapi_quickstart - examples/test_visual_gen.py::test_visual_gen_quickstart diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 8d20bc229c5a..29a3ef7e4964 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -114,7 +114,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard_sa SKIP (ht accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype SKIP (https://nvbugs/5520319) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6162122) -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] SKIP (https://nvbugs/5775326) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) @@ -198,7 +197,6 @@ examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bf examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (https://nvbugs/4961624) examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) examples/test_nemotron_nas.py::test_nemotron_nas_summary_2gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) -examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp16] SKIP (https://nvbugs/5612313) examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (https://nvbugs/5447530) examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/5612502) examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin] SKIP (https://nvbugs/5174573) @@ -227,8 +225,6 @@ full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-p full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) full:GH200/unittest/trt/model_api/test_model_quantization.py SKIP (https://nvbugs/4979955) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] SKIP (https://nvbugs/6222480) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] SKIP (https://nvbugs/6222480) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696) From a1d6a3730f64a55a3269d9afa20881b207816a39 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:37:27 +0800 Subject: [PATCH 2/2] update stress tests Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/defs/.test_durations | 1591 +++++++++++++---------- tests/integration/test_lists/waives.txt | 2 + 2 files changed, 889 insertions(+), 704 deletions(-) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index f01220bcc5d7..9b2e9851b370 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -1,706 +1,889 @@ { - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 229.88366167014465, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 1291.2246230191085, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 226.58150382409804, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 230.04569808510132, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 153.0148511910811, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 151.06352188810706, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 157.06875282898545, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 153.20611042692326, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 71.2399792142678, - "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False]": 286.7775873204227537, - "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True]": 286.6778334858827293, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 781.7928658421151, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 270.3750694899354, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2]": 195.4896494857967, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4]": 205.93911361903884, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2]": 188.56422709790058, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4]": 199.29050170327537, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2]": 127.93316596397199, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4]": 129.7617962991353, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2]": 127.73340241820551, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4]": 129.19083517300896, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 329.7597716320306, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 232.4293970640283, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 198.69031671597622, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 199.68801344232634, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False]": 161.8193401999306, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 107.20401050220244, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 163.33450849773362, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 137.72523731505498, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 209.07444706000388, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 149.8242256443482, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram": 286.3013918437063694, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2]": 199.44922504294664, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 173.89489603298716, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2]": 172.84839022206143, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2]": 128.10282056825235, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 121.90447079204023, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2]": 117.0786016730126, - "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[False]": 64428.639228201006, - "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[True]": 572.5455802679062, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False]": 472.62511800276116, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True]": 273.7770717362873, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 56.07656032079831, - "accuracy/test_llm_api.py::TestLlama3_1_8B::test_fp8_rowwise": 361.5573864541948, - "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph": 95.2069768682122, - "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 92.18154831900029, - "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 92.545280149206519, - "accuracy/test_llm_api.py::TestLlama3_2_1B::test_auto_dtype": 48.46169294399442, - "accuracy/test_llm_api.py::TestLlama3_2_1B::test_fp8_rowwise": 46.47006619500462, - "accuracy/test_llm_api.py::TestMixtral8x7B::test_tp2": 226.48782553203637, - "accuracy/test_llm_api.py::TestQwen2_5_0_5BInstruct::test_fp8": 93.85909735393943, - "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_auto_dtype": 173.21916160301771, - "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_fp8": 176.52263558004051, - "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_weight_only": 137.54228180646896, - "accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8": 223.2249169460265, - "accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8_kvcache": 293.131719612516463, - "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_auto_dtype": 146.72803921002196, - "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_fp8": 149.11297382606426, - "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_weight_only": 594.9357111975551, - "accuracy/test_llm_api_pytorch.py::TestBielik11BInstruct::test_auto_dtype": 3600.001755183912, - "accuracy/test_llm_api_pytorch.py::TestBielik11BInstruct::test_fp8": 3600.0018868579646, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput]": 7291.4317992888391, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale_chunked_prefill[latency]": 7291.9177003782242537, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale_chunked_prefill[throughput]": 7291.818120779618621, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency]": 7292.3961801091209054, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp]": 7292.2807201944456277, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen]": 7292.6643901705286214, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen_adp_lmtp]": 7292.5567501965034455, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput]": 7292.690330147743225, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4]": 7292.468819195404649, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp8]": 7292.699600299820304, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_chunked_prefill[latency]": 7292.5356449983082712, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_chunked_prefill[throughput_tp4]": 7292.413349622860551, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_corner_case": 7292.248919965699315, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 591.2785023800097, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 107.58471493399702, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 143.84012729604729, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 295.3527018489549, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 306.84709841990843, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 220.57452515885234, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 165.08514453098178, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 202.22269394202158, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 113.82226522010751, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 205.7252635700861, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True]": 213.78996226208983, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=0-moe_backend=WIDEEP]": 292.7267158059985377, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP]": 292.5756296711042523, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[-attention_dp-cuda_graph-overlap_scheduler-torch_compile=False]": 326.1317654890008, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 647.6109309499152, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 184.20976317999884, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 226.01353620411828, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 506.1045090719126, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-attention_dp-cuda_graph-overlap_scheduler-torch_compile=False]": 336.02580665098503, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 413.903915906325, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 143.841789112892, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 246.64391099987552, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 202.37037238897756, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus_static_eplb": 3600.0020909640007, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 3600.441698686045129, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 3600.591623628977686, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 3600.642337311059237, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 3600.572194146050606, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=0]": 3600.003432472993, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=2]": 3600.21731633093441, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0]": 3600.002661294944, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2]": 3600.0022540579666, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 89.92349556891713, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 85.24235329206567, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 252.70569713797886, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 90.21807348495349, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 175.661773331929, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 360.0003233290044590831, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=False-moe_backend=WIDEEP]": 360.9275938109494746, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=WIDEEP]": 360.0002855450729839504, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=0.75-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 360.0003064870252273977, - "accuracy/test_llm_api_pytorch.py::TestEXAONE4::test_auto_dtype": 3600.0004039629711769521, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 360.00032637204276397824, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-fp8]": 360.0003586999955587089, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto]": 360.6586053780047223, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-auto]": 360.0003633099840953946, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-fp8]": 360.00036422599805518985, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 360.00032637204276397824, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-fp8]": 360.0003586999955587089, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-triton-auto]": 360.6586053780047223, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-auto]": 360.0003633099840953946, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-fp8]": 360.00036422599805518985, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-cutlass-auto]": 360.0003378289984539151, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-triton-auto]": 360.9436147869564593, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-trtllm-auto]": 360.0003398499684408307, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-trtllm-fp8]": 360.0002922280109487474, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-cutlass-auto]": 360.0003666180418804288, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-triton-auto]": 360.9300670439261012, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-trtllm-auto]": 360.0002812399761751294, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-trtllm-fp8]": 360.0008064290159381926, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-cutlass-auto]": 360.0003697940264828503, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-triton-auto]": 360.8670774899655953, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-trtllm-auto]": 360.00040231598541140556, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-trtllm-fp8]": 360.0003254589391872287, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-cutlass-auto]": 745.8583740849863, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 745.9345730679342523, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-auto]": 745.0004936959594488144, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-fp8]": 745.00031642295653000474, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-cutlass-auto]": 658.1757711600512, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 745.9436021829606034, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 745.0004371170070953667, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-fp8]": 745.0004142870311625302, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-auto]": 676.3980704760179, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 745.0292645250447094, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-auto]": 745.0003769229515455663, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-fp8]": 677.000331886054482311, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-cutlass-auto]": 745.8583740849863, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 745.9345730679342523, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-auto]": 745.0004936959594488144, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-fp8]": 745.00031642295653000474, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutlass-auto]": 658.1757711600512, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 745.9436021829606034, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 745.0004371170070953667, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-fp8]": 745.0004142870311625302, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto]": 676.3980704760179, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-triton-auto]": 745.0292645250447094, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 745.0003769229515455663, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-fp8]": 677.000331886054482311, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto]": 643.3513998010312, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto]": 764.9216735750087537, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-auto]": 764.0002969659981317818, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-fp8]": 764.0008383550448343158, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 764.8800516680348665, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8]": 764.00035094103077426553, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 14400, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler]": 14400, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 88.30407958402066, - "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype": 892.3242024959764, - "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 451.8630696780165, - "accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype": 86.96273394301534, - "accuracy/test_llm_api_pytorch.py::TestKimiK2::test_fp8_blockscale[latency]": 0.19557904999237508, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 56.31924073398113, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False]": 115.23303954396397, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True]": 115.0261728389305, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False]": 98.29266697796993, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=True]": 96.56704166403506, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False]": 98.47511212801328, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True]": 96.76115251897136, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 307.12596721109, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 443.91388061689213, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 191.10617867391557, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 166.85348949534819, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[attn_backend=FLASHINFER]": 167.15153613401344, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[attn_backend=TRTLLM]": 90.12104846700095, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 1112.0988524899585, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 979.2759481471148, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 237.24446990108117, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 226.39608797896653, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 174.38962662010454, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 313.69273760309443, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 409.8932851999998, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 344.8807112099603, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 103.82129427790642, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 164.91815144987777, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False]": 124.62386814301135, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True]": 121.9821372089209, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False]": 77.16783500701422, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=True]": 75.89485901995795, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False]": 77.46995012206025, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True]": 75.93042165302904, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_llm_sampler": 61.986203632026445, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 44.978786278981715, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 46.60325947904494, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[llguidance]": 52.44408063602168, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 52.84089746000245, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False]": 36.739327706047334, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 67.41741452802671, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 36.758924948982894, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 32.73584783205297, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[llguidance]": 31.598825128981844, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[xgrammar]": 31.997449714050163, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_ngram": 404.38593446696177, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_1B::test_auto_dtype": 49.4644276680192, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_1B::test_fp8_prequantized": 40.83171262202086, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype": 60.00025596999330446124, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_fp8_prequantized": 67.63213626696961, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False]": 3770.7388199700508, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False]": 4010.9139676889754, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False]": 2616.262340236979, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 3600.00026297004660591483, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4-cuda_graph=False]": 7200.00030252401484176517, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4ep2-cuda_graph=True]": 7200.0002791329752653837, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4ep4-cuda_graph=True]": 7200.00028608099091798067, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False]": 7200.0002498350222595036, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True]": 7200.00023661594605073333, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True]": 7200.00023339298786595464, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_chunked_prefill[attn_backend=FLASHINFER]": 7200.0002561529981903732, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_chunked_prefill[attn_backend=TRTLLM]": 7200.0052206520340405405, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4-cuda_graph=True]": 7200.2161163400160149, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4ep2-cuda_graph=True]": 7200.2132116109714843, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4ep4-cuda_graph=True]": 7200.3660773960873485, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8-cuda_graph=True]": 7200.00025232898769900203, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8ep4-cuda_graph=True]": 7200.00025386200286448, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8ep8-cuda_graph=True]": 7200.00023917207727208734, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_chunked_prefill[tp8ep8-cuda_graph=False]": 7200.5301868109382, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_chunked_prefill[tp8ep8-cuda_graph=True]": 72600.1488124400494, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4-cuda_graph=False]": 3600.0010551271309959702, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4ep2-cuda_graph=True]": 3600.0009890546519891359, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4ep4-cuda_graph=True]": 3600.000870058874017559, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-cuda_graph=False]": 3600.0022709049517, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep4-cuda_graph=True]": 3600.0008703919593, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep8-cuda_graph=True]": 3600.001674739062, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4[tp4-cuda_graph=True]": 3600.0003222679952159524, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4[tp8ep8-cuda_graph=True]": 3600.0004189839819446206, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4_chunked_prefill[tp4ep4-cuda_graph=True]": 3600.0009446179610677063, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp4-cuda_graph=True]": 3600.0600651470012963, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp8ep8-cuda_graph=True]": 3600.0020443379763, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8_chunked_prefill[tp4ep4-cuda_graph=True]": 3600.0016280449927, - "accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_auto_dtype": 300.0017418859643, - "accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8": 300.001715709921, - "accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized": 48.064747432945296, - "accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype": 96.73990149900783, - "accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype": 3600.0012007239857, - "accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype_tp2": 3600.001961252012, - "accuracy/test_llm_api_pytorch.py::TestMistralSmall24B::test_auto_dtype": 3600.0003251209855079651, - "accuracy/test_llm_api_pytorch.py::TestMistralSmall24B::test_fp8": 196.57955891895108, - "accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2": 3600.0022730380297, - "accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2": 3600.7994798690197058, - "accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8": 3600.5389363930444, - "accuracy/test_llm_api_pytorch.py::TestPhi4::test_auto_dtype": 3600.0017098310054, - "accuracy/test_llm_api_pytorch.py::TestPhi4::test_fp8": 3600.0018840720295, - "accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype": 3600.000989172084, - "accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype_long_rope": 3600.001809718029, - "accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype": 3600.0018334789784, - "accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype": 49.38216367200948, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency]": 7576.4847942629713, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency]": 75934.1885519769276, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass]": 7570.00041150598553940654, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm]": 7570.0005082660354673862, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp]": 7570.0003929820377379656, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_trtllm_eagle3]": 7570.000994006055407226, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 3240.3035402488895, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 3240.3568000392988324, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False]": 3240.326488903677091, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True]": 3240.3251879825256765, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False]": 3240.3220480284653604, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]": 3240.12236930197104812, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM]": 3240.3637070371955633, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]": 3240.5122077039559372, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRITON]": 3240.46138638898264617, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]": 3240.4374056719825603, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]": 3240.4651110970880836, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]": 3240.4263977239606902, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.5-fp8kv=False]": 1202.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.9-fp8kv=False]": 1127.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS]": 1091.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-TRTLLM]": 1091.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False]": 313.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=True]": 313.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_6_27B::test_fp8": 569.04, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency-torch_compile=False]": 149.19146074401215, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 104.32479889906244, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 240.30756398336961865, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[mxfp8-latency]": 240.27633493300527334, - "cpp/test_e2e.py::test_benchmarks[bart-90]": 271.95234084688127, - "cpp/test_e2e.py::test_benchmarks[gpt-80]": 1376.0404928650241, - "cpp/test_e2e.py::test_benchmarks[t5-90]": 523.07, - "cpp/test_e2e.py::test_model[-bart-90]": 391.84748707409017, - "cpp/test_e2e.py::test_model[-eagle-86]": 850.5158995762467, - "cpp/test_e2e.py::test_model[-enc_dec_language_adapter-90]": 416.06, - "cpp/test_e2e.py::test_model[-gpt-80]": 1568.98, - "cpp/test_e2e.py::test_model[-gpt_executor-80]": 1495.14, - "cpp/test_e2e.py::test_model[-gpt_tests-80]": 1206.79, - "cpp/test_e2e.py::test_model[-mamba-86]": 893.8684413917363, - "cpp/test_e2e.py::test_model[-medusa-86]": 577.0913726426661, - "cpp/test_e2e.py::test_model[-redrafter-86]": 356.56682327389717, - "cpp/test_e2e.py::test_model[-t5-90]": 170.26, - "cpp/test_e2e.py::test_model[fp8-llama-90]": 385.98, - "cpp/test_unit_tests.py::test_unit_tests[batch_manager-80]": 1005.24, - "cpp/test_unit_tests.py::test_unit_tests[common-80]": 38.98, - "cpp/test_unit_tests.py::test_unit_tests[common-90]": 25.06, - "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 425.16, - "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 2009.96, - "cpp/test_unit_tests.py::test_unit_tests[kernels-90]": 1333.18, - "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 2209.11, - "cpp/test_unit_tests.py::test_unit_tests[layers-90]": 1627.07, - "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 1671.42, - "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 6.76, - "cpp/test_unit_tests.py::test_unit_tests[thop-90]": 4.16, - "cpp/test_unit_tests.py::test_unit_tests[utils-80]": 8.53, - "cpp/test_unit_tests.py::test_unit_tests[utils-90]": 5.28, - "disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0]": 54.26733888499439, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0]": 64.01942083099857, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0]": 67.32925470802002, - "disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0]": 50.57874152995646, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8]": 105.12023228011094, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one[DeepSeek-V3-Lite-fp8]": 98.13158084987663, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one_mtp[DeepSeek-V3-Lite-fp8]": 104.78005758393556, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_mpi[DeepSeek-V3-Lite-fp8]": 680.20395052596, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 102.03138376423158, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 90.40784636512399, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8]": 124.17078560194932, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 102.67233599093743, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 224.28071974776685, - "disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0]": 52.78952780482359, - "disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0]": 73.48997121001594, - "disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0]": 67.3897166326642, - "disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0]": 54.22262764698826, - "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 98.97588296607137, - "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0]": 67.9668476767838, - "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0]": 82.28277984517626, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8]": 238.76137515995651, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 78.98068026197143, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 77.51831256924197, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 99.81417108187452, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-True-DeepSeek-V3-Lite-fp8/fp8]": 67.32832619687542, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 48.16434509307146, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 36.88020430901088, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 46.302398771978915, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 38.81214914191514, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 31.12580855889246, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 29.8858498937916, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 55.79476427496411, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 55.29072010796517, - "disaggregated/test_workers.py::test_workers_conditional_disaggregation[TinyLlama-1.1B-Chat-v1.0]": 99.67464494984597, - "disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0]": 113.69421282503754, - "disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0]": 195.86076739802957, - "disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0]": 76.3156570668798, - "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:1-pp:1-float32-BertModel-bert/bert-base-uncased]": 111.17977902293205, - "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:1-pp:1-float32-RobertaModel-bert/roberta-base]": 115.47540166974068, - "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-BertForQuestionAnswering-bert/bert-base-cased-squad2]": 64.32757595699513, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 55.807815481035504, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-BertModel-bert/bert-base-uncased]": 60.69481396203628, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertForQuestionAnswering-bert/bert-base-cased-squad2]": 73.41519981494639, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 73.00019321503350511193, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 121.36917966976762, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaForQuestionAnswering-bert/roberta-base-squad2]": 74.99331583303865, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaForSequenceClassification-bert/twitter-roberta-base-emotion]": 73.000189481012057513, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaModel-bert/roberta-base]": 122.99858937039971, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha_fp32_acc-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 55.26069303997792, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-chat-disable_weight_only]": 3640.0045928549953, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-chat-enable_weight_only]": 3235.5094731519348, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-disable_weight_only]": 3273.7859199331142, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-enable_weight_only]": 3602.5807401749771, - "examples/test_commandr.py::test_llm_commandr_v01_single_gpu_summary[disable_weight_only]": 1856.5704392530024, - "examples/test_commandr.py::test_llm_commandr_v01_single_gpu_summary[enable_weight_only]": 654.6126579149859, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 244.9744301661849, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_logits-draft_len_8-float16-bs1]": 398.78198734589387, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 241.73137632384896, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-llama_v2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 1099.3959164519329, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 249.52418848499656, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 257.3995385244489, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_tokens-draft_len_8-float16-bs1]": 397.20915103994776, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-llama_v2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 582.6556157508167, - "examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle1]": 254.24225717037916, - "examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle2]": 246.98607586231083, - "examples/test_eagle.py::test_llm_eagle_1gpu_modelopt_ckpt[llama3.1-eagle-8b-hf_v0.5-float16-bs8]": 489.4540088879876, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-bfloat16-enable_gemm_plugin-enable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 232.2934926636517, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-bfloat16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 281.1201816312969, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 2954.5586752621457, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:2-disable_fp8]": 276.10329104214907, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 233.94542215764523, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 283.8865255280398, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-disable_gemm_plugin-disable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 220.32321695238352, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-disable_gemm_plugin-enable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 218.02495155483484, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 96.25796012202045, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 568.2032693652436, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:2-nb:1-enable_fp8]": 354.8090338760521, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-mbart-large-50-many-to-one-mmt-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 160.50506488600513, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-mbart-large-50-many-to-one-mmt-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:2-nb:1-enable_fp8]": 300.20793505245819688, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-disable_gemm_plugin-disable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 218.7171499580145, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 217.96836187317967, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 498.1236152825877, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:2-disable_fp8]": 205.83720442652702, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:1-nb:1-enable_fp8]": 295.6078515049885, - "examples/test_enc_dec.py::test_llm_enc_dec_general[no_compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 252.0442810030654, - "examples/test_enc_dec.py::test_llm_enc_dec_general[no_compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:1-nb:1-disable_fp8]": 95.48429084802046, - "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-disable_fp8]": 422.4394793640822, - "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-enable_fp8]": 1074.875556848012, - "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_3.0_7.8b_instruct-float16-nb:1]": 3002.19654186069965, - "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_3.0_7.8b_instruct-float16-nb:4]": 3074.5337073504925, - "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_deep_2.4b-float16-nb:4]": 243.4259528592229, - "examples/test_exaone.py::test_llm_exaone_1gpu[enable_weight_only-exaone_deep_2.4b-float16-nb:1]": 212.50885355699575, - "examples/test_exaone.py::test_llm_exaone_2gpu[exaone_3.0_7.8b_instruct-float16-nb:1]": 7155.35844087804435, - "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-27b-it]": 317.7816583644599, - "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-9b-it]": 317.7816583644599, - "examples/test_gemma.py::test_llm_gemma_1gpu_summary_vswa[gemma-3-1b-it-other-bfloat16-8]": 195.3050664511975, - "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[non_streaming-use_cpp_session-enable_gemm_plugin]": 114.20040711760521, - "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[non_streaming-use_py_session-disable_gemm_plugin]": 105.38906556204893, - "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[streaming-use_cpp_session-enable_gemm_plugin]": 113.51056583970785, - "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[non_streaming-use_cpp_session]": 194.89961875230074, - "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[non_streaming-use_py_session]": 200.52475621178746, - "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[streaming-use_cpp_session]": 195.00627667084336, - "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[non_streaming-use_cpp_session]": 194.90547297894955, - "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[non_streaming-use_py_session]": 194.89357279613614, - "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[streaming-use_cpp_session]": 194.72326660901308, - "examples/test_gpt.py::test_llm_gpt2_next_prompt_tuning[use_cpp_session-tp1]": 460.1370678450912, - "examples/test_gpt.py::test_llm_gpt2_next_prompt_tuning[use_py_session-tp1]": 403.39630596572533, - "examples/test_gpt.py::test_llm_minitron_fp8_with_pseudo_loras[4b]": 259.4826051471755, - "examples/test_granite.py::test_granite_bf16_lora[granite-3.0-1b-a400m-instruct]": 140.08338637300767, - "examples/test_granite.py::test_granite_bf16_lora[granite-3.0-2b-instruct]": 146.46410073013976, - "examples/test_granite.py::test_llm_granite[granite-3.0-1b-a400m-instruct-bfloat16]": 141.05149138718843, - "examples/test_granite.py::test_llm_granite[granite-3.0-2b-instruct-bfloat16]": 1055.801738537848, - "examples/test_internlm.py::test_llm_internlm2_7b_1node_1gpu[bfloat16-enable_context_fmha-enable_gemm_plugin-enable_attention_plugin-nb:2]": 2690.7412294782698, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.1-8b]": 160.33107751235366, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.2-1b]": 117.10959041584283, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.2-3b]": 215.69512976403348, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-v2-7b-hf]": 376.96383721905295, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-v3-8b-instruct-hf]": 317.46144750900567, - "examples/test_llama.py::test_llm_llama_1gpu[llama-3.1-8b-instruct-hf-fp8-enable_fp8-float16-summarization-nb:1]": 853.2910006027669, - "examples/test_llama.py::test_llm_llama_1gpu_batched_beam_search[llama-7b]": 182.20104870200157, - "examples/test_llama.py::test_llm_llama_1gpu_fp8_kv_cache[llama-v2-7b-hf-bfloat16]": 313.6555140609853, - "examples/test_llama.py::test_llm_llama_long_alpaca_8gpu_summary[pg64317-tp8pp1-nb:1]": 441.2820012559532, - "examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b]": 167.92376559507102, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_awq]": 420.1779588930076, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp16]": 895.7611340929288, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp8]": 314.3205590210273, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_int8_wo]": 329.1954380639363, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_sq_ootb]": 216.2645359209855, - "examples/test_llama.py::test_llm_llama_v3_1_1node_single_gpu[llama-3.2-1b-disable_fp8]": 382.12588274572045, - "examples/test_llama.py::test_llm_llama_v3_dora_1gpu[commonsense-llama-v3-8b-dora-r32-llama-v3-8b-hf-base_fp16]": 517.2770831151865, - "examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0]": 64.8964971601963, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-1.4b-float16-enable_gemm_plugin]": 181.03255189501215, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-130m-float16-disable_gemm_plugin]": 136.8141469657421, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-130m-float16-enable_gemm_plugin]": 112.04011878371239, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-2.8b-float16-disable_gemm_plugin]": 336.9864642599714, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-370m-float16-enable_gemm_plugin]": 87.74965502799023, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-790m-float16-disable_gemm_plugin]": 148.56857219303492, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-codestral-7B-v0.1-float16-disable_gemm_plugin]": 405.1586506664753, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-codestral-7B-v0.1-float16-enable_gemm_plugin]": 384.9690850973129, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-1.3b-float16-enable_gemm_plugin]": 126.15442710905336, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-130m-float16-disable_gemm_plugin]": 129.8332964628935, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-130m-float16-enable_gemm_plugin]": 118.37521690130234, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-2.7b-float16-disable_gemm_plugin]": 212.40601160994265, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-370m-float16-enable_gemm_plugin]": 75.9140890170238, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-780m-float16-disable_gemm_plugin]": 114.81139776791679, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs1]": 217.8724013082683, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs8]": 460.24718615040183, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_py_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs1]": 129.02341048512608, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_py_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs8]": 524.8282293006778, - "examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1]": 383.3182801879011, - "examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_py_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1]": 293.03625723719597, - "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-chunked_summarization_long]": 429.02448211982846, - "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-summarization]": 603.6547773182392, - "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-summarization_long]": 391.7267559207976, - "examples/test_mixtral.py::test_llm_mixtral_moe_plugin_lora_4gpus[Mixtral-8x7B-v0.1-chinese-mixtral-lora]": 3600.00015190104022622108, - "examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-cnn_dailymail-Qwen2-VL-7B-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False]": 332.0248579243198, - "examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-scienceqa-Llama-3.2-11B-Vision-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 411.7690062429756, - "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 385.0684349639341, - "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:2-bfloat16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3-vision-128k-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 282.8564471802674, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3-vision-128k-instruct-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 90.19939221709501, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3.5-vision-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 275.5947739640251, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-4-multimodal-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 644.3520091949031, - "examples/test_multimodal.py::test_llm_multimodal_general[Qwen2-VL-7B-Instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:4]": 656.4784073680639, - "examples/test_multimodal.py::test_llm_multimodal_general[deplot-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 249.87254932150245, - "examples/test_multimodal.py::test_llm_multimodal_general[deplot-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 100.08572303387336, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1]": 492.22362083010375, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1]": 333.81485258904286, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 459.2980541479774, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 176.8547668098472, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-onevision-qwen2-7b-ov-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 533.2010767317843, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-onevision-qwen2-7b-ov-hf-video-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 200.52463799191173, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-v1.6-mistral-7b-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 274.8723033480346, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-v1.6-mistral-7b-hf-vision-trtllm-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 306.38610201328993, - "examples/test_multimodal.py::test_llm_multimodal_general[nougat-base-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 226.57772368215956, - "examples/test_multimodal.py::test_llm_multimodal_general[nougat-base-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 198.46779074892402, - "examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 260.0002240890171378851, - "examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 260.00014175008982419968, - "examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B]": 335.41048416192643, - "examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs1]": 200.00026965001598000526, - "examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2]": 196.1214354224503, - "examples/test_ngram.py::test_llm_ngram_1gpu[streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs1]": 200.0001331439707428217, - "examples/test_ngram.py::test_llm_ngram_1gpu[streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2]": 195.90045699477196, - "examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp8]": 220.0002483620774000883, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3-mini-128k-instruct-fp8-float16]": 360.0002299160696566105, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3.5-MoE-instruct-fp8-bfloat16]": 360.0001645770389586687, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3.5-mini-instruct-fp8-float16]": 360.0001653869403526187, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-4-mini-instruct-fp8-bfloat16]": 360.00032463017851114273, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[phi-2-fp8-bfloat16]": 154.69453522900585, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_0.5b_instruct]": 99.85199454193935, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_1.5b_instruct]": 193.38715927954763, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2_0.5b_instruct]": 128.65416727401316, - "examples/test_qwen.py::test_llm_qwen1_5_7b_single_gpu_lora[qwen1.5_7b_chat-Qwen1.5-7B-Chat-750Mb-lora]": 338.59182655182667, - "examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct]": 423.2417808100581, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.0001445889938622713, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-fp8-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00012858898844569921, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-int4_awq-float16-enable_attn_plugin-enable_gemm_plugin]": 648.7579195387661, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-int8_sq-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00033479504054412246, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-flax-no_paged_cache-disable_quant-float16-enable_attn_plugin-disable_gemm_plugin]": 286.15992603078485, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-no_paged_cache-disable_quant-float16-disable_attn_plugin-enable_gemm_plugin]": 680.0001984360278584063, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-no_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00013055797899141908, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.0001333290128968656, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_cpp_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb5-bs8]": 386.68252966180444, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_cpp_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb8-bs8]": 204.7229775050655, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_py_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb5-bs8]": 411.88197461143136, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_py_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb8-bs8]": 429.239758990705, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 327.95307156071067, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_cpp_runtime]": 249.98457504063845, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 225.60136043280363, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-int4-float16-nb:1-use_cpp_runtime]": 114.39965145103633, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-int8-float16-nb:1-use_cpp_runtime]": 109.14321990706958, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-enable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 110.00021485507022589445, - "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_logging": 458.4059731345624, - "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_default": 40.70597969903611, - "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_tensorrt": 99.26517963293009, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-codellama/CodeLlama-7b-Instruct-hf]": 383.1450063039083, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-llama-models/llama-7b-hf]": 101.96415167767555, - "llmapi/test_llm_examples.py::test_llmapi_server_example": 107.44753806525841, - "perf/test_perf.py::test_perf[bert_base-cpp-ootb-float16-bs:32-input_len:32]": 111.37450777366757, - "perf/test_perf.py::test_perf[bert_base-cpp-plugin-float16-bs:32-input_len:32]": 95.00738414749503, - "perf/test_perf.py::test_perf[roberta_base-cpp-plugin-float16-bs:32-input_len:128+512]": 140.2516261599958, - "test_e2e.py::test_build_time_benchmark_sanity": 165.71592589840293, - "test_e2e.py::test_gpt3_175b_1layers_build_only": 131.34366285055876, - "test_e2e.py::test_llama_e2e[use_cpp_session-remove_input_padding-]": 90.12235352798598, - "test_e2e.py::test_llama_e2e[use_py_session--]": 92.68217968731187, - "test_e2e.py::test_llama_e2e[use_py_session-remove_input_padding-]": 93.41917416104116, - "llmapi/test_llm_examples.py::test_llmapi_chat_example": 105.19824166595936, - "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 73.23708964884281, - "llmapi/test_llm_examples.py::test_llmapi_example_inference": 66.82718145102262, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 65.93082024902105, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 67.49109892174602, - "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 72.87169548124075, - "llmapi/test_llm_e2e.py::test_llmapi_exit": 32.64902823418379, - "llmapi/test_llm_e2e.py::test_llmapi_load_ckpt_from_convert_command": 180.59318951144814, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command_with_lora[llama-llama-models-v2/llama-v2-7b-hf]": 225.2778383679688, - "llmapi/test_llm_examples.py::test_llmapi_quickstart": 66.53727849572897, - "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 110.45052940770984, - "test_e2e.py::test_mistral_e2e[use_cpp_session-remove_input_padding--]": 192.74050169810653, - "test_e2e.py::test_mistral_e2e[use_py_session---]": 160.08483010903, - "test_e2e.py::test_mistral_e2e[use_py_session-remove_input_padding--]": 157.39577213302255, - "test_e2e.py::test_mistral_large_hidden_vocab_size": 81.36711680702865, - "test_e2e.py::test_openai_chat_example": 876.1966922096908, - "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 55.12449237401597, - "test_e2e.py::test_openai_chat_harmony": 1162.7252594940364, - "test_e2e.py::test_openai_chat_multimodal_example": 215.8254322744906, - "test_e2e.py::test_openai_misc_example": 256.0453990884125, - "test_e2e.py::test_openai_multi_chat_example": 684.1194127409253, - "test_e2e.py::test_ptp_quickstart": 45.67732956283726, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 37.10870039300062, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 81.43792725296225, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 56.05445321695879, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.2-11B-BF16-llama-3.2-models/Llama-3.2-11B-Vision]": 458.2611172522884, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 1198.8076391071081, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Maverick-17B-128E-Instruct-FP8-llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8-False]": 7750.113898515003, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Maverick-17B-128E-Instruct-FP8-llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8-True]": 7404.8092977448832, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Scout-17B-16E-Instruct-FP4-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4-True]": 3600.00027390988543629646, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Scout-17B-16E-Instruct-FP8-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8-True]": 2183.7999707763083, - "test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]": 8346.51794013707, - "test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8]": 11215.495469792979, - "test_e2e.py::test_ptp_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B]": 109.26379436196294, - "test_e2e.py::test_ptp_quickstart_advanced_mixed_precision": 80.88908524392173, - "test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]": 99.42739840806462, - "test_e2e.py::test_ptp_quickstart_advanced_ngram[Llama-3.1-8B-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct]": 71.96910276100971, - "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 21.019993914989755, - "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 18.753523574909195, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[llava-v1.6-mistral-7b-llava-v1.6-mistral-7b-hf-0.8-image]": 354.18758003995754, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[phi4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct-0.8-image]": 313.79139575595036, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[qwen2.5-vl-7b-instruct-Qwen2.5-VL-7B-Instruct-0.8-image]": 47.84053734713234, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[qwen2.5-vl-7b-instruct-Qwen2.5-VL-7B-Instruct-0.8-video]": 58.646747548133135, - "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[Phi-4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct]": 0.00018897396512329578, - "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[gemma-3-27b-it-gemma/gemma-3-27b-it]": 81.0151858178433, - "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[mistral-small-3.1-24b-instruct-Mistral-Small-3.1-24B-Instruct-2503]": 60.082822071621194, - "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[audio]": 89.50578387826681, - "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[image]": 66.199569062097, - "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[image_audio]": 62.389084175927565, - "test_e2e.py::test_ptp_scaffolding[DeepSeek-R1-Distill-Qwen-7B-DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B]": 7200.0001350759994238615, - "test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B]": 137.7278483910486, - "test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]": 12134.278186964104, - "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 109.25386995915323, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 163.86223009089008, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 114.50899445591494, - "test_e2e.py::test_trtllm_bench_iteration_log[TRT-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 285.3362849447876, - "test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 310.9046222809702, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 114.17938271397725, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 115.74023819994181, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 276.64185731019825, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 252.97791706770658, - "test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b]": 482.31134233786725, - "test_e2e.py::test_trtllm_benchmark_serving[llama-3.1-model/Meta-Llama-3.1-8B]": 80.71726501686499, - "test_e2e.py::test_trtllm_multimodal_benchmark_serving": 360.00022564898245036602, - "test_e2e.py::test_trtllm_serve_example": 200.09309104084969, - "test_e2e.py::test_trtllm_serve_multimodal_example": 130.2214687075466, - "test_unittests.py::test_unittests_v2[unittest/_torch/attention/test_attention_mla.py]": 26.32902159006335, - "test_unittests.py::test_unittests_v2[unittest/_torch/attention]": 588.56, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu]": 539.3006387590431, - "test_unittests.py::test_unittests_v2[unittest/_torch/compilation]": 31.94, - "test_unittests.py::test_unittests_v2[unittest/_torch/debugger]": 36.69, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor]": 170.86, - "test_unittests.py::test_unittests_v2[unittest/_torch/flashinfer/test_trtllm_flashinfer_symbol_collision.py]": 1004.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/misc]": 600.5, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_llama\"]": 718.749935634085, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mixtral\"]": 208.1838396479725, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mllama\"]": 749.5508671940188, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron\"]": 1952.3731448464096, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nas\"]": 498.8839871880482, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_out_of_tree\"]": 55.078535287990235, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen\"]": 551.1881373599754, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen_moe\"]": 401.2630233000382, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_vila\"]": 79.90315388399176, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules]": 158.5, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"]": 68.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"]": 1598.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"]": 22.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DENSEGEMM\"]": 149.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"TRTLLM\"]": 998.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling -k \"deepseek\"]": 393.0210295501165, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_external_embedding.py]": 30.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_find_num_image_tokens.py]": 240.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_fuse_input_embeds.py]": 60.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_mm_encoder_standalone.py]": 1800.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_multimodal_runtime.py]": 180.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_share_multiparams.py]": 30.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler]": 1020.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/speculative]": 1850.16, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel]": 1463.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/serial]": 18.96, - "test_unittests.py::test_unittests_v2[unittest/api_stability]": 33.137137457728386, - "test_unittests.py::test_unittests_v2[unittest/bindings]": 1119.2564616799355, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_build_cache.py]": 34.61376368254423, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_executor.py]": 378.7100401185453, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm.py -m \"not part0\"]": 1883.5484512336552, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm.py -m \"part0\"]": 1601.0243577323854, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"not (part0 or part1)\"]": 825.9972547292709, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"part0\"]": 163.72848848626018, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"part1\"]": 538.573951125145, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_perf_evaluator.py]": 118.36046380549669, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py::test_gemma3_1b_instruct_multi_lora]": 101.58543362899218, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py]": 539.5857984796166, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_quant.py]": 477.989566125907, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_utils.py]": 125.15857975929976, - "test_unittests.py::test_unittests_v2[unittest/test_model_runner_cpp.py]": 973.1355891097337, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_bert_attention.py]": 99.96196278184652, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition0\"]": 77.31474154582247, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition1\"]": 84.67568279313855, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition2\"]": 75.39135546097532, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition3\"]": 78.77339706313796, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"trtllm_gen\"]": 376.012343961047, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"xqa_generic\"]": 267.40264504775405, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_IFB.py]": 85.18935105204582, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_no_cache.py]": 49.3486054521054, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_sage_attention.py unittest/llmapi/test_llm_download.py unittest/llmapi/test_llm_kv_cache_events.py unittest/llmapi/test_mpi_session.py unittest/trt/model/redrafter unittest/trt/model/test_phi.py unittest/trt/model/test_unet.py unittest/trt/python_plugin unittest/tools unittest/utils unittest/others]": 940.7867036014795, - "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_fp4_gemm.py]": 302.49857676401734, - "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_moe.py]": 220.60184395778924, - "test_unittests.py::test_unittests_v2[unittest/trt/functional]": 778.6451135131065, - "test_unittests.py::test_unittests_v2[unittest/trt/model/eagle]": 212.3223411180079, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"other\"]": 125.12117889150977, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition0\"]": 300.0489609502256, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition1\"]": 265.11671224981546, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition2\"]": 357.6496359631419, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition3\"]": 371.381394200027, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt_e2e.py]": 537.5006402550498, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_llama.py]": 1494.1103300452232, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mamba.py]": 76.84791256207973, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mistral.py]": 366.95385985821486, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_nemotron_nas.py -k \"not fp8\"]": 1041.1297603696585, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_api_multi_gpu.py]": 27.33125525712967, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_level_api.py]": 33.818626184016466, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_quantization.py]": 493.8186915554106, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py]": 214.35422350093722, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_quant_matmul.py]": 100.81762219779193, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization]": 673.2582192085683 + "cpp/test_e2e.py::test_model[-eagle-86]": 467.31710943579674, + "cpp/test_e2e.py::test_model[-medusa-86]": 559.4042182117701, + "test_fmha.py::test_fmha": 1843.8030557027087, + "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 371.68835608288646, + "llmapi/test_llm_examples.py::test_llmapi_example_inference": 390.4545612037182, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 28.57412961218506, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 71.92784738913178, + "llmapi/test_llm_examples.py::test_llmapi_example_logits_processor": 28.32273621391505, + "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 46.27545684855431, + "llmapi/test_llm_examples.py::test_llmapi_quickstart": 29.23038684949279, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mha]]": 16.939650654792786, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mla]]": 16.761926997452974, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[NIXL-mha-ctx_fp16_gen_fp16]]": 12.963495340198278, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[UCX-mha-ctx_fp16_gen_fp16]]": 12.825301811099052, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 75.91953653842211, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 36.62238800525665, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 47.69775189831853, + "llmapi/test_llm_api_connector.py::test_connector_async_onboard[False]": 20.00915778055787, + "llmapi/test_llm_api_connector.py::test_connector_async_save[False]": 13.049194630235434, + "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[False]": 27.849122423678637, + "llmapi/test_llm_api_connector.py::test_connector_e2e_persistent_cache": 24.956512186676264, + "llmapi/test_llm_api_connector.py::test_connector_priorities": 15.430590681731701, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[attention_dp]": 5.342560388147831, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[host_offloading]": 12.192664381116629, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[True]": 17.182394333183765, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[True]": 11.133101604878902, + "llmapi/test_llm_api_connector.py::test_connector_simple[True]": 12.597462255507708, + "test_e2e.py::test_openai_completions_example[pytorch]": 256.56871559098363, + "test_e2e.py::test_openai_mmencoder_example": 128.8613565787673, + "test_e2e.py::test_openai_prometheus": 165.67273128777742, + "test_e2e.py::test_openai_responses_entrypoint": 100.25372431054711, + "test_e2e.py::test_trtllm_bench_invalid_token_pytorch[TinyLlama-1.1B-Chat-v1.0-TinyLlama-1.1B-Chat-v1.0]": 70.06262697279453, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 255.5959302908741, + "test_e2e.py::test_trtllm_serve_multimodal_example": 185.39177925512195, + "thirdparty/test_git_modules.py::test_gitmodules": 0.0012150108814239502, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_async_transfer_manager.py]": 24.65272542834282, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_kv_cache_budget_split.py]": 24.599674943834543, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_scheduler_serializable_output.py]": 23.298357471823692, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_mistral.py]": 31.256194911897182, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_pixtral.py]": 30.9562314376235, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_nemotron_nano_preprocessing.py]": 56.75650619342923, + "test_unittests.py::test_unittests_v2[unittest/_torch/models/checkpoints/hf/test_weight_loader.py]": 23.097423560917377, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_torch_sampler.py]": 261.76642061397433, + "test_unittests.py::test_unittests_v2[unittest/_torch/test_model_config.py]": 22.64773726090789, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_block.py]": 19.638685578946024, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_cluster_storage.py]": 41.06863783299923, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_openai_client.py]": 26.17624157294631, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_extractor.py]": 19.388083309866488, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_messenger.py]": 21.500893251039088, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_openai_disagg_service.py]": 27.999925918877125, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_remoteDictionary.py]": 49.35567116411403, + "test_unittests.py::test_unittests_v2[unittest/executor/test_fatal_error_health_check.py]": 25.49836080148816, + "test_unittests.py::test_unittests_v2[unittest/executor/test_rpc.py]": 497.4198516495526, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_content_format.py]": 23.498350095003843, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_url_validation.py]": 24.147701989859343, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_harmony_channel_validation.py]": 36.30120064690709, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_additional_model_outputs.py -m \"gpu1\"]": 48.403994899243116, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_kv_cache_dtype_override.py]": 24.848896216601133, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryArchitectureExtraction]": 57.554038252681494, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_utils.py]": 87.86072044074535, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_reasoning_parser.py]": 23.850911132991314, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_serialization.py]": 23.949409436434507, + "test_unittests.py::test_unittests_v2[unittest/media/test_encoding.py]": 23.400118369609118, + "test_unittests.py::test_unittests_v2[unittest/others/test_lora_manager.py]": 33.45008610934019, + "test_unittests.py::test_unittests_v2[unittest/others/test_time_breakdown.py]": 25.352738097310066, + "test_unittests.py::test_unittests_v2[unittest/scripts]": 29.149033349007368, + "test_unittests.py::test_unittests_v2[unittest/usage/test_config.py]": 19.241729954257607, + "test_unittests.py::test_unittests_v2[unittest/usage/test_opt_out.py]": 19.389309899881482, + "test_unittests.py::test_unittests_v2[unittest/usage/test_schema.py]": 18.941728390287608, + "test_unittests.py::test_unittests_v2[unittest/utils/test_logger.py]": 23.649205207824707, + "test_unittests.py::test_unittests_v2[unittest/visual_gen/test_output.py]": 24.097700644284487, + "disaggregated/test_disaggregated.py::test_disaggregated_benchmark_gen_only_insufficient_kv[TinyLlama-1.1B-Chat-v1.0]": 83.51802018284798, + "disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0]": 99.42717270739377, + "disaggregated/test_disaggregated.py::test_disaggregated_ngram[TinyLlama-1.1B-Chat-v1.0]": 91.52266521565616, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 20.080937860999256, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 60.11040577664971, + "llmapi/test_llm_api_connector.py::test_connector_async_onboard[True]": 17.928842270746827, + "llmapi/test_llm_api_connector.py::test_connector_async_save[True]": 12.977049142122269, + "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[True]": 25.513779915869236, + "llmapi/test_llm_api_connector.py::test_connector_multi_request": 9.052204478532076, + "llmapi/test_llm_api_connector.py::test_connector_priorities_default": 12.085481505841017, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[beam_search]": 7.714497704058886, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[False]": 16.96005078405142, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[False]": 9.707948675379157, + "llmapi/test_llm_api_connector.py::test_connector_simple[False]": 13.015593154355884, + "test_e2e.py::test_get_ci_container_port": 0.0005284938961267471, + "test_e2e.py::test_openai_chat_example[pytorch]": 561.0242523606867, + "test_e2e.py::test_openai_misc_example[pytorch]": 274.1224825847894, + "test_e2e.py::test_openai_perf_metrics": 79.67633541114628, + "test_e2e.py::test_openai_reasoning[pytorch]": 240.12692171894014, + "test_e2e.py::test_openai_tool_call": 88.49165302142501, + "test_e2e.py::test_trtllm_serve_top_logprobs[pytorch]": 87.6148243919015, + "thirdparty/test_cmake_third_party.py::test_cmake_listfiles": 0.10550227761268616, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_disagg_index_mapper_early_release.py]": 25.352552112191916, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_kv_cache_estimation.py]": 25.74726054444909, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_cohere2.py]": 48.46714654006064, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_parakeet.py]": 42.047723326832056, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_radio.py]": 25.22459696419537, + "test_unittests.py::test_unittests_v2[unittest/_torch/models/checkpoints/hf/test_checkpoint_loader.py]": 56.12043555267155, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_torch_multi_arange.py]": 41.8165452927351, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_trtllm_sampler.py]": 173.59781158529222, + "test_unittests.py::test_unittests_v2[unittest/_torch/visual_gen/test_visual_gen_params.py]": 24.5291297249496, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_agent_multi_backends.py]": 55.291444917209446, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_cluster_manager_worker.py]": 50.3197379950434, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_utils.py]": 18.99160304525867, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_mamba_transfer.py]": 32.886980537325144, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_openai_disagg_server.py]": 24.754129583016038, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_peer.py]": 18.989873898215592, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_router.py]": 22.49254883499816, + "test_unittests.py::test_unittests_v2[unittest/executor/test_ipc.py]": 31.610522534698248, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_chat_template_dispatch.py]": 25.169238595291972, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_multimodal.py]": 25.204775847494602, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_chat_utils.py]": 24.531344229355454, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_tool_parsers.py]": 28.049183627590537, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_gc_utils.py]": 31.62360288016498, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_args.py]": 166.04291549883783, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryPyTorchBackend]": 59.91823088936508, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_request_priority.py]": 25.325514804571867, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_utils.py]": 26.306996766477823, + "test_unittests.py::test_unittests_v2[unittest/others/test_convert_utils.py]": 23.858180426061153, + "test_unittests.py::test_unittests_v2[unittest/others/test_lora_module_count.py]": 25.352086970582604, + "test_unittests.py::test_unittests_v2[unittest/others/test_tracing.py]": 134.26669367216527, + "test_unittests.py::test_unittests_v2[unittest/usage/test_collectors.py]": 21.589319362770766, + "test_unittests.py::test_unittests_v2[unittest/usage/test_e2e_capture.py]": 21.25535298185423, + "test_unittests.py::test_unittests_v2[unittest/usage/test_reporter.py]": 21.342137079220265, + "test_unittests.py::test_unittests_v2[unittest/usage/test_transport.py]": 23.60382394399494, + "test_unittests.py::test_unittests_v2[unittest/utils/test_util.py]": 26.621632516384125, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 85.27059441700112, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_cpp_runtime]": 193.6268605251098, + "test_unittests.py::test_unittests_v2[unittest/api_stability]": 25.643251301022246, + "test_unittests.py::test_unittests_v2[unittest/bindings]": 714.101646464318, + "test_unittests.py::test_unittests_v2[unittest/dynamo]": 26.950018653995357, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryTRTBackend]": 166.76355233497452, + "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command_with_lora[llama-llama-models-v2/llama-v2-7b-hf]": 185.91652243910357, + "llmapi/test_llm_examples.py::test_llmapi_chat_example": 78.83368532638997, + "llmapi/test_llm_examples.py::test_llmapi_server_example": 83.81749124731869, + "test_e2e.py::test_openai_misc_example[trt]": 193.8965124920942, + "test_e2e.py::test_trtllm_bench_sanity[--non-streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 352.73563918611035, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition0\"]": 73.73264672234654, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_no_cache.py]": 54.35324762901291, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_llama.py]": 1274.6575344190933, + "test_e2e.py::test_openai_reasoning[trt]": 112.70626465464011, + "test_e2e.py::test_trtllm_serve_example": 209.98838954232633, + "test_e2e.py::test_trtllm_serve_top_logprobs[trt]": 140.92444621305913, + "test_unittests.py::test_unittests_v2[unittest/test_model_runner_cpp.py]": 476.4086992479861, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition1\"]": 73.379359818995, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition2\"]": 72.02512616757303, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_IFB.py]": 126.87880356237292, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mistral.py]": 328.5478213331662, + "llmapi/test_llm_e2e.py::test_llmapi_exit": 18.432179793715477, + "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-llama-models/llama-7b-hf]": 232.97426022961736, + "llmapi/test_llm_examples.py::test_llmapi_kv_cache_connector[Qwen2-0.5B]": 82.59823241084814, + "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 114.59312928467989, + "test_e2e.py::test_openai_health": 60.943585466593504, + "test_e2e.py::test_trtllm_bench_latency_sanity[FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 246.85433532483876, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_build_cache.py]": 23.103414099663496, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition3\"]": 68.93004452623427, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"xqa_generic\"]": 801.098383044824, + "test_unittests.py::test_unittests_v2[unittest/trt/functional]": 196.5955226458609, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization]": 487.10545333847404, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[gemma-3-1b]": 17.166659578680992, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[phi-4-mini]": 14.305965591222048, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[qwen2-7b]": 13.09041291847825, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[qwen3-0.6b]": 5.315278757363558, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[starcoder2-3b]": 17.601711992174387, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[tinyllama-1.1b]": 6.3743207305669785, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encode_matches_huggingface_classification[bert-yelp]": 3.595465302467346, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encode_matches_huggingface_per_token_reward[qwen2.5-prm-7b]": 11.411922980099916, + "test_e2e.py::test_openai_lora": 173.81692308560014, + "test_e2e.py::test_trtllm_serve_lora_example": 98.36129532754421, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_encode.py]": 58.21390492469072, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part0\"]": 598.7036472780164, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part1\"]": 126.27206432109233, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part2\"]": 484.83895123400725, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part3\"]": 208.49013988906518, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_memory_profiling.py::test_pyexecutor_and_kvcache_share_execution_stream]": 40.03293885290623, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_quantization.py]": 23.93977590277791, + "triton_server/test_triton.py::test_medusa[medusa]": 569.5001850500703, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/compile]": 37.790911596268415, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/custom_ops]": 623.6737900990993, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/models]": 115.3445010241121, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/shim]": 22.633222276344895, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/smoke]": 670.1154091879725, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/transformations]": 300.3593855574727, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/utils]": 24.93424261920154, + "cpp/test_e2e.py::test_model[-gpt_tests-80]": 615.122148225084, + "cpp/test_unit_tests.py::test_unit_tests[common-80]": 12.664204696193337, + "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 687.9856592249125, + "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 1.2464399840682745, + "cpp/test_unit_tests.py::test_unit_tests[utils-80]": 1.8540223143063486, + "cpp/test_e2e.py::test_model[-gpt_executor-80]": 814.0748630631715, + "cpp/test_unit_tests.py::test_unit_tests[batch_manager-80]": 292.39521062839776, + "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 278.46642002556473, + "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 856.879348281771, + "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 548.5782862240449, + "test_e2e.py::test_openai_chat_with_logit_bias[torch_sampler]": 73.78876239433885, + "test_e2e.py::test_openai_chat_with_logit_bias[trtllm_sampler]": 73.2108123600483, + "test_e2e.py::test_openai_completions_with_logit_bias[trtllm_sampler]": 72.54250191617757, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen\"]": 754.3435163404793, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_beam_search.py]": 789.6666672006249, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_logits_logprobs.py]": 392.0245815804228, + "test_e2e.py::test_openai_completions_with_logit_bias[torch_sampler]": 75.10106470668688, + "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 28.074005853384733, + "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 24.315578892827034, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nas\"]": 32.59954630583525, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_out_of_tree\"]": 138.8535058517009, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_phi3\"]": 26.83701508399099, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen_moe\"]": 281.13565809698775, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_beam_search_speculative_d2h.py]": 213.0422897520475, + "triton_server/test_triton.py::test_fill_template[fill-template]": 1.6639938056468964, + "triton_server/test_triton.py::test_llama[llama]": 179.10436917375773, + "triton_server/test_triton.py::test_llmapi_unit_tests[llmapi-unit-tests]": 4.315617096610367, + "triton_server/test_triton.py::test_mistral[mistral]": 175.61003254074603, + "triton_server/test_triton.py::test_mistral_ib_streaming[mistral-ib-streaming]": 342.24799779988825, + "triton_server/test_triton.py::test_python_preproc_unit_tests[python-preproc-unit-tests]": 9.682531874626875, + "triton_server/test_triton_llm.py::test_benchmark_core_model[gptj_6b-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization-4096--1-1-1-False]": 425.68877257127315, + "triton_server/test_triton_llm.py::test_gpt_350m_ifb[test_basic-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 103.82896339520812, + "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[e2e]": 106.43146257847548, + "triton_server/test_triton_llm.py::test_gpt_disaggregated_serving_bls[test_basic-False-1-top_k_top_p--False-True-True-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-True-tensorrt_llm_bls]": 98.34344942122698, + "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[False-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 268.7699463283643, + "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-False-ensemble]": 13.90028147213161, + "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[batched_inputs-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 181.1615214087069, + "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[test_stop_words-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 16.108110164292157, + "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-enableDecoupleMode-tensorrt_llm]": 78.61408129800111, + "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-python-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 114.62468159664422, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 122.2498293183744, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 33.313044152222574, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 17.912641955539584, + "triton_server/test_triton_llm.py::test_whisper_large_v3_ifb[True-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-0.5-guaranteed_no_evict--24000-1-1-1-False-ensemble]": 137.97018379811198, + "triton_server/test_triton.py::test_gpt[gpt]": 182.20508953463286, + "triton_server/test_triton.py::test_gptj[gptj]": 223.35692731942981, + "triton_server/test_triton.py::test_python_multimodal_encoders_unit_tests[python-multimodal-encoders-unit-tests]": 28.360558917745948, + "triton_server/test_triton.py::test_whisper[whisper]": 204.72570630162954, + "triton_server/test_triton_llm.py::test_benchmark_core_model[llama_v2_7b-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization-4096--1-1-1-False]": 389.1086704917252, + "triton_server/test_triton_llm.py::test_eagle_vicuna_7b_ifb[False-1-eagle--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 189.3712582429871, + "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[accuracy]": 149.95370694343, + "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 278.40986166801304, + "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[batched_inputs-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-enableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 161.46982649620622, + "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-disableDecoupleMode-tensorrt_llm]": 90.04027471225709, + "triton_server/test_triton_llm.py::test_llmapi_lora[1-tensorrt_llm]": 63.79091303702444, + "triton_server/test_triton_llm.py::test_medusa_vicuna_7b_ifb[False-1-medusa--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 161.17319475393742, + "triton_server/test_triton_llm.py::test_t5_small_enc_dec_ifb[test_basic-False-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap---guaranteed_no_evict--4096-1-1-1-False-tensorrt_llm_bls]": 125.47026043757796, + "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-tensorrtllm-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 119.73666133265942, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 114.50461501348764, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 18.529517172835767, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 18.59724544081837, + "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp": 251.41180250793695, + "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_ngram": 98.84480216354132, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False]": 228.78805011400254, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True]": 189.88735772500513, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True]": 294.4758595749736, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 246.82779098799801, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 119.71112632699078, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 222.06951643901994, + "test_unittests.py::test_unittests_v2[unittest/_torch/attention]": 307.9887097366154, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor]": 557.2529671951197, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/mamba]": 221.1838954249397, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"]": 151.7901495929982, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"]": 340.960238839034, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"]": 47.828515230008634, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]]": 48.728463740990264, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]]": 24.82597585298936, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 21.67635251401225, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 25.978178004006622, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]]": 20.424516523984494, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 19.677430056006415, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 24.725896997988457, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 23.124266630999045, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 20.727860749000683, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]]": 42.2266049849859, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_awq_quantization.py]": 19.188348875846714, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_activation_quant.py]": 20.014674406964332, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_add_rms_norm_quant.py]": 19.493550593033433, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_moe.py]": 19.816725137876347, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_group_rmn_norm.py]": 23.089757694862783, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_load_balancer.py]": 19.26597039285116, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_routing.py]": 28.89103313582018, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_rotary_embedding.py]": 23.866294936975464, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_triton_linear.py]": 23.79145319806412, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/tests_lora_modules]": 659.0716862599365, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel]": 4351.49280942278, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/serial]": 58.17412027902901, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 217.26752274786122, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 222.7607175433077, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 302.9856714049997, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 223.62056540699996, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 200.86565831400003, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 189.0692499190004, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 1031.1528510910002, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 461.9252251499993, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=False]": 267.6105636049997, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]]": 71.63301057300032, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]]": 48.08178372900056, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 58.63044721000006, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]]": 85.04990262999945, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 50.014132944000266, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]]": 27.723852323000756, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 20.322831061000215, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 28.123089660000005, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 48.727174462999756, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 50.479853326999546, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 48.27790872099922, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 17.62364664799952, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 27.62391243500042, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 18.274859787999958, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_host_sharer.py]": 20.292041711043566, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python": 163.95312987302896, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 287.43463359299994, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 196.23940314585343, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 216.26651366100032, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 249.59442985600026, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 261.57555265400015, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 207.06534601002932, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 155.4231541629415, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 79.62999053788371, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True]": 419.2233988319995, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_cutlass-torch_compile=True]": 275.25382337999963, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_llama\"]": 119.63326477212831, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_mla_helix.py]": 976.3970404448919, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling -k \"deepseek\"]": 354.90453113700005, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 178.26867335097631, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 229.5246064819803, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 172.92849959799787, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 147.37839249300305, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 266.96483929408714, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 206.03644429100677, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 188.69488530093804, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 164.00336496997625, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 172.82999583697529, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 251.55259347101673, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False]": 139.83139117099927, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=False]": 135.59422479500063, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 111.3068211893551, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=False]": 183.35118844200042, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True]": 201.5803480189934, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_gpt_oss\"]": 107.69878626195714, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mixtral\"]": 41.068741515045986, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-one_model-overlap_scheduler]": 2229.254062668013, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-two_model-overlap_scheduler]": 2873.1229173520114, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 1908.5162332250038, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler]": 2359.458676673996, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False]": 222.66332006501034, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 210.72954666399164, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-True]": 235.43180733395275, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-False]": 186.75015839299886, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-True]": 215.9784758510068, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-True]": 225.26497576397378, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-True]": 181.96771040401654, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 79.8115957059781, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 92.91837230202509, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram": 451.23835995496484, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True]": 393.2114061889588, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True-True]": 371.7093208580045, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu -m \"not post_merge\"]": 2881.170321949001, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_additional_model_outputs.py -m \"gpu2\"]": 45.69425537396455, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_mpi_session.py::test_llmapi_launch_multiple_tasks]": 88.89764528500382, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync": 275.9294940570253, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 280.02051854197634, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-False]": 226.95165822003037, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-True]": 176.63856029702583, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False]": 213.76671027002158, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-False]": 221.16686225897865, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-False]": 180.42018839297816, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-False]": 198.51033656601794, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 188.66204771603225, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 293.0652321569505, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 179.5286056349869, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 93.58281131199328, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False]": 393.84327019704506, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill": 393.43859653401887, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_pixtral.py::test_tensor_parallelism]": 44.9982833910035, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_disagg_serving_perf_metrics.py]": 71.39680112700444, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu2\"]": 272.88288813398685, + "examples/test_ray.py::test_llm_inference_distributed_ray[pp2]": 138.11034928797744, + "examples/test_ray.py::test_llm_inference_distributed_ray[tep2]": 109.29040575103136, + "examples/test_ray.py::test_llm_inference_distributed_ray[tp2]": 72.43276466900716, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu2\"]": 235.5710942610167, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part0\"]": 341.03334485401865, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part1\"]": 341.3838682859787, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part2\"]": 179.14875201904215, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part3\"]": 180.01359257102013, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part4\"]": 215.3152888190234, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"gpu2\"]": 71.13692150497809, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer]": 191.26553119625896, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[Qwen_QwQ-32B-False]": 440.4416093239561, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[google_gemma-3-1b-it-False]": 61.59581129811704, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.1-8B-Instruct-False]": 46.71345825213939, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.3-70B-Instruct-False]": 171.76501162629575, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Codestral-22B-v0.1-False]": 240.30174564756453, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Ministral-8B-Instruct-2410-False]": 75.44280633982271, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True]": 127.60458957124501, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm]": 399.2673514550552, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-mpi_kvcache-90]": 566.998665202409, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-nixl_kvcache-90]": 9.834118995815516, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-ucx_kvcache-90]": 10.331790680065751, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-mpi_kvcache-90]": 23.657401267439127, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-nixl_kvcache-90]": 23.906753124669194, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-ucx_kvcache-90]": 23.80666296184063, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-mpi_kvcache-90]": 12.536427475512028, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-nixl_kvcache-90]": 12.840721806511283, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-ucx_kvcache-90]": 12.540847543627024, + "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-mpi_kvcache-90]": 18.39725266955793, + "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-nixl_kvcache-90]": 16.89188546873629, + "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-ucx_kvcache-90]": 16.693953413516283, + "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-nixl_kvcache-90]": 30.972941564396024, + "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-ucx_kvcache-90]": 29.168602090328932, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-mpi_kvcache-90]": 4.774993252009153, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-nixl_kvcache-90]": 4.874061470851302, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-ucx_kvcache-90]": 4.673807211220264, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-mpi_kvcache-90]": 9.332997119054198, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-nixl_kvcache-90]": 9.682227460667491, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-ucx_kvcache-90]": 9.431488024070859, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-mpi_kvcache-90]": 11.735960626974702, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-nixl_kvcache-90]": 11.684558764100075, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-ucx_kvcache-90]": 11.736474914476275, + "cpp/test_multi_gpu.py::test_enc_dec[t5-90]": 6.025917749851942, + "cpp/test_multi_gpu.py::test_llama_executor[llama-leader-90]": 84.81573174893856, + "cpp/test_multi_gpu.py::test_llama_executor[llama-orchestrator-90]": 162.71292873471975, + "cpp/test_multi_gpu.py::test_llama_executor_guided_decoding[llama-90]": 13.140320116654038, + "cpp/test_multi_gpu.py::test_llama_executor_logits_proc[llama-90]": 44.34581693634391, + "cpp/test_multi_gpu.py::test_trt_gpt_real_decoder[llama-90]": 113.51448027789593, + "cpp/test_multi_gpu.py::test_mpi_utils[90]": 45.64661745913327, + "cpp/test_multi_gpu.py::test_user_buffer[2proc-90]": 2.11952923797071, + "cpp/test_multi_gpu.py::test_fused_gemm_allreduce[4proc-90]": 19.49912805482745, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 134.54751868266612, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 154.7853800514713, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 135.73957254644483, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 174.52036147098988, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 152.75700911972672, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 211.04387662932277, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 129.80014774948359, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_transceiver_runtime_python[DeepSeek-V3-Lite-fp8]": 159.481474686414, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp4-bf16-trtllm-deepseekv3_lite]]": 102.35796185210347, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 152.4078388279304, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 119.35970550915226, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 130.15719587402418, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 200.01216824213043, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus_static_eplb": 124.69750702101737, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_empty_batch[DeepSeek-V3-Lite-bf16]": 202.9623187012039, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 139.4163328241557, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp1-bf16-trtllm-deepseekv3_lite]]": 93.75344497803599, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 1437.00570011104, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 541.5369369090186, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 471.5579449380166, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 866.0648840669892, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 594.6447642149869, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache_no_reuse-tp4-cutlass-auto]": 419.3978291969979, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 376.8179885330028, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2]": 154.61260438500904, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2]": 100.9895223529893, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 161.34927469299873, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 88.99735977302771, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 99.42438653996214, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[noadp-ctx_tp2pp1-gen_tp1pp1]": 151.80676046095323, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 104.44514593598433, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 54.41597771999659, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache]": 374.00853590102633, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache]": 210.51360351400217, + "disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin]": 100.25905621499987, + "disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin]": 98.90331856295234, + "disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin]": 49.15359141398221, + "disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin]": 49.10201428201981, + "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin]": 206.90428009995958, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0]": 64.96553604403744, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python[TinyLlama-1.1B-Chat-v1.0]": 85.28914290998364, + "test_e2e.py::test_ptp_quickstart_advanced_deepseek_v3_lite_4gpus_adp_balance[DeepSeek-V3-Lite-FP8-DeepSeek-V3-Lite/fp8]": 111.94489605998388, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and FP8 and not MXFP8\"]": 1379.5952013240312, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_AWQ\"]": 669.7905237200321, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb]": 109.54814359301236, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu4\"]": 234.50615247199312, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2]": 167.242011513561, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2]": 98.52402048092335, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2]": 199.50387456268072, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 146.5718833785504, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2]": 94.77812515385449, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[adp-ctx_tp2pp1-gen_tp2pp1]": 156.16143955662847, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 80.73761726729572, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 76.1688071032986, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 74.1406664410606, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache]": 372.94147814717144, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache]": 213.26513709686697, + "disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin]": 139.31395231373608, + "disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin]": 98.91171756386757, + "disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware]": 49.066187175922096, + "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing]": 213.4513011770323, + "disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing]": 207.4394314819947, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0]": 70.3319569742307, + "test_e2e.py::test_ptp_quickstart_advanced_bs1": 112.39259800966829, + "test_e2e.py::test_trtllm_bench_llmapi_launch[pytorch_backend-llama-v3-llama3-8b]": 128.85917745530605, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A16_MXFP4\"]": 1016.8723992183805, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W8A16\"]": 691.5421204390004, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_py_cache_transceiver_mp.py]": 2267.29654201027, + "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp1_4instances]": 112.71113630395848, + "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp2_2instances]": 116.105620782997, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu4\"]": 469.73704576998716, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp1_4instances]]": 290.26655738794943, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp2_2instances]]": 318.72108415700495, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"gpu4\"]": 77.99721742799738, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 132.69184266900993, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 158.83748530130833, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse": 153.4440853409469, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse": 82.38394147483632, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler": 71.19869335694239, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=False-overlap_scheduler=False]": 1218.607248812914, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 80.05104345735162, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 107.66202485794201, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format": 62.41016404423863, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash": 621.2223827303387, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8": 152.73669020738453, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 110.01207454688847, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[False-TinyLlama-1.1B-Chat-v1.0]": 35.032475878018886, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 101.00177443493158, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 83.89112774282694, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 63.88218005700037, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[no-overlap-chunked]": 168.74234423786402, + "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 61.069874382112175, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 250.20505770668387, + "test_unittests.py::test_unittests_v2[unittest/_torch/compilation]": 25.54568713111803, + "test_unittests.py::test_unittests_v2[unittest/_torch/debugger]": 19.291544676292688, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal]": 1385.4457194129936, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_aux.py]": 18.838812944013625, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_request_id.py]": 19.28899485990405, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mla-ctx_fp16_gen_fp16]]": 31.09049210883677, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 129.92204322700854, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True]": 235.29396068817005, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 190.64960246894043, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=2]": 127.35095311468467, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 220.07175700203516, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler": 75.0205802610144, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse": 72.13469798583537, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format": 34.00468918774277, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_fp8": 300.7987077347934, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 309.6223061759956, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 247.2587843281217, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[True-TinyLlama-1.1B-Chat-v1.0]": 25.400752289104275, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 85.75992965977639, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 83.36212210077792, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 53.837384417653084, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[max-util-chunked]": 158.87441884400323, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[python-scheduler]": 172.01433536224067, + "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 148.76338731311262, + "test_e2e.py::test_openai_responses": 352.3726217681542, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 92.3254858199507, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 99.50422196800355, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_gemma3\"]": 243.37643587309867, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTLASS\"]": 228.12391290487722, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel_hw_agnostic]": 585.0550106270239, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_region.py]": 20.752864133100957, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disaggregated_params.py]": 21.85614148201421, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mha-ctx_fp16_gen_fp16]]": 33.907329918816686, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True]": 233.35579176666215, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format": 96.40042386110872, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=0]": 120.64296809816733, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=fp8-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True]": 131.03063145186752, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse": 82.81574466405436, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse": 72.67876957170665, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=True-eagle3_one_model=True-overlap_scheduler=True]": 1075.0943210227415, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 48.7322347429581, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True]": 170.77184127038345, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16": 511.3651969926432, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 126.62458716612309, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=True-enable_draft_len_schedule=False]": 109.76810259604827, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 113.73702853103168, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8]": 848.9961165660061, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 148.54536304995418, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_cancel_gen_requests[TinyLlama-1.1B-Chat-v1.0]": 36.3188966489397, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[False-TinyLlama-1.1B-Chat-v1.0]": 28.20248524507042, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-True-DeepSeek-V3-Lite-fp8/fp8]": 127.0615609427914, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 45.532342946040444, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[guaranteed-chunked]": 155.79833736736327, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-no-chunked]": 161.94301183428615, + "test_e2e.py::test_openai_chat_harmony": 256.37394262477756, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 94.71067717205733, + "test_e2e.py::test_trtllm_benchmark_serving[llama-3.1-model/Meta-Llama-3.1-8B]": 125.41995501983911, + "test_unittests.py::test_unittests_v2[unittest/_torch/lora]": 20.80229289084673, + "test_unittests.py::test_unittests_v2[unittest/_torch/misc]": 344.32345556328073, + "test_unittests.py::test_unittests_v2[unittest/_torch/speculative/test_eagle3.py]": 1019.0683842310682, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_page.py]": 20.143666464835405, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_kv_transfer_mp.py]": 123.84979211073369, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_rank_info.py]": 18.993583410046995, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py]": 550.8318111081608, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse": 162.39044524217024, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse": 82.52972493879497, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available": 81.73615213809535, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=TRTLLM]": 149.4546632859856, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=True-overlap_scheduler=True]": 1072.4692367021926, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 77.7370796087198, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 100.55706254392862, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False]": 120.40908256312832, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dflash": 184.22292581619695, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dummy_load_format": 59.57457195688039, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=True]": 110.55068488512188, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8]": 111.89929933100939, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0]": 23.80086028506048, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 70.33974885300267, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[True-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 41.7932134908624, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-chunked]": 163.14287832612172, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix_smoke": 130.64739427994937, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 70.83530108863488, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_sanity]": 253.82326687918976, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler]": 1191.2096670051105, + "test_unittests.py::test_unittests_v2[unittest/_torch/speculative/hw_agnostic]": 1831.2627570661716, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_cache_transceiver_single_process.py]": 3324.7812275849283, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_kv_transfer.py]": 928.6214334205724, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_perf_logger.py]": 21.391628997866064, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 152.97960573100136, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 153.88833729398903, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 121.66436125000473, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 134.81132309199893, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 171.62797562000924, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 127.72052242897917, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 154.6956266699999, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 129.47093768097693, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 134.80305856201448, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 156.33859626899357, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 127.20496735500637, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 154.19827068597078, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 116.73455706000095, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 125.1877486439771, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 147.13845685200067, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 123.54514406202361, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 127.23275308904704, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 81.83421968104085, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False]": 279.30084877600893, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype": 957.9583136999863, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 89.2490907079773, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 135.65254550502868, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 89.36297566298163, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=FLASHINFER]": 243.73127793997992, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 75.11799066798994, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 93.10347031999845, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 45.586120243999176, + "accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill]": 215.60936970298644, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill]": 344.82283938402543, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 137.0733454680012, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 143.5359488390095, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 116.0004205229925, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 123.336180874001, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 166.7623178389913, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 118.65510969999013, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 145.0461449699942, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 124.612122652994, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 122.85179915899062, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 140.71412417298416, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 114.79413927401765, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 131.09561426599976, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 119.48417889501434, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 176.2980228930246, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 122.85712360299658, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 121.1533630049671, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 121.68564532196615, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 124.12095914199017, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 141.94041900796583, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 129.43495701398933, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True]": 317.6746448139893, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 447.11634525697445, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 127.43863378401147, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=FLASHINFER]": 187.84411286801333, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 124.13719056698028, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 111.11976641096408, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 140.97133619396482, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 168.74814063502708, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 233.80381705239415, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 77.1988166347146, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 79.65342007577419, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 69.40490325912833, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 76.35176690667868, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 81.16481507569551, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 72.47008088231087, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 83.34465102478862, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 67.5397269576788, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 89.43130768090487, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 76.1686408482492, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 72.9958230741322, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 78.06165297329426, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 75.27438813075423, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 80.74985931068659, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 86.10687870532274, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 82.69737496972084, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 87.65647116675973, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 82.57926809042692, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 75.8943274691701, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 75.05017046257854, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 122.5022951066494, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 72.29567366093397, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 83.35832703486085, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 77.41014160588384, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 104.96686546504498, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 75.51850606128573, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 85.44486574828625, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 71.4350159689784, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 83.38494031503797, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 153.37503067031503, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 99.90067131072283, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 94.13767189905047, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 104.74171372875571, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 94.56583361327648, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 115.35695255175233, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 103.06190882995725, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 95.00528683885932, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 106.75661451742053, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 115.989338260144, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 99.97725444287062, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 123.66422721371055, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 103.99547399580479, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 103.63337088748813, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 96.55534584447742, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 98.63648730143905, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 95.0242466814816, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 96.21232785284519, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 98.30433483421803, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False]": 153.95823365077376, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=False]": 63.16813847795129, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=True]": 76.7680161446333, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=False]": 63.61583887413144, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=True]": 72.02205647155643, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=True]": 88.9733154848218, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 112.0710693411529, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 60.62598153203726, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 61.945570688694715, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 91.45242443680763, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 62.574849516153336, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=False]": 992.8535194210708, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True]": 620.6253603063524, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False]": 808.6647146046162, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=True]": 600.0486651174724, + "test_e2e.py::test_trtllm_bench_mgmn": 91.79827606678009, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_235b_a22b_fp8_hopper-qwen3_235b_a22b_fp8_tp4_ep4_cutlass_8k1k]": 910.4367664121091, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_32b_fp8_hopper-qwen3_32b_fp8_tp2_6k1k]": 1242.9062750637531, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 292.60953721031547, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 175.63857275992632, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 186.52553553506732, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 161.51636945083737, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4]": 190.38547903299332, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4]": 131.49605344608426, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4]": 104.33410734310746, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4]": 99.25327169150114, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2]": 152.5729262419045, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2]": 89.9921805858612, + "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[False]": 661.3829264938831, + "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[True]": 322.62455509230494, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[latency]": 2162.4878128543496, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_off]": 231.63562587695196, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0]": 89.56677887961268, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0]": 67.95142366364598, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora]": 304.1238304525614, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 16.223822474479675, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.float16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 16.374391281977296, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency]": 246.29444654472172, + "test_unittests.py::test_unittests_v2[unittest/_torch/attention/test_attention_mla.py]": 231.48642779141665, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_kv_cache_aware_routing[mtp_nextn=0]": 163.56748677929863, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=CUTLASS]": 252.98956363508478, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP]": 251.82381451921538, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=2]": 211.0549951121211, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=2]": 225.89990004291758, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 157.48752847802825, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 204.017874710029, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 284.236327491235, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 166.91178651992232, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 174.30949333589524, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=TRTLLM]": 228.28387101413682, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-overlap_scheduler]": 439.3828970207833, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-two_model-overlap_scheduler]": 385.81541442498565, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 548.5454997923225, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus_online_eplb[fp8]": 396.00846621999517, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False]": 688.351709254086, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 691.9655529111624, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP]": 410.2264897967689, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTLASS]": 369.08950016694143, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[no_cuda_graph_overlap-cutlass]": 343.0884657520801, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep1-cutlass]": 422.295906488318, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_off-trtllm]": 407.53346419893205, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_boundary]": 353.82153057679534, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 558.5696469410323, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_8k1k]": 813.6045300079277, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k]": 560.361906999955, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k]": 468.6927351569757, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 295.37768525700085, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 391.1908592169639, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 417.4842175709782, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_8k1k]": 733.6566050739493, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tep2_1k8k]": 787.2090990659781, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 511.71572106005624, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 2908.4839029698633, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 333.19444385915995, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k]": 452.1892409371212, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 411.64431313984096, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 337.6239333390258, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k]": 945.6405130829662, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k]": 708.4553801259026, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 473.57998897507787, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 178.40271015278995, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k8k]": 2171.321392316837, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_1k1k]": 233.66355675505474, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k]": 405.2331366450526, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 546.0889781760052, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 168.79803335713223, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_8k1k]": 266.5968868341297, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k]": 203.08553342614323, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 463.4555850980105, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 450.5373125029728, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k8k]": 521.3741318379762, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep2_1k1k]": 439.9891236410476, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp4_eagle3_1k1k]": 229.3437453189399, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2190.739206559956, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL]": 5787.14215046796, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 667.5444930130616, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 456.85187179397326, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 603.7066158190137, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 2627.6349721120205, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 2105.2029822249897, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 238.50409000902437, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 206.6761712988373, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 219.68855385808274, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 144.7110821590759, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 283.6205424251966, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 256.5663748360239, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 85.24103148793802, + "examples/test_visual_gen.py::test_visual_gen_quickstart": 244.85972903110087, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_quant.py]": 341.68943272158504, + "test_unittests.py::test_unittests_v2[unittest/trt/model/eagle]": 35.844667091965675, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt_e2e.py]": 375.8710653409362, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_api_multi_gpu.py]": 16.811496037989855, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_level_api.py]": 16.826461140066385, + "accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False]": 880.4145672637969, + "accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma3n_e2b_it": 845.1492025442421, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton_paged-False-1]": 365.7970229499042, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-1]": 83.0432817041874, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm]": 185.77484482899308, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-trtllm]": 528.6534720119089, + "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True]": 659.9385115206242, + "examples/test_ad_guided_decoding.py::test_autodeploy_guided_decoding_main_json": 105.7096260804683, + "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[flashinfer-torch-simple]": 64.08092588745058, + "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[trtllm-torch-cudagraph]": 45.57437503896654, + "examples/test_ad_speculative_decoding.py::test_eagle_model_with_weights": 0.8169050384312868, + "examples/test_ad_speculative_decoding.py::test_nemotron_mtp_model_with_weights": 891.9159074425697, + "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]": 245.15667858114466, + "examples/test_ray.py::test_llm_inference_async_ray": 98.74219689599704, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_overlap_scheduler.py]": 552.1893509819638, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py]": 34.445088005042635, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_sleep.py]": 79.70088898506947, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part0\"]": 555.2823958380613, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part1\"]": 546.5310950400308, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part2\"]": 401.5109062109841, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"not (gpu2 or gpu4)\"]": 140.57188469800167, + "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8": 419.2079811580479, + "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8_gemm_plugin": 211.37097293697298, + "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8_gemm_swiglu_plugin": 213.21514989249408, + "accuracy/test_cli_flow.py::TestLlama3_1_8BInstruct::test_fp8_prequantized": 162.781315760687, + "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-disable_fp8]": 332.239777687937, + "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-9b-it]": 277.80719018913805, + "examples/test_gemma.py::test_llm_gemma_1gpu_summary_vswa[gemma-3-1b-it-other-bfloat16-8]": 93.52066849544644, + "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.1-8b]": 149.5570853278041, + "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_1.5b_instruct]": 159.4304439369589, + "test_e2e.py::test_mistral_large_hidden_vocab_size": 65.75925209745765, + "test_e2e.py::test_trtllm_bench_sanity[--streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 223.21146443486214, + "test_e2e.py::test_trtllm_bench_sanity[-extra_config-streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 222.07093558087945, + "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_moe.py]": 280.5644433479756, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py]": 50.19224583171308, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_quant_matmul.py]": 57.290618147701025, + "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]": 428.9558831639588, + "accuracy/test_llm_api_pytorch_multimodal.py::TestVILA1_5_3B::test_auto_dtype": 407.1404729729984, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_siglip\"]": 223.38703144699684, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_llava_next.py::test_llava_next_expand_prompt_token_ids_for_mm]": 23.44378970999969, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen3vl.py::TestQwen3VL::test_all]": 138.33508067900038, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/_test_openai_chat_multimodal.py::test_single_chat_session_image_embeds -m needs_l40s]": 158.215519313002, + "accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype": 856.7462824780014, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nano_v2_vl\"]": 511.051033497999, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_vila\"]": 36.8462807270007, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_llava_next.py::TestLlavaNext::test_all]": 36.84895586700077, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen2_5vl.py::TestQwen2_5_VL::test_all]": 101.95980946499913, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen3vl_moe.py::TestQwen3VLMoe::test_all]": 116.45382200300082, + "examples/test_llama.py::test_llm_llama_1gpu[llama-3.1-8b-instruct-hf-fp8-enable_fp8-float16-summarization-nb:1]": 678.0718930736184, + "examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b]": 798.493692047894, + "test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b]": 113.39240002073348, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 169.85340281389654, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 109.8938184119761, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 67.28465028293431, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 967.6696398910135, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_fp8_hf-Qwen3/saved_models_Qwen3-30B-A3B_fp8_hf]": 1818.6560711394995, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf]": 566.2712351139635, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py]": 18.0341038107872, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py]": 24.46495814807713, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_w4a16_linear.py]": 19.433096984401345, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_w4a8_linear.py]": 18.131888270378113, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_weight_only_quant_gemm.py]": 44.08381740376353, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_weight_only_quant_linear.py]": 20.011995047330856 } diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 29a3ef7e4964..c3f2bab7e545 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -225,6 +225,8 @@ full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-p full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) full:GH200/unittest/trt/model_api/test_model_quantization.py SKIP (https://nvbugs/4979955) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_triton_stress] SKIP (https://nvbugs/6250439) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_triton_stress] SKIP (https://nvbugs/6222480) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696)