diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml similarity index 97% rename from tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml rename to tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml index d5fa9d45a931..b7e04b714d00 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml @@ -22,7 +22,7 @@ context_servers: enable_block_reuse: false free_gpu_memory_fraction: 0.8 moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: null print_iter_log: true cache_transceiver_config: @@ -45,7 +45,7 @@ generation_servers: enable_block_reuse: false free_gpu_memory_fraction: 0.8 moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: enable_padding: true batch_sizes: diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml new file mode 100644 index 000000000000..1ebbe92ac159 --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml @@ -0,0 +1,67 @@ +model: gpt_oss/gpt-oss-120b +hostname: localhost +backend: pytorch +context_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 16640 + max_seq_len: 8232 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: &eagle_spec + decoding_type: Eagle + max_draft_len: 3 + eagle3_one_model: true + speculative_model: gpt_oss/gpt-oss-120b-Eagle3 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: TRTLLM + cuda_graph_config: null + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 +generation_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 10240 + max_seq_len: 10240 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: *eagle_spec + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: TRTLLM + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml similarity index 96% rename from tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml rename to tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml index 47b619faf141..1d6f4d336e01 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml @@ -17,7 +17,7 @@ context_servers: free_gpu_memory_fraction: 0.8 disable_overlap_scheduler: true moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: null print_iter_log: true cache_transceiver_config: @@ -39,7 +39,7 @@ generation_servers: free_gpu_memory_fraction: 0.8 disable_overlap_scheduler: true moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: enable_padding: true batch_sizes: diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index f320690ad787..b8136359f972 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -298,12 +298,14 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml", "deepseek_r1_v2_fp4_mtp_stress": f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml", - "gpt_oss_120b_stress": + "gpt_oss_120b_trtllm_stress": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml", - "gpt_oss_120b_eagle_stress": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml", - "gpt_oss_120b_cutlass_stress": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml", + "gpt_oss_120b_eagle_triton_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml", + "gpt_oss_120b_eagle_trtllm_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml", + "gpt_oss_120b_triton_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml", "qwen3_5_4b_fp8_stress": f"{test_configs_root}/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml", "gpt_oss_120b_harmony": @@ -2297,7 +2299,7 @@ def test_disaggregated_gpt_oss_120b_harmony(disaggregated_test_root, cancellation_delay=0.5), marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell)), pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_stress', + test_desc='gpt_oss_120b_trtllm_stress', request_count=60000, accuracy_threshold=0.42, cancellation_rate=10, @@ -2306,22 +2308,34 @@ def test_disaggregated_gpt_oss_120b_harmony(disaggregated_test_root, pytest.param( TestConfig( model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_eagle_stress', + test_desc='gpt_oss_120b_eagle_triton_stress', request_count=60000, accuracy_threshold=0.42, speculative_model_path='gpt_oss/gpt-oss-120b-Eagle3', cancellation_rate=10, cancellation_delay=0.5, ), - marks=(pytest.mark.skip_less_device(8), skip_pre_hopper), + marks=(pytest.mark.skip_less_device(8), skip_no_hopper), + ), + pytest.param( + TestConfig( + model_path='gpt_oss/gpt-oss-120b', + test_desc='gpt_oss_120b_eagle_trtllm_stress', + request_count=60000, + accuracy_threshold=0.42, + speculative_model_path='gpt_oss/gpt-oss-120b-Eagle3', + cancellation_rate=10, + cancellation_delay=0.5, + ), + marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell), ), pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_cutlass_stress', + test_desc='gpt_oss_120b_triton_stress', request_count=60000, accuracy_threshold=0.42, cancellation_rate=10, cancellation_delay=0.5), - marks=(pytest.mark.skip_less_device(4), skip_pre_hopper)), + marks=(pytest.mark.skip_less_device(4), skip_no_hopper)), pytest.param(TestConfig(model_path='Qwen3.5-4B-FP8', test_desc='qwen3_5_4b_fp8_stress', request_count=3000, diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index e6a923f5d0e9..31812cdab731 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -661,7 +661,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torc accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False] diff --git a/tests/integration/test_lists/qa/llm_function_rtx6k.txt b/tests/integration/test_lists/qa/llm_function_rtx6k.txt index 25e7eabd5da7..43ec270512d8 100644 --- a/tests/integration/test_lists/qa/llm_function_rtx6k.txt +++ b/tests/integration/test_lists/qa/llm_function_rtx6k.txt @@ -123,7 +123,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] diff --git a/tests/integration/test_lists/qa/llm_function_stress.txt b/tests/integration/test_lists/qa/llm_function_stress.txt index fa8a710359f8..1bf7c4f3f77b 100644 --- a/tests/integration/test_lists/qa/llm_function_stress.txt +++ b/tests/integration/test_lists/qa/llm_function_stress.txt @@ -1,8 +1,9 @@ disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_mtp_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_trtllm_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_triton_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_trtllm_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_triton_stress] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_5_4b_fp8_stress] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_fp8_8gpus accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_nvfp4_4gpus diff --git a/tests/integration/test_lists/test-db/l0_l40s.yml b/tests/integration/test_lists/test-db/l0_l40s.yml index c0fb7b7cbfba..19d4f222f4f6 100644 --- a/tests/integration/test_lists/test-db/l0_l40s.yml +++ b/tests/integration/test_lists/test-db/l0_l40s.yml @@ -60,7 +60,6 @@ l0_l40s: - examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp16] - examples/test_llama.py::test_llm_llama_v3_dora_1gpu[commonsense-llama-v3-8b-dora-r32-llama-v3-8b-hf-base_fp16] - examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b] - - examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp16] - examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] - llmapi/test_llm_examples.py::test_llmapi_quickstart - examples/test_visual_gen.py::test_visual_gen_quickstart diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index e0f5e7bf2342..25c932aad85c 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -116,7 +116,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard_sa SKIP (ht accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype SKIP (https://nvbugs/5520319) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6162122) -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] SKIP (https://nvbugs/5775326) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) @@ -203,7 +202,6 @@ examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bf examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (https://nvbugs/4961624) examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) examples/test_nemotron_nas.py::test_nemotron_nas_summary_2gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) -examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp16] SKIP (https://nvbugs/5612313) examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (https://nvbugs/5447530) examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/5612502) examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin] SKIP (https://nvbugs/5174573) @@ -232,8 +230,8 @@ full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-p full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) full:GH200/unittest/trt/model_api/test_model_quantization.py SKIP (https://nvbugs/4979955) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] SKIP (https://nvbugs/6222480) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] SKIP (https://nvbugs/6222480) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_triton_stress] SKIP (https://nvbugs/6250439) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_triton_stress] SKIP (https://nvbugs/6222480) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696)