diff --git a/examples/configs/curated/deepseek-r1-deepgemm.yaml b/examples/configs/curated/deepseek-r1-deepgemm.yaml index 2d1c496a4eb0..3c00b146e160 100644 --- a/examples/configs/curated/deepseek-r1-deepgemm.yaml +++ b/examples/configs/curated/deepseek-r1-deepgemm.yaml @@ -6,7 +6,6 @@ trust_remote_code: true enable_attention_dp: true cuda_graph_config: enable_padding: true - max_batch_size: 128 kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.8 diff --git a/examples/configs/curated/deepseek-r1-latency.yaml b/examples/configs/curated/deepseek-r1-latency.yaml index 0560f6509e05..35ce5864dbaf 100644 --- a/examples/configs/curated/deepseek-r1-latency.yaml +++ b/examples/configs/curated/deepseek-r1-latency.yaml @@ -3,10 +3,10 @@ tensor_parallel_size: 8 moe_expert_parallel_size: 2 max_num_tokens: 32768 trust_remote_code: true -moe_backend: TRTLLM -use_cuda_graph: true kv_cache_config: free_gpu_memory_fraction: 0.75 +moe_config: + backend: TRTLLM speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 diff --git a/examples/configs/curated/deepseek-r1-throughput.yaml b/examples/configs/curated/deepseek-r1-throughput.yaml index d8ba33cce5c8..13566c7fc258 100644 --- a/examples/configs/curated/deepseek-r1-throughput.yaml +++ b/examples/configs/curated/deepseek-r1-throughput.yaml @@ -6,7 +6,6 @@ trust_remote_code: true enable_attention_dp: true cuda_graph_config: enable_padding: true - max_batch_size: 128 kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.8 diff --git a/examples/configs/curated/gpt-oss-120b-latency.yaml b/examples/configs/curated/gpt-oss-120b-latency.yaml index 4dcaa0314221..6e22e4e1339d 100644 --- a/examples/configs/curated/gpt-oss-120b-latency.yaml +++ b/examples/configs/curated/gpt-oss-120b-latency.yaml @@ -3,9 +3,6 @@ max_num_tokens: 16384 tensor_parallel_size: 8 moe_expert_parallel_size: 1 trust_remote_code: true -enable_attention_dp: false -kv_cache_config: - free_gpu_memory_fraction: 0.9 cuda_graph_config: enable_padding: true max_batch_size: 64 diff --git a/examples/configs/curated/gpt-oss-120b-throughput.yaml b/examples/configs/curated/gpt-oss-120b-throughput.yaml index eb811abcc1db..7d457d0d4f6c 100644 --- a/examples/configs/curated/gpt-oss-120b-throughput.yaml +++ b/examples/configs/curated/gpt-oss-120b-throughput.yaml @@ -4,8 +4,6 @@ tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true enable_attention_dp: true -kv_cache_config: - free_gpu_memory_fraction: 0.9 cuda_graph_config: enable_padding: true max_batch_size: 720 diff --git a/examples/configs/curated/kimi-k2-thinking.yaml b/examples/configs/curated/kimi-k2-thinking.yaml index c3da19a74967..70fe82da7c15 100644 --- a/examples/configs/curated/kimi-k2-thinking.yaml +++ b/examples/configs/curated/kimi-k2-thinking.yaml @@ -4,7 +4,6 @@ max_seq_len: 8212 tensor_parallel_size: 8 moe_expert_parallel_size: 8 enable_attention_dp: true -pipeline_parallel_size: 1 print_iter_log: true kv_cache_config: free_gpu_memory_fraction: 0.75 diff --git a/examples/configs/curated/llama-3.3-70b.yaml b/examples/configs/curated/llama-3.3-70b.yaml index 8020b7f687dc..a921cfb0dcfb 100644 --- a/examples/configs/curated/llama-3.3-70b.yaml +++ b/examples/configs/curated/llama-3.3-70b.yaml @@ -1,12 +1,9 @@ max_batch_size: 1024 max_num_tokens: 2048 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -enable_attention_dp: false cuda_graph_config: enable_padding: true max_batch_size: 1024 kv_cache_config: dtype: fp8 - free_gpu_memory_fraction: 0.9 diff --git a/examples/configs/curated/llama-4-scout.yaml b/examples/configs/curated/llama-4-scout.yaml index 8020b7f687dc..a921cfb0dcfb 100644 --- a/examples/configs/curated/llama-4-scout.yaml +++ b/examples/configs/curated/llama-4-scout.yaml @@ -1,12 +1,9 @@ max_batch_size: 1024 max_num_tokens: 2048 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -enable_attention_dp: false cuda_graph_config: enable_padding: true max_batch_size: 1024 kv_cache_config: dtype: fp8 - free_gpu_memory_fraction: 0.9 diff --git a/examples/configs/curated/lookup.yaml b/examples/configs/curated/lookup.yaml new file mode 100644 index 000000000000..3e66a4d4028e --- /dev/null +++ b/examples/configs/curated/lookup.yaml @@ -0,0 +1,34 @@ +# arch: MODEL_CLASS_MAPPING key; required when model has get_model_defaults. Add when adding entries. +- model: Qwen/Qwen3-Next-80B-A3B-Thinking + arch: Qwen3NextForCausalLM + config_path: examples/configs/curated/qwen3-next.yaml +- model: Qwen/Qwen3-30B-A3B + arch: Qwen3MoeForCausalLM + config_path: examples/configs/curated/qwen3.yaml +- model: Qwen/Qwen3-30B-A3B + arch: Qwen3MoeForCausalLM + config_path: examples/configs/curated/qwen3-disagg-prefill.yaml +- model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/deepseek-r1-latency.yaml +- model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/deepseek-r1-throughput.yaml +- model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/deepseek-r1-deepgemm.yaml +- model: openai/gpt-oss-120b + arch: GptOssForCausalLM + config_path: examples/configs/curated/gpt-oss-120b-latency.yaml +- model: openai/gpt-oss-120b + arch: GptOssForCausalLM + config_path: examples/configs/curated/gpt-oss-120b-throughput.yaml +- model: nvidia/Llama-3.3-70B-Instruct-FP8 + arch: LlamaForCausalLM + config_path: examples/configs/curated/llama-3.3-70b.yaml +- model: nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 + arch: Llama4ForConditionalGeneration + config_path: examples/configs/curated/llama-4-scout.yaml +- model: nvidia/Kimi-K2-Thinking-NVFP4 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/kimi-k2-thinking.yaml diff --git a/examples/configs/curated/qwen3-disagg-prefill.yaml b/examples/configs/curated/qwen3-disagg-prefill.yaml index d69add008012..3e0228c42d6d 100644 --- a/examples/configs/curated/qwen3-disagg-prefill.yaml +++ b/examples/configs/curated/qwen3-disagg-prefill.yaml @@ -1,9 +1,7 @@ max_batch_size: 161 max_num_tokens: 1160 -kv_cache_config: - free_gpu_memory_fraction: 0.8 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -print_iter_log: true enable_attention_dp: true +kv_cache_config: + free_gpu_memory_fraction: 0.8 diff --git a/examples/configs/curated/qwen3-next.yaml b/examples/configs/curated/qwen3-next.yaml index b9aa4f1b63e3..cf29737c5740 100644 --- a/examples/configs/curated/qwen3-next.yaml +++ b/examples/configs/curated/qwen3-next.yaml @@ -3,14 +3,10 @@ max_num_tokens: 4096 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -enable_attention_dp: false cuda_graph_config: enable_padding: true max_batch_size: 720 moe_config: - backend: TRTLLM + backend: TRTLLM stream_interval: 20 num_postprocess_workers: 4 -kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 diff --git a/examples/configs/curated/qwen3.yaml b/examples/configs/curated/qwen3.yaml index 0b54a729f76b..74b8076dff08 100644 --- a/examples/configs/curated/qwen3.yaml +++ b/examples/configs/curated/qwen3.yaml @@ -1,8 +1,5 @@ max_batch_size: 161 max_num_tokens: 1160 -kv_cache_config: - free_gpu_memory_fraction: 0.8 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 cuda_graph_config: enable_padding: true @@ -17,5 +14,6 @@ cuda_graph_config: - 128 - 256 - 384 -print_iter_log: true enable_attention_dp: true +kv_cache_config: + free_gpu_memory_fraction: 0.8 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml index 5fba0289d393..2d2a1e423d79 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml index 2b54bf087b04..ffcfae42cb73 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml @@ -1,6 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 enable_attention_dp: true print_iter_log: true kv_cache_config: @@ -16,6 +15,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/lookup.yaml b/examples/configs/database/lookup.yaml index d1ac7143ceb9..59e257d20b9e 100644 --- a/examples/configs/database/lookup.yaml +++ b/examples/configs/database/lookup.yaml @@ -1,4 +1,5 @@ - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -6,6 +7,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -13,6 +15,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -20,6 +23,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -27,6 +31,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -34,6 +39,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -41,6 +47,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -48,6 +55,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -55,6 +63,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -62,6 +71,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -69,6 +79,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -76,6 +87,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -83,6 +95,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -90,6 +103,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -97,6 +111,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -104,6 +119,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -111,6 +127,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -118,6 +135,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -125,6 +143,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -132,6 +151,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -139,6 +159,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -146,6 +167,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -153,6 +175,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -160,6 +183,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -167,6 +191,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -174,6 +199,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -181,6 +207,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -188,6 +215,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -195,6 +223,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -202,6 +231,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -209,6 +239,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -216,6 +247,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -223,6 +255,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -230,6 +263,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -237,6 +271,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -244,6 +279,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -251,6 +287,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -258,6 +295,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -265,6 +303,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -272,6 +311,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -279,6 +319,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -286,6 +327,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -293,6 +335,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -300,6 +343,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -307,6 +351,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -314,6 +359,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -321,6 +367,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -328,6 +375,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -335,6 +383,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -342,6 +391,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -349,6 +399,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -356,6 +407,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -363,6 +415,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -370,6 +423,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -377,6 +431,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -384,6 +439,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -391,6 +447,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -398,6 +455,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -405,6 +463,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -412,6 +471,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -419,6 +479,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -426,6 +487,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -433,6 +495,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -440,6 +503,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -447,6 +511,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -454,6 +519,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -461,6 +527,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -468,6 +535,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -475,6 +543,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -482,6 +551,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -489,6 +559,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -496,6 +567,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -503,6 +575,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -510,6 +583,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -517,6 +591,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -524,6 +599,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -531,6 +607,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -538,6 +615,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -545,6 +623,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -552,6 +631,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -559,6 +639,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -566,6 +647,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -573,6 +655,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -580,6 +663,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -587,6 +671,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -594,6 +679,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -601,6 +687,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -608,6 +695,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -615,6 +703,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -622,6 +711,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -629,6 +719,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -636,6 +727,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -643,6 +735,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -650,6 +743,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -657,6 +751,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -664,6 +759,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -671,6 +767,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -678,6 +775,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -685,6 +783,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -692,6 +791,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -699,6 +799,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -706,6 +807,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -713,6 +815,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -720,6 +823,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -727,6 +831,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -734,6 +839,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -741,6 +847,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -748,6 +855,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -755,6 +863,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -762,6 +871,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -769,6 +879,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -776,6 +887,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -783,6 +895,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -790,6 +903,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -797,6 +911,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -804,6 +919,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -811,6 +927,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -818,6 +935,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -825,6 +943,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -832,6 +951,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -839,6 +959,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -846,6 +967,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -853,6 +975,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -860,6 +983,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -867,6 +991,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -874,6 +999,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -881,6 +1007,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -888,6 +1015,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -895,6 +1023,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -902,6 +1031,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -909,6 +1039,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -916,6 +1047,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -923,6 +1055,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -930,6 +1063,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -937,6 +1071,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -944,6 +1079,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -951,6 +1087,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -958,6 +1095,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -965,6 +1103,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -972,6 +1111,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -979,6 +1119,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -986,6 +1127,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -993,6 +1135,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1000,6 +1143,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1007,6 +1151,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1014,6 +1159,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1021,6 +1167,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1028,6 +1175,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1035,6 +1183,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1042,6 +1191,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1049,6 +1199,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1056,6 +1207,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1063,6 +1215,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1070,6 +1223,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1077,6 +1231,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1084,6 +1239,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1091,6 +1247,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1098,6 +1255,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1105,6 +1263,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1112,6 +1271,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1119,6 +1279,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1126,6 +1287,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1133,6 +1295,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1140,6 +1303,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1147,6 +1311,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1154,6 +1319,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1161,6 +1327,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1168,6 +1335,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml index d86cb71568e2..0faad6ed89bf 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1216 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml index 18fbe5eec97e..fd1718aaf981 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1344 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml index 696ad9bb03e6..824e7e6b4607 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1216 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml index 86707e68fb93..eeeac04724ea 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1344 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml index 6d881085020f..e36e4f7e551c 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8384 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml index bff28a8fa5ff..1bfc9ec9573c 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8512 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml index 17ca6555ef18..46084314e82d 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml index cfff9caf5e2d..d6d278f0797f 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8384 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml index f4488125d239..43f58836ca57 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8512 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml index eb2fb249d137..83dea42f5566 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml index 323d5260e365..867425e8464d 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml index 4e42c74c5dcb..5ac815532384 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml index 41a3501db86b..a8067336e1c8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml index ef8559949b9f..5867861391db 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml index cf3f1435815f..ba8755ccba8b 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml index 89f16c083483..38875d1078b0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml index 50a9a01eabd9..2c7f0513e54a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml index 9e38bf35d192..71eccd39becb 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml index 828ad265824c..d01d07cf84ac 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml index 27a1f1b0a513..d16eb6bbdcf2 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml index d85b3d01a4ee..e0ac7cc6a25c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml index 9ed994fa4bf7..522acd56cc4e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml index f7d77d7296d4..08f4fcb8d9ba 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml index 8d4215121a62..1472917945d9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml index 0c0101c83287..3bd661c6b439 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml index dda82a593e4d..9520ba0219ce 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml index c7700969144b..f49d781dd0f8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml index 55e10c0d157f..2e3b2430549d 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml index f606123c32f4..883c0fcd2266 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml index 2d284fcf8cff..68f52a366a93 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml index b17f7425cc6b..f136968130d1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml index e559dcc0b8b8..2fa09ee400d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml index af72d4f4949b..50c1b7062477 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml index 3e3f969fc72f..e9319d9a7e35 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml index aef4e1636b28..e42f06bc2138 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml index 879c2fba7094..4bd4b9377fbd 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml index 322f73709e86..5e12525cc48a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml index 84cac88c8643..87877d6280d0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml index 031a0fd74482..ae6938feba4c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml index 445cec1d82cf..95fb72d0a9d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml index 4d9ce00dbcdb..58e92ed78cd6 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml index e790e0d98029..2c0b0d06df83 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml index f72680460cef..5e84439ebcb0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml index ddceeb61fa0c..3e4f7ddb7f2e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml index f08dca8cd190..a8446c8e35da 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml index 95cdf3a61f7b..b936b317e0db 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml index 74d4e29f677f..630aa9ab0aff 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml index 59b9319ee8e0..4c2f9d688669 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml index 365f467d6e43..b7ef6b8d2516 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml index befde425ae46..7ce727c75571 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml index b17f7425cc6b..f136968130d1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml index e559dcc0b8b8..2fa09ee400d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml index af72d4f4949b..50c1b7062477 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml index 3e3f969fc72f..e9319d9a7e35 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml index aef4e1636b28..e42f06bc2138 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml index 879c2fba7094..4bd4b9377fbd 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml index 322f73709e86..5e12525cc48a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml index 84cac88c8643..87877d6280d0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml index 031a0fd74482..ae6938feba4c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml index 445cec1d82cf..95fb72d0a9d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml index 4d9ce00dbcdb..58e92ed78cd6 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml index e790e0d98029..2c0b0d06df83 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml index f72680460cef..5e84439ebcb0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml index ddceeb61fa0c..3e4f7ddb7f2e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml index f08dca8cd190..a8446c8e35da 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml index 95cdf3a61f7b..b936b317e0db 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml index 74d4e29f677f..630aa9ab0aff 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml index 59b9319ee8e0..4c2f9d688669 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml index 365f467d6e43..b7ef6b8d2516 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml index befde425ae46..7ce727c75571 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml index 6736070cc707..595e44706bbb 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml index a6fec264e879..4cf0e25a5239 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml index c98b557f7963..de9e3357e084 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml index 8144ce5b4d20..057f95b9c800 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml index 45864a956ea3..81f16ee5fc12 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml index 4762e577704f..80822a662b7a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml index 3a16da2d4a45..2b0bc6e46096 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml index 2b789c4add66..d0d0b9fef405 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml index 94c40f2ac280..b95889fc6b72 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml index 2b866099d81b..d536457d4449 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml index ff543422a0be..7cdcf64adaf9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml index 95477ae879dd..891d01c63a9e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml index dca7e5a63a1f..11079f4187f8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml index 1b2ad812edad..b209e1d39fae 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml index 0d1ee33fee14..5f589609ba07 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml index 142a9c07af7b..aaca4f4f67f4 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml index c42c0506e1d5..d6009857f7bc 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml index 161ebf4cdb60..888c079af725 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml index f7f9a1b76ddc..a0ed865c4a1e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml index 0c9084d25cac..7b3323f0f218 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml index 466565e489df..3d6ddd08f9f1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml index 4f0632784c88..74343e0022e8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml index 57f3c1547c6f..88174dc7920a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml index e61ad0b3de98..170efc76914a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml index 53b5461cf467..a554dc2bb91e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml index 41d9a88b2007..e6d7028862a3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml index 5bd3aa86c7ca..c3e23e3c2239 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml index 90854c5fbdc3..331ee8ed0c56 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml index d16f3ecc9544..49a223b458f0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml index 36171627426d..b6c71c91cb1a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml index bb3a41bc2fca..43c9fee6976b 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml index 1dd6379a4e8e..9ad1a65df871 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml index e63805377ed7..a606aabd9cf1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml index 52be12ec84d6..b8a12b4ea250 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml index a80d8f26abf8..269424525ced 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml index 4c416c78bb3d..ee0548ee1630 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml index eade98620983..d1a258c6b91c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml index 24916df78a24..b2560d08767e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml index a609e99a880b..f8423b90df1e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml index 9c37e6359dc7..a8ca13ef61c9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml index 466565e489df..3d6ddd08f9f1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml index 4f0632784c88..74343e0022e8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml index 57f3c1547c6f..88174dc7920a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml index e61ad0b3de98..170efc76914a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml index 53b5461cf467..a554dc2bb91e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml index 41d9a88b2007..e6d7028862a3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml index 5bd3aa86c7ca..c3e23e3c2239 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml index 90854c5fbdc3..331ee8ed0c56 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml index d16f3ecc9544..49a223b458f0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml index 36171627426d..b6c71c91cb1a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml index bb3a41bc2fca..43c9fee6976b 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml index 1dd6379a4e8e..9ad1a65df871 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml index e63805377ed7..a606aabd9cf1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml index 52be12ec84d6..b8a12b4ea250 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml index a80d8f26abf8..269424525ced 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml index 4c416c78bb3d..ee0548ee1630 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml index eade98620983..d1a258c6b91c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml index 24916df78a24..b2560d08767e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml index a609e99a880b..f8423b90df1e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml index 9c37e6359dc7..a8ca13ef61c9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 4084481ef2cc..aa1906b352e4 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,62 +12,210 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""L0 tests for validating config database YAML files against TorchLlmArgs.""" +"""L0 tests for validating curated and database YAML configs against TorchLlmArgs.""" from pathlib import Path -from unittest.mock import Mock, patch +from unittest import mock import pytest import yaml -from tensorrt_llm.llmapi.llm_args import TorchLlmArgs, update_llm_args_with_extra_dict +from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING +from tensorrt_llm.commands.serve import main as serve_main +from tensorrt_llm.llmapi import llm_args as llm_args_module + +from .yaml_validation_harness import ( + assert_no_default_valued_leaves, + collect_yaml_files, + load_yaml_dict, + mock_cuda_for_schema_validation, + validate_torch_llm_args_config, +) CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" +REPO_ROOT = CONFIG_ROOT.parent.parent +CURATED_DIR = CONFIG_ROOT / "curated" DATABASE_DIR = CONFIG_ROOT / "database" -DATABASE_CONFIGS = ( - [c for c in DATABASE_DIR.rglob("*.yaml") if c.name != "lookup.yaml"] - if DATABASE_DIR.exists() - else [] -) +CURATED_CONFIGS = collect_yaml_files(CURATED_DIR, "**/*.yaml", exclude_names={"lookup.yaml"}) +DATABASE_CONFIGS = collect_yaml_files(DATABASE_DIR, "**/*.yaml", exclude_names={"lookup.yaml"}) +ALL_CONFIGS = sorted(CURATED_CONFIGS + DATABASE_CONFIGS) +ALL_LOOKUP_PATHS = sorted((DATABASE_DIR / "lookup.yaml", CURATED_DIR / "lookup.yaml")) + + +def _load_config_path_to_lookup_entry() -> dict[str, dict]: + """Build config_path -> {model, arch?} from database and curated lookup.yaml files.""" + result = {} + for lookup_path in ALL_LOOKUP_PATHS: + if not lookup_path.exists(): + continue + with open(lookup_path, encoding="utf-8") as f: + entries = yaml.safe_load(f) + if not entries: + continue + for entry in entries: + if not isinstance(entry, dict) or "config_path" not in entry: + continue + key = entry["config_path"] + result[key] = {"model": entry.get("model"), "arch": entry.get("arch")} + return result + + +_CONFIG_PATH_TO_ENTRY = _load_config_path_to_lookup_entry() + + +def _get_default_values_for_config(config_path: Path) -> dict: + """Get the default values for a config path. + + Some models may have custom default values that are different from the global defaults. + Based on the model architecture, get the default values that are actually applied. + """ + try: + config_key = config_path.relative_to(REPO_ROOT).as_posix() + except ValueError: + config_key = None + entry = _CONFIG_PATH_TO_ENTRY.get(config_key) if config_key else None + global_default = llm_args_module.TorchLlmArgs( + model="dummy/model", skip_tokenizer_init=True + ).model_dump(mode="json") + if not entry or not entry.get("arch") or not entry.get("model"): + return global_default + + model = entry["model"] + arch = entry["arch"] + model_cls = MODEL_CLASS_MAPPING.get(arch) + if not model_cls or not hasattr(model_cls, "get_model_defaults"): + return global_default + + base_args = llm_args_module.TorchLlmArgs(model=model, skip_tokenizer_init=True).model_dump( + mode="json" + ) + + model_defaults = model_cls.get_model_defaults(base_args) + base_args.update(model_defaults) + return base_args @pytest.fixture(autouse=True) def mock_gpu_environment(): - """Mock GPU functions for CPU-only test execution.""" - mock_props = Mock() - mock_props.major = 8 - - with patch("torch.cuda.device_count", return_value=8): - with patch("torch.cuda.get_device_properties", return_value=mock_props): - with patch("torch.cuda.is_available", return_value=True): - yield + """Mock GPU functions for CPU-only schema test execution.""" + with mock_cuda_for_schema_validation(): + yield def get_config_id(config_path: Path) -> str: - return str(config_path.relative_to(DATABASE_DIR)) + return str(config_path.relative_to(CONFIG_ROOT)) + + +def _assert_kv_cache_block_reuse_policy(config_dict: dict) -> None: + assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False, ( + "KV cache block reuse should not be disabled" + ) + + +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_validates_against_llm_args(config_path: Path): + config_dict = load_yaml_dict(config_path) + validate_torch_llm_args_config(config_dict) + + +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_does_not_disable_kv_cache_block_reuse(config_path: Path): + config_dict = load_yaml_dict(config_path) + _assert_kv_cache_block_reuse_policy(config_dict) @pytest.mark.part0 -@pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) -def test_config_validates_against_llm_args(config_path: Path): - with open(config_path) as f: - config_dict = yaml.safe_load(f) or {} +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_does_not_set_default_leaves(config_path: Path): + config_dict = load_yaml_dict(config_path) + default_cfg = _get_default_values_for_config(config_path) + assert_no_default_valued_leaves(config_dict, default_cfg) - base_args = TorchLlmArgs(model="dummy/model", skip_tokenizer_init=True) - merged = update_llm_args_with_extra_dict(base_args.model_dump(), config_dict) - TorchLlmArgs(**merged) + +@pytest.mark.part0 +def test_database_yaml_config_count(): + assert len(ALL_CONFIGS) > 0, "No curated or database config files found" @pytest.mark.part0 -@pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) -def test_config_does_not_disable_kv_cache_block_reuse(config_path: Path): - with open(config_path) as f: - config_dict = yaml.safe_load(f) or {} +def test_all_configs_have_lookup_entry(): + """Every config in ALL_CONFIGS must have an entry in the lookup files (ALL_LOOKUP_PATHS).""" + config_keys = {p.relative_to(REPO_ROOT).as_posix() for p in ALL_CONFIGS} + lookup_keys = set(_CONFIG_PATH_TO_ENTRY) + missing = config_keys - lookup_keys + assert not missing, ( + "The following configs have no entry in the lookup files. " + "Add each to the appropriate lookup.yaml file:\n" + "\n".join(sorted(missing)) + ) - assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False + +@pytest.mark.part0 +def test_all_lookup_entries_have_arch(): + """Every entry in the lookup files must have the required 'arch' field.""" + entries_missing_arch = [] + for lookup_path in ALL_LOOKUP_PATHS: + if not lookup_path.exists(): + continue + with open(lookup_path, encoding="utf-8") as f: + entries = yaml.safe_load(f) + if not entries: + continue + for entry in entries: + if not isinstance(entry, dict) or "config_path" not in entry: + continue + config_path = entry.get("config_path", "") + if not entry.get("arch"): + entries_missing_arch.append( + f" {lookup_path.relative_to(REPO_ROOT)}: {config_path}" + ) + assert not entries_missing_arch, ( + "The following lookup entries are missing the required 'arch' field. " + "Add 'arch: ' to each entry:\n" + + "\n".join(sorted(entries_missing_arch)) + ) + + +def _serve_cli_args(config_path: Path, port: int = 17999): + """CLI argv for serve, like: trtllm-serve --config --port .""" + return [ + "dummy/model", + "--config", + str(config_path), + "--port", + str(port), + ] + + +async def _noop_serve(_host, _port, _sockets=None): + pass + + +class _MockOpenAIServer: + def __init__(self, generator, model, **kwargs): + self.generator = generator + self.model = model + + def __call__(self, host, port, sockets=None): + return _noop_serve(host, port, sockets) @pytest.mark.part0 -def test_database_config_count(): - assert len(DATABASE_CONFIGS) > 0, "No database config files found" +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_serve_cli(config_path: Path): + """Invoke serve via CLI (as a user would); Click supplies defaults; server/LLM mocked.""" + mock_llm = mock.Mock() + mock_pytorch_llm = mock.Mock(return_value=mock_llm) + + with ( + mock.patch("tensorrt_llm.commands.serve.get_is_diffusion_model", return_value=False), + mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1), + mock.patch("tensorrt_llm.commands.serve.PyTorchLLM", mock_pytorch_llm), + mock.patch("tensorrt_llm.commands.serve.OpenAIServer", _MockOpenAIServer), + ): + serve_main(args=_serve_cli_args(config_path), standalone_mode=False) + mock_pytorch_llm.assert_called_once() + call_kwargs = mock_pytorch_llm.call_args[1] + llm_args_module.TorchLlmArgs(**call_kwargs) diff --git a/tests/unittest/llmapi/yaml_validation_harness.py b/tests/unittest/llmapi/yaml_validation_harness.py new file mode 100644 index 000000000000..b6181e81a860 --- /dev/null +++ b/tests/unittest/llmapi/yaml_validation_harness.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared YAML validation helpers for llmapi config tests.""" + +from __future__ import annotations + +import copy +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from unittest import mock + +import yaml + +from tensorrt_llm.llmapi import llm_args as llm_args_module + + +def collect_yaml_files( + parent_dir: Path, include_glob: str, exclude_names: Iterable[str] = () +) -> list[Path]: + excluded = set(exclude_names) + if not parent_dir.exists(): + return [] + + return sorted( + path + for path in parent_dir.glob(include_glob) + if path.is_file() and path.name not in excluded + ) + + +def load_yaml_dict(path: Path) -> dict[str, Any]: + with open(path, encoding="utf-8") as handle: + loaded = yaml.safe_load(handle) + + if loaded is None: + return {} + if not isinstance(loaded, dict): + raise TypeError(f"YAML at {path} must parse to a dict, got {type(loaded).__name__}") + return loaded + + +def validate_torch_llm_args_config(cfg: dict[str, Any]) -> llm_args_module.TorchLlmArgs: + base_args = llm_args_module.TorchLlmArgs(model="dummy/model", skip_tokenizer_init=True) + merged = llm_args_module.update_llm_args_with_extra_dict( + # NOTE: using model_dump with mode='python' so enums (e.g. load_format) stay as enum, + # not int. + copy.deepcopy(base_args.model_dump()), + copy.deepcopy(cfg), + ) + return llm_args_module.TorchLlmArgs(**merged) + + +def assert_no_default_valued_leaves( + cfg: dict[str, Any], + default_cfg: dict[str, Any], + allowlist: Iterable[str | tuple[str, ...]] = (), +) -> None: + normalized_allowlist: set[tuple[str, ...]] = set() + for item in allowlist: + if isinstance(item, str): + normalized_allowlist.add(tuple(item.split("."))) + else: + normalized_allowlist.add(tuple(item)) + + violations: list[str] = [] + + def walk(candidate: dict[str, Any], defaults: dict[str, Any], path: tuple[str, ...]) -> None: + for key, value in candidate.items(): + full_path = path + (str(key),) + if full_path in normalized_allowlist: + continue + + if key not in defaults: + continue + default_value = defaults[key] + + if isinstance(value, dict) and isinstance(default_value, dict): + walk(value, default_value, full_path) + continue + + if value == default_value: + violations.append(".".join(full_path)) + + walk(cfg, default_cfg, ()) + if violations: + raise AssertionError( + "Found default-valued config leaves:\n" + "\n".join(sorted(violations)) + ) + + +@contextmanager +def mock_cuda_for_schema_validation( + device_count: int = 8, compute_capability_major: int = 8, is_available: bool = True +) -> Iterator[None]: + mock_props = mock.Mock() + mock_props.major = compute_capability_major + + with mock.patch("torch.cuda.device_count", return_value=device_count): + with mock.patch("torch.cuda.get_device_properties", return_value=mock_props): + with mock.patch("torch.cuda.is_available", return_value=is_available): + yield