From d6badd3657e3a402909f3121d2b55857d6074d60 Mon Sep 17 00:00:00 2001 From: Venky <23023424+venkywonka@users.noreply.github.com> Date: Thu, 19 Feb 2026 08:25:08 -0800 Subject: [PATCH 01/12] [None][test] unify yaml validation harness for curated and database configs Signed-off-by: Venky <23023424+venkywonka@users.noreply.github.com> --- .../configs/curated/deepseek-r1-deepgemm.yaml | 3 +- .../configs/curated/deepseek-r1-latency.yaml | 7 +- .../curated/deepseek-r1-throughput.yaml | 3 +- .../configs/curated/gpt-oss-120b-latency.yaml | 2 - .../curated/gpt-oss-120b-throughput.yaml | 1 - examples/configs/curated/llama-3.3-70b.yaml | 3 - examples/configs/curated/llama-4-scout.yaml | 3 - .../configs/curated/qwen3-disagg-prefill.yaml | 4 +- examples/configs/curated/qwen3-next.yaml | 4 +- examples/configs/curated/qwen3.yaml | 4 +- tests/unittest/llmapi/test_config_database.py | 39 ++--- .../llmapi/test_curated_yaml_configs.py | 74 ++++++++++ .../llmapi/yaml_validation_harness.py | 134 ++++++++++++++++++ 13 files changed, 233 insertions(+), 48 deletions(-) create mode 100644 tests/unittest/llmapi/test_curated_yaml_configs.py create mode 100644 tests/unittest/llmapi/yaml_validation_harness.py diff --git a/examples/configs/curated/deepseek-r1-deepgemm.yaml b/examples/configs/curated/deepseek-r1-deepgemm.yaml index bc12f12b4522..3c00b146e160 100644 --- a/examples/configs/curated/deepseek-r1-deepgemm.yaml +++ b/examples/configs/curated/deepseek-r1-deepgemm.yaml @@ -1,15 +1,14 @@ max_batch_size: 1024 max_num_tokens: 3200 -kv_cache_free_gpu_memory_fraction: 0.8 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true enable_attention_dp: true cuda_graph_config: enable_padding: true - max_batch_size: 128 kv_cache_config: dtype: fp8 + free_gpu_memory_fraction: 0.8 stream_interval: 10 speculative_config: decoding_type: MTP diff --git a/examples/configs/curated/deepseek-r1-latency.yaml b/examples/configs/curated/deepseek-r1-latency.yaml index 80aaedc8b50f..35ce5864dbaf 100644 --- a/examples/configs/curated/deepseek-r1-latency.yaml +++ b/examples/configs/curated/deepseek-r1-latency.yaml @@ -3,9 +3,10 @@ tensor_parallel_size: 8 moe_expert_parallel_size: 2 max_num_tokens: 32768 trust_remote_code: true -kv_cache_free_gpu_memory_fraction: 0.75 -moe_backend: TRTLLM -use_cuda_graph: true +kv_cache_config: + free_gpu_memory_fraction: 0.75 +moe_config: + backend: TRTLLM speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 diff --git a/examples/configs/curated/deepseek-r1-throughput.yaml b/examples/configs/curated/deepseek-r1-throughput.yaml index 4e59d9acb241..13566c7fc258 100644 --- a/examples/configs/curated/deepseek-r1-throughput.yaml +++ b/examples/configs/curated/deepseek-r1-throughput.yaml @@ -1,15 +1,14 @@ max_batch_size: 1024 max_num_tokens: 3200 -kv_cache_free_gpu_memory_fraction: 0.8 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true enable_attention_dp: true cuda_graph_config: enable_padding: true - max_batch_size: 128 kv_cache_config: dtype: fp8 + free_gpu_memory_fraction: 0.8 stream_interval: 10 speculative_config: decoding_type: MTP diff --git a/examples/configs/curated/gpt-oss-120b-latency.yaml b/examples/configs/curated/gpt-oss-120b-latency.yaml index 225081c364fe..6e22e4e1339d 100644 --- a/examples/configs/curated/gpt-oss-120b-latency.yaml +++ b/examples/configs/curated/gpt-oss-120b-latency.yaml @@ -1,10 +1,8 @@ max_batch_size: 64 max_num_tokens: 16384 -kv_cache_free_gpu_memory_fraction: 0.9 tensor_parallel_size: 8 moe_expert_parallel_size: 1 trust_remote_code: true -enable_attention_dp: false cuda_graph_config: enable_padding: true max_batch_size: 64 diff --git a/examples/configs/curated/gpt-oss-120b-throughput.yaml b/examples/configs/curated/gpt-oss-120b-throughput.yaml index 4d4189837437..7d457d0d4f6c 100644 --- a/examples/configs/curated/gpt-oss-120b-throughput.yaml +++ b/examples/configs/curated/gpt-oss-120b-throughput.yaml @@ -1,6 +1,5 @@ max_batch_size: 720 # Depends on max_sequence_length max_num_tokens: 16384 -kv_cache_free_gpu_memory_fraction: 0.9 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true diff --git a/examples/configs/curated/llama-3.3-70b.yaml b/examples/configs/curated/llama-3.3-70b.yaml index 8887bd2955f1..a921cfb0dcfb 100644 --- a/examples/configs/curated/llama-3.3-70b.yaml +++ b/examples/configs/curated/llama-3.3-70b.yaml @@ -1,10 +1,7 @@ max_batch_size: 1024 max_num_tokens: 2048 -kv_cache_free_gpu_memory_fraction: 0.9 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -enable_attention_dp: false cuda_graph_config: enable_padding: true max_batch_size: 1024 diff --git a/examples/configs/curated/llama-4-scout.yaml b/examples/configs/curated/llama-4-scout.yaml index 8887bd2955f1..a921cfb0dcfb 100644 --- a/examples/configs/curated/llama-4-scout.yaml +++ b/examples/configs/curated/llama-4-scout.yaml @@ -1,10 +1,7 @@ max_batch_size: 1024 max_num_tokens: 2048 -kv_cache_free_gpu_memory_fraction: 0.9 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -enable_attention_dp: false cuda_graph_config: enable_padding: true max_batch_size: 1024 diff --git a/examples/configs/curated/qwen3-disagg-prefill.yaml b/examples/configs/curated/qwen3-disagg-prefill.yaml index 93de3e7cf5dc..0404ee51a236 100644 --- a/examples/configs/curated/qwen3-disagg-prefill.yaml +++ b/examples/configs/curated/qwen3-disagg-prefill.yaml @@ -1,8 +1,8 @@ max_batch_size: 161 max_num_tokens: 1160 -kv_cache_free_gpu_memory_fraction: 0.8 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true print_iter_log: true enable_attention_dp: true +kv_cache_config: + free_gpu_memory_fraction: 0.8 diff --git a/examples/configs/curated/qwen3-next.yaml b/examples/configs/curated/qwen3-next.yaml index b9aa4f1b63e3..7a149cf34ca6 100644 --- a/examples/configs/curated/qwen3-next.yaml +++ b/examples/configs/curated/qwen3-next.yaml @@ -3,14 +3,12 @@ max_num_tokens: 4096 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -enable_attention_dp: false cuda_graph_config: enable_padding: true max_batch_size: 720 moe_config: - backend: TRTLLM + backend: TRTLLM stream_interval: 20 num_postprocess_workers: 4 kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.9 diff --git a/examples/configs/curated/qwen3.yaml b/examples/configs/curated/qwen3.yaml index c47d4904c337..55a38602d59b 100644 --- a/examples/configs/curated/qwen3.yaml +++ b/examples/configs/curated/qwen3.yaml @@ -1,7 +1,5 @@ max_batch_size: 161 max_num_tokens: 1160 -kv_cache_free_gpu_memory_fraction: 0.8 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 cuda_graph_config: enable_padding: true @@ -18,3 +16,5 @@ cuda_graph_config: - 384 print_iter_log: true enable_attention_dp: true +kv_cache_config: + free_gpu_memory_fraction: 0.8 diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 4084481ef2cc..67944856d942 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,36 +12,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""L0 tests for validating config database YAML files against TorchLlmArgs.""" +"""L0 wrapper tests for validating database YAML files against TorchLlmArgs.""" from pathlib import Path -from unittest.mock import Mock, patch import pytest -import yaml -from tensorrt_llm.llmapi.llm_args import TorchLlmArgs, update_llm_args_with_extra_dict +from . import yaml_validation_harness as yaml_harness CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" DATABASE_DIR = CONFIG_ROOT / "database" -DATABASE_CONFIGS = ( - [c for c in DATABASE_DIR.rglob("*.yaml") if c.name != "lookup.yaml"] - if DATABASE_DIR.exists() - else [] +DATABASE_CONFIGS = yaml_harness.collect_yaml_files( + DATABASE_DIR, "**/*.yaml", exclude_names={"lookup.yaml"} ) @pytest.fixture(autouse=True) def mock_gpu_environment(): """Mock GPU functions for CPU-only test execution.""" - mock_props = Mock() - mock_props.major = 8 - - with patch("torch.cuda.device_count", return_value=8): - with patch("torch.cuda.get_device_properties", return_value=mock_props): - with patch("torch.cuda.is_available", return_value=True): - yield + with yaml_harness.mock_cuda_for_schema_validation(): + yield def get_config_id(config_path: Path) -> str: @@ -51,21 +42,19 @@ def get_config_id(config_path: Path) -> str: @pytest.mark.part0 @pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) def test_config_validates_against_llm_args(config_path: Path): - with open(config_path) as f: - config_dict = yaml.safe_load(f) or {} + config_dict = yaml_harness.load_yaml_dict(config_path) + yaml_harness.validate_torch_llm_args_config(config_dict) + - base_args = TorchLlmArgs(model="dummy/model", skip_tokenizer_init=True) - merged = update_llm_args_with_extra_dict(base_args.model_dump(), config_dict) - TorchLlmArgs(**merged) +def _assert_kv_cache_block_reuse_policy(config_dict: dict) -> None: + assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False @pytest.mark.part0 @pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) def test_config_does_not_disable_kv_cache_block_reuse(config_path: Path): - with open(config_path) as f: - config_dict = yaml.safe_load(f) or {} - - assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False + config_dict = yaml_harness.load_yaml_dict(config_path) + yaml_harness.assert_custom_policy(config_dict, _assert_kv_cache_block_reuse_policy) @pytest.mark.part0 diff --git a/tests/unittest/llmapi/test_curated_yaml_configs.py b/tests/unittest/llmapi/test_curated_yaml_configs.py new file mode 100644 index 000000000000..2f91e6c78cd8 --- /dev/null +++ b/tests/unittest/llmapi/test_curated_yaml_configs.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""L0 tests for validating curated YAML files against TorchLlmArgs.""" + +from pathlib import Path + +import pytest + +from tensorrt_llm.llmapi import llm_args as llm_args_module + +from . import yaml_validation_harness as yaml_harness + +CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" +CURATED_DIR = CONFIG_ROOT / "curated" + +CURATED_CONFIGS = yaml_harness.collect_yaml_files(CURATED_DIR, "**/*.yaml") + +DEPRECATED_KEY_MAP = { + "kv_cache_free_gpu_memory_fraction": "kv_cache_config.free_gpu_memory_fraction", + "moe_backend": "moe_config.backend", + "use_cuda_graph": "cuda_graph_config", +} + + +@pytest.fixture(autouse=True) +def mock_gpu_environment(): + """Mock GPU functions for CPU-only schema test execution.""" + with yaml_harness.mock_cuda_for_schema_validation(): + yield + + +def get_config_id(config_path: Path) -> str: + return str(config_path.relative_to(CURATED_DIR)) + + +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) +def test_curated_config_validates_against_llm_args(config_path: Path): + config_dict = yaml_harness.load_yaml_dict(config_path) + yaml_harness.validate_torch_llm_args_config(config_dict) + + +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) +def test_curated_config_does_not_use_deprecated_keys(config_path: Path): + config_dict = yaml_harness.load_yaml_dict(config_path) + yaml_harness.assert_no_deprecated_keys(config_dict, DEPRECATED_KEY_MAP) + + +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) +def test_curated_config_does_not_set_default_leaves(config_path: Path): + config_dict = yaml_harness.load_yaml_dict(config_path) + default_cfg = llm_args_module.TorchLlmArgs( + model="dummy/model", skip_tokenizer_init=True + ).model_dump(mode="json") + yaml_harness.assert_no_default_valued_leaves(config_dict, default_cfg) + + +@pytest.mark.part0 +def test_curated_config_count(): + assert len(CURATED_CONFIGS) > 0, "No curated config files found" diff --git a/tests/unittest/llmapi/yaml_validation_harness.py b/tests/unittest/llmapi/yaml_validation_harness.py new file mode 100644 index 000000000000..ad0c3c25de10 --- /dev/null +++ b/tests/unittest/llmapi/yaml_validation_harness.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared YAML validation helpers for llmapi config tests.""" + +from __future__ import annotations + +import copy +from collections.abc import Callable, Iterable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from unittest import mock + +import yaml + +from tensorrt_llm.llmapi import llm_args as llm_args_module + + +def collect_yaml_files( + parent_dir: Path, include_glob: str, exclude_names: Iterable[str] = () +) -> list[Path]: + excluded = set(exclude_names) + if not parent_dir.exists(): + return [] + + return sorted( + path + for path in parent_dir.glob(include_glob) + if path.is_file() and path.name not in excluded + ) + + +def load_yaml_dict(path: Path) -> dict[str, Any]: + with open(path, encoding="utf-8") as handle: + loaded = yaml.safe_load(handle) + + if loaded is None: + return {} + if not isinstance(loaded, dict): + raise TypeError(f"YAML at {path} must parse to a dict, got {type(loaded).__name__}") + return loaded + + +def validate_torch_llm_args_config(cfg: dict[str, Any]) -> llm_args_module.TorchLlmArgs: + base_args = llm_args_module.TorchLlmArgs(model="dummy/model", skip_tokenizer_init=True) + merged = llm_args_module.update_llm_args_with_extra_dict( + copy.deepcopy(base_args.model_dump(mode="json")), copy.deepcopy(cfg) + ) + return llm_args_module.TorchLlmArgs(**merged) + + +def assert_no_deprecated_keys(cfg: dict[str, Any], deprecated_map: dict[str, str]) -> None: + violations: list[str] = [] + + def walk_dict(node: dict[str, Any], path: tuple[str, ...]) -> None: + for key, value in node.items(): + full_path = path + (str(key),) + if key in deprecated_map: + replacement = deprecated_map[key] + violations.append(f"{'.'.join(full_path)} -> {replacement}") + + if isinstance(value, dict): + walk_dict(value, full_path) + + walk_dict(cfg, ()) + if violations: + raise AssertionError("Found deprecated config keys:\n" + "\n".join(sorted(violations))) + + +def assert_no_default_valued_leaves( + cfg: dict[str, Any], + default_cfg: dict[str, Any], + allowlist: Iterable[str | tuple[str, ...]] = (), +) -> None: + normalized_allowlist: set[tuple[str, ...]] = set() + for item in allowlist: + if isinstance(item, str): + normalized_allowlist.add(tuple(item.split("."))) + else: + normalized_allowlist.add(tuple(item)) + + violations: list[str] = [] + + def walk(candidate: dict[str, Any], defaults: dict[str, Any], path: tuple[str, ...]) -> None: + for key, value in candidate.items(): + full_path = path + (str(key),) + if full_path in normalized_allowlist: + continue + + if key not in defaults: + continue + default_value = defaults[key] + + if isinstance(value, dict) and isinstance(default_value, dict): + walk(value, default_value, full_path) + continue + + if value == default_value: + violations.append(".".join(full_path)) + + walk(cfg, default_cfg, ()) + if violations: + raise AssertionError( + "Found default-valued config leaves:\n" + "\n".join(sorted(violations)) + ) + + +def assert_custom_policy(cfg: dict[str, Any], policy_fn: Callable[[dict[str, Any]], None]) -> None: + policy_fn(cfg) + + +@contextmanager +def mock_cuda_for_schema_validation( + device_count: int = 8, compute_capability_major: int = 8, is_available: bool = True +) -> Iterator[None]: + mock_props = mock.Mock() + mock_props.major = compute_capability_major + + with mock.patch("torch.cuda.device_count", return_value=device_count): + with mock.patch("torch.cuda.get_device_properties", return_value=mock_props): + with mock.patch("torch.cuda.is_available", return_value=is_available): + yield From ac98aac7e21e250f80d589556e12a5a7ec68f151 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Thu, 19 Feb 2026 18:33:36 +0100 Subject: [PATCH 02/12] [None][chore] Removing print_iter_log from curated configs. Signed-off-by: Fadi Saady --- examples/configs/curated/qwen3-disagg-prefill.yaml | 1 - examples/configs/curated/qwen3.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/examples/configs/curated/qwen3-disagg-prefill.yaml b/examples/configs/curated/qwen3-disagg-prefill.yaml index 0404ee51a236..3e0228c42d6d 100644 --- a/examples/configs/curated/qwen3-disagg-prefill.yaml +++ b/examples/configs/curated/qwen3-disagg-prefill.yaml @@ -2,7 +2,6 @@ max_batch_size: 161 max_num_tokens: 1160 moe_expert_parallel_size: 1 trust_remote_code: true -print_iter_log: true enable_attention_dp: true kv_cache_config: free_gpu_memory_fraction: 0.8 diff --git a/examples/configs/curated/qwen3.yaml b/examples/configs/curated/qwen3.yaml index 55a38602d59b..74b8076dff08 100644 --- a/examples/configs/curated/qwen3.yaml +++ b/examples/configs/curated/qwen3.yaml @@ -14,7 +14,6 @@ cuda_graph_config: - 128 - 256 - 384 -print_iter_log: true enable_attention_dp: true kv_cache_config: free_gpu_memory_fraction: 0.8 From a15ea4ffaaf4465886a12d76d26757c3d13e5a36 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Fri, 20 Feb 2026 17:06:21 +0100 Subject: [PATCH 03/12] [None][test] extend curated config unit tests. - Mock serve functions to make sure the curated configs for trtllm-serve. Signed-off-by: Fadi Saady --- .../llmapi/test_curated_yaml_configs.py | 56 +++++++++++++++++++ .../llmapi/yaml_validation_harness.py | 2 + 2 files changed, 58 insertions(+) diff --git a/tests/unittest/llmapi/test_curated_yaml_configs.py b/tests/unittest/llmapi/test_curated_yaml_configs.py index 2f91e6c78cd8..5be5e8e55bab 100644 --- a/tests/unittest/llmapi/test_curated_yaml_configs.py +++ b/tests/unittest/llmapi/test_curated_yaml_configs.py @@ -14,10 +14,13 @@ # limitations under the License. """L0 tests for validating curated YAML files against TorchLlmArgs.""" +import asyncio from pathlib import Path +from unittest import mock import pytest +from tensorrt_llm.commands.serve import main as serve_main from tensorrt_llm.llmapi import llm_args as llm_args_module from . import yaml_validation_harness as yaml_harness @@ -72,3 +75,56 @@ def test_curated_config_does_not_set_default_leaves(config_path: Path): @pytest.mark.part0 def test_curated_config_count(): assert len(CURATED_CONFIGS) > 0, "No curated config files found" + + +def _serve_cli_args(config_path: Path, port: int = 17999): + """CLI argv for serve, like: trtllm-serve --config --port .""" + return [ + "dummy/model", + "--config", + str(config_path), + "--port", + str(port), + ] + + +async def _noop_serve(_host, _port, sockets=None): + pass + + +class _MockOpenAIServer: + def __init__(self, generator, model, **kwargs): + self.generator = generator + self.model = model + + def __call__(self, host, port, sockets=None): + return _noop_serve(host, port, sockets) + + +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) +def test_curated_config_serve_cli(config_path: Path): + """Invoke serve via CLI (as a user would); Click supplies defaults; server/LLM mocked.""" + mock_llm = mock.Mock() + mock_pytorch_llm = mock.Mock(return_value=mock_llm) + + # Run the coroutine (mock server's _noop_serve) so it is awaited and we avoid + # "coroutine was never awaited". Must capture the real asyncio.run before + # patching: we patch tensorrt_llm.commands.serve.asyncio.run, which is the + # same asyncio module this test uses, so asyncio.run would recurse otherwise. + _real_asyncio_run = asyncio.run + + def _run_coroutine(coroutine): + return _real_asyncio_run(coroutine) + + with ( + mock.patch("tensorrt_llm.commands.serve.get_is_diffusion_model", return_value=False), + mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1), + mock.patch("tensorrt_llm.commands.serve.PyTorchLLM", mock_pytorch_llm), + mock.patch("tensorrt_llm.commands.serve.OpenAIServer", _MockOpenAIServer), + mock.patch("tensorrt_llm.commands.serve.asyncio.run", side_effect=_run_coroutine), + ): + serve_main(args=_serve_cli_args(config_path), standalone_mode=False) + mock_pytorch_llm.assert_called_once() + call_kwargs = mock_pytorch_llm.call_args[1] + llm_args_module.TorchLlmArgs(**call_kwargs) diff --git a/tests/unittest/llmapi/yaml_validation_harness.py b/tests/unittest/llmapi/yaml_validation_harness.py index ad0c3c25de10..0544e523edc5 100644 --- a/tests/unittest/llmapi/yaml_validation_harness.py +++ b/tests/unittest/llmapi/yaml_validation_harness.py @@ -58,6 +58,8 @@ def validate_torch_llm_args_config(cfg: dict[str, Any]) -> llm_args_module.Torch merged = llm_args_module.update_llm_args_with_extra_dict( copy.deepcopy(base_args.model_dump(mode="json")), copy.deepcopy(cfg) ) + if "load_format" in merged: + merged["load_format"] = llm_args_module.LoadFormat(merged["load_format"]) return llm_args_module.TorchLlmArgs(**merged) From 7ca25441552b534b0293f51b0dd301c509b832c5 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Fri, 20 Feb 2026 17:42:23 +0100 Subject: [PATCH 04/12] [None][test] removing load_format hack from yaml_validation_harness. - Instead of converting load_format to Enum after dumping and loading, using the default mode='python' for model dump. Signed-off-by: Fadi Saady --- tests/unittest/llmapi/yaml_validation_harness.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unittest/llmapi/yaml_validation_harness.py b/tests/unittest/llmapi/yaml_validation_harness.py index 0544e523edc5..e519e1959230 100644 --- a/tests/unittest/llmapi/yaml_validation_harness.py +++ b/tests/unittest/llmapi/yaml_validation_harness.py @@ -56,10 +56,11 @@ def load_yaml_dict(path: Path) -> dict[str, Any]: def validate_torch_llm_args_config(cfg: dict[str, Any]) -> llm_args_module.TorchLlmArgs: base_args = llm_args_module.TorchLlmArgs(model="dummy/model", skip_tokenizer_init=True) merged = llm_args_module.update_llm_args_with_extra_dict( - copy.deepcopy(base_args.model_dump(mode="json")), copy.deepcopy(cfg) + # NOTE: using model_dump with mode='python' so enums (e.g. load_format) stay as enum, + # not int. + copy.deepcopy(base_args.model_dump()), + copy.deepcopy(cfg), ) - if "load_format" in merged: - merged["load_format"] = llm_args_module.LoadFormat(merged["load_format"]) return llm_args_module.TorchLlmArgs(**merged) From fca59e4a2f1a7541cf1178d6005ee304e4f9ebdb Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Mon, 23 Feb 2026 15:40:04 +0100 Subject: [PATCH 05/12] [None][test] Moving config database tests into one place. - Using same tests for curated/ and database/ configs. - Removed tests for hardcoded deprecated keys since it will be covered in test_database_yaml_config_validates_against_llm_args. Signed-off-by: Fadi Saady --- examples/configs/curated/qwen3-next.yaml | 2 - .../B200/1k1k_tp8_conc16.yaml | 2 - .../B200/1k1k_tp8_conc32.yaml | 2 - .../DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml | 2 - .../B200/1k1k_tp8_conc64.yaml | 2 - .../DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml | 2 - .../B200/8k1k_tp8_conc16.yaml | 2 - .../B200/8k1k_tp8_conc32.yaml | 2 - .../DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml | 2 - .../B200/8k1k_tp8_conc64.yaml | 1 - .../DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml | 2 - .../H200/1k1k_tp8_conc16.yaml | 3 - .../H200/1k1k_tp8_conc32.yaml | 3 - .../DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml | 3 - .../H200/1k1k_tp8_conc64.yaml | 3 - .../DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml | 3 - .../H200/8k1k_tp8_conc16.yaml | 3 - .../H200/8k1k_tp8_conc32.yaml | 3 - .../DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml | 3 - .../H200/8k1k_tp8_conc64.yaml | 2 - .../DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml | 3 - .../B200/1k1k_tp4_conc128.yaml | 2 - .../B200/1k1k_tp4_conc16.yaml | 2 - .../B200/1k1k_tp4_conc256.yaml | 1 - .../B200/1k1k_tp4_conc32.yaml | 2 - .../B200/1k1k_tp4_conc4.yaml | 2 - .../B200/1k1k_tp4_conc64.yaml | 2 - .../B200/1k1k_tp4_conc8.yaml | 2 - .../B200/1k1k_tp8_conc128.yaml | 2 - .../B200/1k1k_tp8_conc16.yaml | 2 - .../B200/1k1k_tp8_conc256.yaml | 1 - .../B200/1k1k_tp8_conc32.yaml | 2 - .../B200/1k1k_tp8_conc4.yaml | 2 - .../B200/1k1k_tp8_conc64.yaml | 2 - .../B200/1k1k_tp8_conc8.yaml | 2 - .../B200/8k1k_tp4_conc128.yaml | 1 - .../B200/8k1k_tp4_conc16.yaml | 2 - .../B200/8k1k_tp4_conc256.yaml | 1 - .../B200/8k1k_tp4_conc32.yaml | 2 - .../B200/8k1k_tp4_conc4.yaml | 2 - .../B200/8k1k_tp4_conc64.yaml | 1 - .../B200/8k1k_tp4_conc8.yaml | 2 - .../B200/8k1k_tp8_conc128.yaml | 1 - .../B200/8k1k_tp8_conc16.yaml | 2 - .../B200/8k1k_tp8_conc256.yaml | 1 - .../B200/8k1k_tp8_conc32.yaml | 2 - .../B200/8k1k_tp8_conc4.yaml | 2 - .../B200/8k1k_tp8_conc64.yaml | 1 - .../B200/8k1k_tp8_conc8.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp1_conc16.yaml | 3 - .../gpt-oss-120b/B200/1k1k_tp1_conc32.yaml | 3 - .../gpt-oss-120b/B200/1k1k_tp1_conc4.yaml | 3 - .../gpt-oss-120b/B200/1k1k_tp1_conc64.yaml | 3 - .../gpt-oss-120b/B200/1k1k_tp1_conc8.yaml | 3 - .../gpt-oss-120b/B200/1k1k_tp2_conc16.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp2_conc32.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp2_conc4.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp2_conc64.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp2_conc8.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp4_conc16.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp4_conc32.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp4_conc4.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp4_conc64.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp4_conc8.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp8_conc16.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp8_conc32.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp8_conc4.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp8_conc64.yaml | 2 - .../gpt-oss-120b/B200/1k1k_tp8_conc8.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp1_conc16.yaml | 3 - .../gpt-oss-120b/B200/1k8k_tp1_conc32.yaml | 3 - .../gpt-oss-120b/B200/1k8k_tp1_conc4.yaml | 3 - .../gpt-oss-120b/B200/1k8k_tp1_conc64.yaml | 3 - .../gpt-oss-120b/B200/1k8k_tp1_conc8.yaml | 3 - .../gpt-oss-120b/B200/1k8k_tp2_conc16.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp2_conc32.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp2_conc4.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp2_conc64.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp2_conc8.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp4_conc16.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp4_conc32.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp4_conc4.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp4_conc64.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp4_conc8.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp8_conc16.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp8_conc32.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp8_conc4.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp8_conc64.yaml | 2 - .../gpt-oss-120b/B200/1k8k_tp8_conc8.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp1_conc16.yaml | 3 - .../gpt-oss-120b/B200/8k1k_tp1_conc32.yaml | 3 - .../gpt-oss-120b/B200/8k1k_tp1_conc4.yaml | 3 - .../gpt-oss-120b/B200/8k1k_tp1_conc64.yaml | 3 - .../gpt-oss-120b/B200/8k1k_tp1_conc8.yaml | 3 - .../gpt-oss-120b/B200/8k1k_tp2_conc16.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp2_conc32.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp2_conc4.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp2_conc64.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp2_conc8.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp4_conc16.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp4_conc32.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp4_conc4.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp4_conc64.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp4_conc8.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp8_conc16.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp8_conc32.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp8_conc4.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp8_conc64.yaml | 2 - .../gpt-oss-120b/B200/8k1k_tp8_conc8.yaml | 2 - .../gpt-oss-120b/H200/1k1k_tp1_conc16.yaml | 4 - .../gpt-oss-120b/H200/1k1k_tp1_conc32.yaml | 4 - .../gpt-oss-120b/H200/1k1k_tp1_conc4.yaml | 4 - .../gpt-oss-120b/H200/1k1k_tp1_conc64.yaml | 4 - .../gpt-oss-120b/H200/1k1k_tp1_conc8.yaml | 4 - .../gpt-oss-120b/H200/1k1k_tp2_conc16.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp2_conc32.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp2_conc4.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp2_conc64.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp2_conc8.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp4_conc16.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp4_conc32.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp4_conc4.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp4_conc64.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp4_conc8.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp8_conc16.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp8_conc32.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp8_conc4.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp8_conc64.yaml | 3 - .../gpt-oss-120b/H200/1k1k_tp8_conc8.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp1_conc16.yaml | 4 - .../gpt-oss-120b/H200/1k8k_tp1_conc32.yaml | 4 - .../gpt-oss-120b/H200/1k8k_tp1_conc4.yaml | 4 - .../gpt-oss-120b/H200/1k8k_tp1_conc64.yaml | 4 - .../gpt-oss-120b/H200/1k8k_tp1_conc8.yaml | 4 - .../gpt-oss-120b/H200/1k8k_tp2_conc16.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp2_conc32.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp2_conc4.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp2_conc64.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp2_conc8.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp4_conc16.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp4_conc32.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp4_conc4.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp4_conc64.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp4_conc8.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp8_conc16.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp8_conc32.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp8_conc4.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp8_conc64.yaml | 3 - .../gpt-oss-120b/H200/1k8k_tp8_conc8.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp1_conc16.yaml | 4 - .../gpt-oss-120b/H200/8k1k_tp1_conc32.yaml | 4 - .../gpt-oss-120b/H200/8k1k_tp1_conc4.yaml | 4 - .../gpt-oss-120b/H200/8k1k_tp1_conc64.yaml | 4 - .../gpt-oss-120b/H200/8k1k_tp1_conc8.yaml | 4 - .../gpt-oss-120b/H200/8k1k_tp2_conc16.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp2_conc32.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp2_conc4.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp2_conc64.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp2_conc8.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp4_conc16.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp4_conc32.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp4_conc4.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp4_conc64.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp4_conc8.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp8_conc16.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp8_conc32.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp8_conc4.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp8_conc64.yaml | 3 - .../gpt-oss-120b/H200/8k1k_tp8_conc8.yaml | 3 - tests/unittest/llmapi/test_config_database.py | 97 +++++++++++-- .../llmapi/test_curated_yaml_configs.py | 130 ------------------ .../llmapi/yaml_validation_harness.py | 18 --- 172 files changed, 84 insertions(+), 589 deletions(-) delete mode 100644 tests/unittest/llmapi/test_curated_yaml_configs.py diff --git a/examples/configs/curated/qwen3-next.yaml b/examples/configs/curated/qwen3-next.yaml index 7a149cf34ca6..cf29737c5740 100644 --- a/examples/configs/curated/qwen3-next.yaml +++ b/examples/configs/curated/qwen3-next.yaml @@ -10,5 +10,3 @@ moe_config: backend: TRTLLM stream_interval: 20 num_postprocess_workers: 4 -kv_cache_config: - enable_block_reuse: false diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml index 1b4d4b6aaf54..3f5bddc4c3b2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml index 5fba0289d393..2d2a1e423d79 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml index 09ee0c602055..4f2eda86ff4f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 256 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml index 9f80dfd3bcf5..b966b637a825 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml index 2b54bf087b04..ffcfae42cb73 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml @@ -1,6 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 enable_attention_dp: true print_iter_log: true kv_cache_config: @@ -16,6 +15,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml index b2ff0ba8e55d..608ba290a897 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml @@ -1,7 +1,5 @@ cuda_graph_config: enable_padding: true - max_batch_size: 128 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +10,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml index d86cb71568e2..0faad6ed89bf 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1216 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml index 18fbe5eec97e..fd1718aaf981 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1344 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml index 903449e0f320..f8b1ac1e3c33 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml index 696ad9bb03e6..824e7e6b4607 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1216 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml index 86707e68fb93..eeeac04724ea 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1344 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml index fc971bf09ef9..51d333b79da4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 1152 max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml index 6d881085020f..e36e4f7e551c 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8384 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml index bff28a8fa5ff..1bfc9ec9573c 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8512 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml index 17ca6555ef18..46084314e82d 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml index 9b81917cfe93..8d1b57809ea7 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml index cfff9caf5e2d..d6d278f0797f 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8384 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml index f4488125d239..43f58836ca57 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8512 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml index eb2fb249d137..83dea42f5566 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml @@ -16,6 +16,5 @@ attention_dp_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml index c2a9fb067f0f..70789fa5349a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml @@ -1,7 +1,6 @@ cuda_graph_config: enable_padding: true max_batch_size: 512 -enable_attention_dp: false print_iter_log: true kv_cache_config: dtype: fp8 @@ -12,6 +11,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 8320 max_seq_len: 9416 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml index 323d5260e365..867425e8464d 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml index 4e42c74c5dcb..5ac815532384 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml index 41a3501db86b..a8067336e1c8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml index ef8559949b9f..5867861391db 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml index cf3f1435815f..ba8755ccba8b 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml index 89f16c083483..38875d1078b0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml index 50a9a01eabd9..2c7f0513e54a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml index 9e38bf35d192..71eccd39becb 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml index 828ad265824c..d01d07cf84ac 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml index 27a1f1b0a513..d16eb6bbdcf2 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml index d85b3d01a4ee..e0ac7cc6a25c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml index 9ed994fa4bf7..522acd56cc4e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml index f7d77d7296d4..08f4fcb8d9ba 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml index 8d4215121a62..1472917945d9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml index 0c0101c83287..3bd661c6b439 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml index dda82a593e4d..9520ba0219ce 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml index c7700969144b..f49d781dd0f8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml index 55e10c0d157f..2e3b2430549d 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml index f606123c32f4..883c0fcd2266 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml index 2d284fcf8cff..68f52a366a93 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml index b17f7425cc6b..f136968130d1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml index e559dcc0b8b8..2fa09ee400d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml index af72d4f4949b..50c1b7062477 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml index 3e3f969fc72f..e9319d9a7e35 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml index aef4e1636b28..e42f06bc2138 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml index 879c2fba7094..4bd4b9377fbd 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml index 322f73709e86..5e12525cc48a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml index 84cac88c8643..87877d6280d0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml index 031a0fd74482..ae6938feba4c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml index 445cec1d82cf..95fb72d0a9d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml index 4d9ce00dbcdb..58e92ed78cd6 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml index e790e0d98029..2c0b0d06df83 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml index f72680460cef..5e84439ebcb0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml index ddceeb61fa0c..3e4f7ddb7f2e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml index f08dca8cd190..a8446c8e35da 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml index 95cdf3a61f7b..b936b317e0db 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml index 74d4e29f677f..630aa9ab0aff 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml index 59b9319ee8e0..4c2f9d688669 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml index 365f467d6e43..b7ef6b8d2516 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml index befde425ae46..7ce727c75571 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml index b17f7425cc6b..f136968130d1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml index e559dcc0b8b8..2fa09ee400d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml index af72d4f4949b..50c1b7062477 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml index 3e3f969fc72f..e9319d9a7e35 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml index aef4e1636b28..e42f06bc2138 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -13,9 +12,7 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml index 879c2fba7094..4bd4b9377fbd 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml index 322f73709e86..5e12525cc48a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml index 84cac88c8643..87877d6280d0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml index 031a0fd74482..ae6938feba4c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml index 445cec1d82cf..95fb72d0a9d3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml index 4d9ce00dbcdb..58e92ed78cd6 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml index e790e0d98029..2c0b0d06df83 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml index f72680460cef..5e84439ebcb0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml index ddceeb61fa0c..3e4f7ddb7f2e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml index f08dca8cd190..a8446c8e35da 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml index 95cdf3a61f7b..b936b317e0db 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml index 74d4e29f677f..630aa9ab0aff 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml index 59b9319ee8e0..4c2f9d688669 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml index 365f467d6e43..b7ef6b8d2516 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml index befde425ae46..7ce727c75571 100644 --- a/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml @@ -4,7 +4,6 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.85 @@ -16,6 +15,5 @@ moe_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml index 6736070cc707..595e44706bbb 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml index a6fec264e879..4cf0e25a5239 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml index c98b557f7963..de9e3357e084 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml index 8144ce5b4d20..057f95b9c800 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml index 45864a956ea3..81f16ee5fc12 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml index 4762e577704f..80822a662b7a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml index 3a16da2d4a45..2b0bc6e46096 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml index 2b789c4add66..d0d0b9fef405 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml index 94c40f2ac280..b95889fc6b72 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml index 2b866099d81b..d536457d4449 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml index ff543422a0be..7cdcf64adaf9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml index 95477ae879dd..891d01c63a9e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml index dca7e5a63a1f..11079f4187f8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml index 1b2ad812edad..b209e1d39fae 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml index 0d1ee33fee14..5f589609ba07 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml index 142a9c07af7b..aaca4f4f67f4 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml index c42c0506e1d5..d6009857f7bc 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml index 161ebf4cdb60..888c079af725 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml index f7f9a1b76ddc..a0ed865c4a1e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml index 0c9084d25cac..7b3323f0f218 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 2068 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml index 466565e489df..3d6ddd08f9f1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml index 4f0632784c88..74343e0022e8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml index 57f3c1547c6f..88174dc7920a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml index e61ad0b3de98..170efc76914a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml index 53b5461cf467..a554dc2bb91e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml index 41d9a88b2007..e6d7028862a3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml index 5bd3aa86c7ca..c3e23e3c2239 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml index 90854c5fbdc3..331ee8ed0c56 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml index d16f3ecc9544..49a223b458f0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml index 36171627426d..b6c71c91cb1a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml index bb3a41bc2fca..43c9fee6976b 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml index 1dd6379a4e8e..9ad1a65df871 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml index e63805377ed7..a606aabd9cf1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml index 52be12ec84d6..b8a12b4ea250 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml index a80d8f26abf8..269424525ced 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml index 4c416c78bb3d..ee0548ee1630 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml index eade98620983..d1a258c6b91c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml index 24916df78a24..b2560d08767e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml index a609e99a880b..f8423b90df1e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml index 9c37e6359dc7..a8ca13ef61c9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml index 466565e489df..3d6ddd08f9f1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml index 4f0632784c88..74343e0022e8 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml index 57f3c1547c6f..88174dc7920a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml index e61ad0b3de98..170efc76914a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml index 53b5461cf467..a554dc2bb91e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml @@ -3,18 +3,14 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 -tensor_parallel_size: 1 moe_expert_parallel_size: 1 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml index 41d9a88b2007..e6d7028862a3 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml index 5bd3aa86c7ca..c3e23e3c2239 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml index 90854c5fbdc3..331ee8ed0c56 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml index d16f3ecc9544..49a223b458f0 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml index 36171627426d..b6c71c91cb1a 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 2 moe_expert_parallel_size: 2 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml index bb3a41bc2fca..43c9fee6976b 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml index 1dd6379a4e8e..9ad1a65df871 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml index e63805377ed7..a606aabd9cf1 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml index 52be12ec84d6..b8a12b4ea250 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml index a80d8f26abf8..269424525ced 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml index 4c416c78bb3d..ee0548ee1630 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 16 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml index eade98620983..d1a258c6b91c 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 32 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml index 24916df78a24..b2560d08767e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 4 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml index a609e99a880b..f8423b90df1e 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 64 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml index 9c37e6359dc7..a8ca13ef61c9 100644 --- a/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml @@ -3,9 +3,7 @@ env_overrides: cuda_graph_config: enable_padding: true max_batch_size: 8 -enable_attention_dp: false kv_cache_config: - dtype: auto free_gpu_memory_fraction: 0.85 moe_config: backend: TRITON @@ -15,6 +13,5 @@ stream_interval: 20 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -backend: pytorch max_num_tokens: 20000 max_seq_len: 9236 diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 67944856d942..b5de76079ee5 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,51 +12,122 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""L0 wrapper tests for validating database YAML files against TorchLlmArgs.""" +"""L0 tests for validating curated and database YAML configs against TorchLlmArgs.""" +import asyncio from pathlib import Path +from unittest import mock import pytest +from tensorrt_llm.commands.serve import main as serve_main +from tensorrt_llm.llmapi import llm_args as llm_args_module + from . import yaml_validation_harness as yaml_harness CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" +CURATED_DIR = CONFIG_ROOT / "curated" DATABASE_DIR = CONFIG_ROOT / "database" +CURATED_CONFIGS = yaml_harness.collect_yaml_files(CURATED_DIR, "**/*.yaml") DATABASE_CONFIGS = yaml_harness.collect_yaml_files( DATABASE_DIR, "**/*.yaml", exclude_names={"lookup.yaml"} ) +ALL_CONFIGS = sorted(CURATED_CONFIGS + DATABASE_CONFIGS) @pytest.fixture(autouse=True) def mock_gpu_environment(): - """Mock GPU functions for CPU-only test execution.""" + """Mock GPU functions for CPU-only schema test execution.""" with yaml_harness.mock_cuda_for_schema_validation(): yield def get_config_id(config_path: Path) -> str: - return str(config_path.relative_to(DATABASE_DIR)) + return str(config_path.relative_to(CONFIG_ROOT)) + + +def _assert_kv_cache_block_reuse_policy(config_dict: dict) -> None: + assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False @pytest.mark.part0 -@pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) -def test_config_validates_against_llm_args(config_path: Path): +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_validates_against_llm_args(config_path: Path): config_dict = yaml_harness.load_yaml_dict(config_path) yaml_harness.validate_torch_llm_args_config(config_dict) -def _assert_kv_cache_block_reuse_policy(config_dict: dict) -> None: - assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_does_not_disable_kv_cache_block_reuse(config_path: Path): + config_dict = yaml_harness.load_yaml_dict(config_path) + yaml_harness.assert_custom_policy(config_dict, _assert_kv_cache_block_reuse_policy) @pytest.mark.part0 -@pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) -def test_config_does_not_disable_kv_cache_block_reuse(config_path: Path): +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_does_not_set_default_leaves(config_path: Path): config_dict = yaml_harness.load_yaml_dict(config_path) - yaml_harness.assert_custom_policy(config_dict, _assert_kv_cache_block_reuse_policy) + default_cfg = llm_args_module.TorchLlmArgs( + model="dummy/model", skip_tokenizer_init=True + ).model_dump(mode="json") + yaml_harness.assert_no_default_valued_leaves(config_dict, default_cfg) + + +@pytest.mark.part0 +def test_database_yaml_config_count(): + assert len(ALL_CONFIGS) > 0, "No curated or database config files found" + + +def _serve_cli_args(config_path: Path, port: int = 17999): + """CLI argv for serve, like: trtllm-serve --config --port .""" + return [ + "dummy/model", + "--config", + str(config_path), + "--port", + str(port), + ] + + +async def _noop_serve(_host, _port, sockets=None): + pass + + +class _MockOpenAIServer: + def __init__(self, generator, model, **kwargs): + self.generator = generator + self.model = model + + def __call__(self, host, port, sockets=None): + return _noop_serve(host, port, sockets) @pytest.mark.part0 -def test_database_config_count(): - assert len(DATABASE_CONFIGS) > 0, "No database config files found" +@pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) +def test_database_yaml_config_serve_cli(config_path: Path): + """Invoke serve via CLI (as a user would); Click supplies defaults; server/LLM mocked.""" + mock_llm = mock.Mock() + mock_pytorch_llm = mock.Mock(return_value=mock_llm) + + # Run the coroutine (mock server's _noop_serve) so it is awaited and we avoid + # "coroutine was never awaited". Must capture the real asyncio.run before + # patching: we patch tensorrt_llm.commands.serve.asyncio.run, which is the + # same asyncio module this test uses, so asyncio.run would recurse otherwise. + _real_asyncio_run = asyncio.run + + def _run_coroutine(coroutine): + return _real_asyncio_run(coroutine) + + with ( + mock.patch("tensorrt_llm.commands.serve.get_is_diffusion_model", return_value=False), + mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1), + mock.patch("tensorrt_llm.commands.serve.PyTorchLLM", mock_pytorch_llm), + mock.patch("tensorrt_llm.commands.serve.OpenAIServer", _MockOpenAIServer), + mock.patch("tensorrt_llm.commands.serve.asyncio.run", side_effect=_run_coroutine), + ): + serve_main(args=_serve_cli_args(config_path), standalone_mode=False) + mock_pytorch_llm.assert_called_once() + call_kwargs = mock_pytorch_llm.call_args[1] + llm_args_module.TorchLlmArgs(**call_kwargs) diff --git a/tests/unittest/llmapi/test_curated_yaml_configs.py b/tests/unittest/llmapi/test_curated_yaml_configs.py deleted file mode 100644 index 5be5e8e55bab..000000000000 --- a/tests/unittest/llmapi/test_curated_yaml_configs.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""L0 tests for validating curated YAML files against TorchLlmArgs.""" - -import asyncio -from pathlib import Path -from unittest import mock - -import pytest - -from tensorrt_llm.commands.serve import main as serve_main -from tensorrt_llm.llmapi import llm_args as llm_args_module - -from . import yaml_validation_harness as yaml_harness - -CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" -CURATED_DIR = CONFIG_ROOT / "curated" - -CURATED_CONFIGS = yaml_harness.collect_yaml_files(CURATED_DIR, "**/*.yaml") - -DEPRECATED_KEY_MAP = { - "kv_cache_free_gpu_memory_fraction": "kv_cache_config.free_gpu_memory_fraction", - "moe_backend": "moe_config.backend", - "use_cuda_graph": "cuda_graph_config", -} - - -@pytest.fixture(autouse=True) -def mock_gpu_environment(): - """Mock GPU functions for CPU-only schema test execution.""" - with yaml_harness.mock_cuda_for_schema_validation(): - yield - - -def get_config_id(config_path: Path) -> str: - return str(config_path.relative_to(CURATED_DIR)) - - -@pytest.mark.part0 -@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) -def test_curated_config_validates_against_llm_args(config_path: Path): - config_dict = yaml_harness.load_yaml_dict(config_path) - yaml_harness.validate_torch_llm_args_config(config_dict) - - -@pytest.mark.part0 -@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) -def test_curated_config_does_not_use_deprecated_keys(config_path: Path): - config_dict = yaml_harness.load_yaml_dict(config_path) - yaml_harness.assert_no_deprecated_keys(config_dict, DEPRECATED_KEY_MAP) - - -@pytest.mark.part0 -@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) -def test_curated_config_does_not_set_default_leaves(config_path: Path): - config_dict = yaml_harness.load_yaml_dict(config_path) - default_cfg = llm_args_module.TorchLlmArgs( - model="dummy/model", skip_tokenizer_init=True - ).model_dump(mode="json") - yaml_harness.assert_no_default_valued_leaves(config_dict, default_cfg) - - -@pytest.mark.part0 -def test_curated_config_count(): - assert len(CURATED_CONFIGS) > 0, "No curated config files found" - - -def _serve_cli_args(config_path: Path, port: int = 17999): - """CLI argv for serve, like: trtllm-serve --config --port .""" - return [ - "dummy/model", - "--config", - str(config_path), - "--port", - str(port), - ] - - -async def _noop_serve(_host, _port, sockets=None): - pass - - -class _MockOpenAIServer: - def __init__(self, generator, model, **kwargs): - self.generator = generator - self.model = model - - def __call__(self, host, port, sockets=None): - return _noop_serve(host, port, sockets) - - -@pytest.mark.part0 -@pytest.mark.parametrize("config_path", CURATED_CONFIGS, ids=get_config_id) -def test_curated_config_serve_cli(config_path: Path): - """Invoke serve via CLI (as a user would); Click supplies defaults; server/LLM mocked.""" - mock_llm = mock.Mock() - mock_pytorch_llm = mock.Mock(return_value=mock_llm) - - # Run the coroutine (mock server's _noop_serve) so it is awaited and we avoid - # "coroutine was never awaited". Must capture the real asyncio.run before - # patching: we patch tensorrt_llm.commands.serve.asyncio.run, which is the - # same asyncio module this test uses, so asyncio.run would recurse otherwise. - _real_asyncio_run = asyncio.run - - def _run_coroutine(coroutine): - return _real_asyncio_run(coroutine) - - with ( - mock.patch("tensorrt_llm.commands.serve.get_is_diffusion_model", return_value=False), - mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1), - mock.patch("tensorrt_llm.commands.serve.PyTorchLLM", mock_pytorch_llm), - mock.patch("tensorrt_llm.commands.serve.OpenAIServer", _MockOpenAIServer), - mock.patch("tensorrt_llm.commands.serve.asyncio.run", side_effect=_run_coroutine), - ): - serve_main(args=_serve_cli_args(config_path), standalone_mode=False) - mock_pytorch_llm.assert_called_once() - call_kwargs = mock_pytorch_llm.call_args[1] - llm_args_module.TorchLlmArgs(**call_kwargs) diff --git a/tests/unittest/llmapi/yaml_validation_harness.py b/tests/unittest/llmapi/yaml_validation_harness.py index e519e1959230..c3d629e5864f 100644 --- a/tests/unittest/llmapi/yaml_validation_harness.py +++ b/tests/unittest/llmapi/yaml_validation_harness.py @@ -64,24 +64,6 @@ def validate_torch_llm_args_config(cfg: dict[str, Any]) -> llm_args_module.Torch return llm_args_module.TorchLlmArgs(**merged) -def assert_no_deprecated_keys(cfg: dict[str, Any], deprecated_map: dict[str, str]) -> None: - violations: list[str] = [] - - def walk_dict(node: dict[str, Any], path: tuple[str, ...]) -> None: - for key, value in node.items(): - full_path = path + (str(key),) - if key in deprecated_map: - replacement = deprecated_map[key] - violations.append(f"{'.'.join(full_path)} -> {replacement}") - - if isinstance(value, dict): - walk_dict(value, full_path) - - walk_dict(cfg, ()) - if violations: - raise AssertionError("Found deprecated config keys:\n" + "\n".join(sorted(violations))) - - def assert_no_default_valued_leaves( cfg: dict[str, Any], default_cfg: dict[str, Any], From 5a6d7d6143044c6684595ef09a1437a70e3e8e04 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Tue, 24 Feb 2026 13:20:16 +0100 Subject: [PATCH 06/12] [None][test] Removing assert_custom_policy, calling policy directly. - This is cleaner. Signed-off-by: Fadi Saady --- tests/unittest/llmapi/test_config_database.py | 6 ++++-- tests/unittest/llmapi/yaml_validation_harness.py | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index b5de76079ee5..98a8e376c2dd 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -48,7 +48,9 @@ def get_config_id(config_path: Path) -> str: def _assert_kv_cache_block_reuse_policy(config_dict: dict) -> None: - assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False + assert config_dict.get("kv_cache_config", {}).get("enable_block_reuse") is not False, ( + "KV cache block reuse should not be disabled" + ) @pytest.mark.part0 @@ -62,7 +64,7 @@ def test_database_yaml_config_validates_against_llm_args(config_path: Path): @pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) def test_database_yaml_config_does_not_disable_kv_cache_block_reuse(config_path: Path): config_dict = yaml_harness.load_yaml_dict(config_path) - yaml_harness.assert_custom_policy(config_dict, _assert_kv_cache_block_reuse_policy) + _assert_kv_cache_block_reuse_policy(config_dict) @pytest.mark.part0 diff --git a/tests/unittest/llmapi/yaml_validation_harness.py b/tests/unittest/llmapi/yaml_validation_harness.py index c3d629e5864f..b6181e81a860 100644 --- a/tests/unittest/llmapi/yaml_validation_harness.py +++ b/tests/unittest/llmapi/yaml_validation_harness.py @@ -17,7 +17,7 @@ from __future__ import annotations import copy -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Iterable, Iterator from contextlib import contextmanager from pathlib import Path from typing import Any @@ -102,10 +102,6 @@ def walk(candidate: dict[str, Any], defaults: dict[str, Any], path: tuple[str, . ) -def assert_custom_policy(cfg: dict[str, Any], policy_fn: Callable[[dict[str, Any]], None]) -> None: - policy_fn(cfg) - - @contextmanager def mock_cuda_for_schema_validation( device_count: int = 8, compute_capability_major: int = 8, is_available: bool = True From fae9ea3f4a3559ddb9c47892e847c123e14fe74f Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Tue, 24 Feb 2026 14:59:19 +0100 Subject: [PATCH 07/12] [None][fix] Handling model-specific default args in test_config_database.py - Added lookup.yaml for curated configs. - Added optional arch field in lookup to get model defaults for specific model archs. Signed-off-by: Fadi Saady --- examples/configs/curated/lookup.yaml | 31 +++++++ tests/unittest/llmapi/test_config_database.py | 87 ++++++++++++++++--- 2 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 examples/configs/curated/lookup.yaml diff --git a/examples/configs/curated/lookup.yaml b/examples/configs/curated/lookup.yaml new file mode 100644 index 000000000000..72b50940d980 --- /dev/null +++ b/examples/configs/curated/lookup.yaml @@ -0,0 +1,31 @@ +# arch: MODEL_CLASS_MAPPING key; required when model has get_model_defaults. Add when adding entries. +- model: Qwen/Qwen3-Next-80B-A3B-Thinking + arch: Qwen3NextForCausalLM + config_path: examples/configs/curated/qwen3-next.yaml +- model: Qwen/Qwen3-30B-A3B + arch: Qwen3MoeForCausalLM + config_path: examples/configs/curated/qwen3.yaml +- model: Qwen/Qwen3-30B-A3B + arch: Qwen3MoeForCausalLM + config_path: examples/configs/curated/qwen3-disagg-prefill.yaml +- model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/deepseek-r1-latency.yaml +- model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/deepseek-r1-throughput.yaml +- model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/deepseek-r1-deepgemm.yaml +- model: openai/gpt-oss-120b + arch: GptOssForCausalLM + config_path: examples/configs/curated/gpt-oss-120b-latency.yaml +- model: openai/gpt-oss-120b + arch: GptOssForCausalLM + config_path: examples/configs/curated/gpt-oss-120b-throughput.yaml +- model: nvidia/Llama-3.3-70B-Instruct-FP8 + arch: LlamaForCausalLM + config_path: examples/configs/curated/llama-3.3-70b.yaml +- model: nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 + arch: Llama4ForConditionalGeneration + config_path: examples/configs/curated/llama-4-scout.yaml diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 98a8e376c2dd..8db1c9777235 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -19,27 +19,88 @@ from unittest import mock import pytest +import yaml +from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING from tensorrt_llm.commands.serve import main as serve_main from tensorrt_llm.llmapi import llm_args as llm_args_module -from . import yaml_validation_harness as yaml_harness +from .yaml_validation_harness import ( + assert_no_default_valued_leaves, + collect_yaml_files, + load_yaml_dict, + mock_cuda_for_schema_validation, + validate_torch_llm_args_config, +) CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" +REPO_ROOT = CONFIG_ROOT.parent.parent CURATED_DIR = CONFIG_ROOT / "curated" DATABASE_DIR = CONFIG_ROOT / "database" -CURATED_CONFIGS = yaml_harness.collect_yaml_files(CURATED_DIR, "**/*.yaml") -DATABASE_CONFIGS = yaml_harness.collect_yaml_files( - DATABASE_DIR, "**/*.yaml", exclude_names={"lookup.yaml"} -) +CURATED_CONFIGS = collect_yaml_files(CURATED_DIR, "**/*.yaml", exclude_names={"lookup.yaml"}) +DATABASE_CONFIGS = collect_yaml_files(DATABASE_DIR, "**/*.yaml", exclude_names={"lookup.yaml"}) ALL_CONFIGS = sorted(CURATED_CONFIGS + DATABASE_CONFIGS) +ALL_LOOKUP_PATHS = sorted((DATABASE_DIR / "lookup.yaml", CURATED_DIR / "lookup.yaml")) + + +def _load_config_path_to_lookup_entry() -> dict[str, dict]: + """Build config_path -> {model, arch?} from database and curated lookup.yaml files.""" + result = {} + for lookup_path in ALL_LOOKUP_PATHS: + if not lookup_path.exists(): + continue + with open(lookup_path, encoding="utf-8") as f: + entries = yaml.safe_load(f) + if not entries: + continue + for entry in entries: + if not isinstance(entry, dict) or "config_path" not in entry: + continue + key = entry["config_path"] + result[key] = {"model": entry.get("model"), "arch": entry.get("arch")} + return result + + +_CONFIG_PATH_TO_ENTRY = _load_config_path_to_lookup_entry() + + +def _get_default_values_for_config(config_path: Path) -> dict: + """Get the default values for a config path. + + Some models may have custom default values that are different from the global defaults. + Based on the model architecture, get the default values that are actually applied. + """ + try: + config_key = config_path.relative_to(REPO_ROOT).as_posix() + except ValueError: + config_key = None + entry = _CONFIG_PATH_TO_ENTRY.get(config_key) if config_key else None + global_default = llm_args_module.TorchLlmArgs( + model="dummy/model", skip_tokenizer_init=True + ).model_dump(mode="json") + if not entry or not entry.get("arch") or not entry.get("model"): + return global_default + + model = entry["model"] + arch = entry["arch"] + model_cls = MODEL_CLASS_MAPPING.get(arch) + if not model_cls or not hasattr(model_cls, "get_model_defaults"): + return global_default + + base_args = llm_args_module.TorchLlmArgs(model=model, skip_tokenizer_init=True).model_dump( + mode="json" + ) + + model_defaults = model_cls.get_model_defaults(base_args) + base_args.update(model_defaults) + return base_args @pytest.fixture(autouse=True) def mock_gpu_environment(): """Mock GPU functions for CPU-only schema test execution.""" - with yaml_harness.mock_cuda_for_schema_validation(): + with mock_cuda_for_schema_validation(): yield @@ -56,25 +117,23 @@ def _assert_kv_cache_block_reuse_policy(config_dict: dict) -> None: @pytest.mark.part0 @pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) def test_database_yaml_config_validates_against_llm_args(config_path: Path): - config_dict = yaml_harness.load_yaml_dict(config_path) - yaml_harness.validate_torch_llm_args_config(config_dict) + config_dict = load_yaml_dict(config_path) + validate_torch_llm_args_config(config_dict) @pytest.mark.part0 @pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) def test_database_yaml_config_does_not_disable_kv_cache_block_reuse(config_path: Path): - config_dict = yaml_harness.load_yaml_dict(config_path) + config_dict = load_yaml_dict(config_path) _assert_kv_cache_block_reuse_policy(config_dict) @pytest.mark.part0 @pytest.mark.parametrize("config_path", ALL_CONFIGS, ids=get_config_id) def test_database_yaml_config_does_not_set_default_leaves(config_path: Path): - config_dict = yaml_harness.load_yaml_dict(config_path) - default_cfg = llm_args_module.TorchLlmArgs( - model="dummy/model", skip_tokenizer_init=True - ).model_dump(mode="json") - yaml_harness.assert_no_default_valued_leaves(config_dict, default_cfg) + config_dict = load_yaml_dict(config_path) + default_cfg = _get_default_values_for_config(config_path) + assert_no_default_valued_leaves(config_dict, default_cfg) @pytest.mark.part0 From 2ba38703245637b7fc48cf4db71d167ff695bde9 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Tue, 24 Feb 2026 15:14:53 +0100 Subject: [PATCH 08/12] [None][chore] Check that all configs have a lookup entry in test_config_database. Signed-off-by: Fadi Saady --- tests/unittest/llmapi/test_config_database.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 8db1c9777235..7289fc5cdeab 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -141,6 +141,18 @@ def test_database_yaml_config_count(): assert len(ALL_CONFIGS) > 0, "No curated or database config files found" +@pytest.mark.part0 +def test_all_configs_have_lookup_entry(): + """Every config in ALL_CONFIGS must have an entry in the lookup files (ALL_LOOKUP_PATHS).""" + config_keys = {p.relative_to(REPO_ROOT).as_posix() for p in ALL_CONFIGS} + lookup_keys = set(_CONFIG_PATH_TO_ENTRY) + missing = config_keys - lookup_keys + assert not missing, ( + "The following configs have no entry in the lookup files. " + "Add each to the appropriate lookup.yaml file:\n" + "\n".join(sorted(missing)) + ) + + def _serve_cli_args(config_path: Path, port: int = 17999): """CLI argv for serve, like: trtllm-serve --config --port .""" return [ From 3a68691b632f065dc7b6a374bd665ab9b9da77d3 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Wed, 25 Feb 2026 11:48:42 +0100 Subject: [PATCH 09/12] [None][chore] Adding model arch to examples/configs/database/lookup.yaml. - TODO: check that all entries have an arch field in unit tests. Signed-off-by: Fadi Saady --- examples/configs/database/lookup.yaml | 168 ++++++++++++++++++ tests/unittest/llmapi/test_config_database.py | 2 +- 2 files changed, 169 insertions(+), 1 deletion(-) diff --git a/examples/configs/database/lookup.yaml b/examples/configs/database/lookup.yaml index d1ac7143ceb9..59e257d20b9e 100644 --- a/examples/configs/database/lookup.yaml +++ b/examples/configs/database/lookup.yaml @@ -1,4 +1,5 @@ - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -6,6 +7,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -13,6 +15,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -20,6 +23,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -27,6 +31,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -34,6 +39,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -41,6 +47,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -48,6 +55,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -55,6 +63,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -62,6 +71,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -69,6 +79,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -76,6 +87,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -83,6 +95,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -90,6 +103,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -97,6 +111,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -104,6 +119,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -111,6 +127,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -118,6 +135,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -125,6 +143,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -132,6 +151,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -139,6 +159,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -146,6 +167,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -153,6 +175,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -160,6 +183,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -167,6 +191,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -174,6 +199,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -181,6 +207,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -188,6 +215,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -195,6 +223,7 @@ config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -202,6 +231,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -209,6 +239,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -216,6 +247,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -223,6 +255,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -230,6 +263,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -237,6 +271,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -244,6 +279,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -251,6 +287,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -258,6 +295,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -265,6 +303,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -272,6 +311,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -279,6 +319,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -286,6 +327,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -293,6 +335,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -300,6 +343,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -307,6 +351,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -314,6 +359,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -321,6 +367,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -328,6 +375,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 + arch: DeepseekV3ForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -335,6 +383,7 @@ config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -342,6 +391,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -349,6 +399,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -356,6 +407,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -363,6 +415,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -370,6 +423,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -377,6 +431,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -384,6 +439,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -391,6 +447,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -398,6 +455,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -405,6 +463,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -412,6 +471,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -419,6 +479,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -426,6 +487,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -433,6 +495,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -440,6 +503,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -447,6 +511,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -454,6 +519,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -461,6 +527,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -468,6 +535,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 1024 @@ -475,6 +543,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -482,6 +551,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -489,6 +559,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -496,6 +567,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -503,6 +575,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -510,6 +583,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -517,6 +591,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -524,6 +599,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -531,6 +607,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -538,6 +615,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -545,6 +623,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -552,6 +631,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -559,6 +639,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -566,6 +647,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -573,6 +655,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -580,6 +663,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -587,6 +671,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -594,6 +679,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -601,6 +687,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -608,6 +695,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 1024 osl: 8192 @@ -615,6 +703,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -622,6 +711,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -629,6 +719,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -636,6 +727,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -643,6 +735,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -650,6 +743,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -657,6 +751,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -664,6 +759,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -671,6 +767,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -678,6 +775,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -685,6 +783,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -692,6 +791,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -699,6 +799,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -706,6 +807,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -713,6 +815,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -720,6 +823,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -727,6 +831,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -734,6 +839,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -741,6 +847,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -748,6 +855,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: B200_NVL isl: 8192 osl: 1024 @@ -755,6 +863,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -762,6 +871,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -769,6 +879,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -776,6 +887,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -783,6 +895,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -790,6 +903,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -797,6 +911,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -804,6 +919,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -811,6 +927,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -818,6 +935,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -825,6 +943,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -832,6 +951,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -839,6 +959,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -846,6 +967,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -853,6 +975,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -860,6 +983,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -867,6 +991,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -874,6 +999,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -881,6 +1007,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -888,6 +1015,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 1024 @@ -895,6 +1023,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -902,6 +1031,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -909,6 +1039,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -916,6 +1047,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -923,6 +1055,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -930,6 +1063,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -937,6 +1071,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -944,6 +1079,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -951,6 +1087,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -958,6 +1095,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -965,6 +1103,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -972,6 +1111,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -979,6 +1119,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -986,6 +1127,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -993,6 +1135,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1000,6 +1143,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1007,6 +1151,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1014,6 +1159,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1021,6 +1167,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1028,6 +1175,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 1024 osl: 8192 @@ -1035,6 +1183,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1042,6 +1191,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1049,6 +1199,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1056,6 +1207,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1063,6 +1215,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1070,6 +1223,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1077,6 +1231,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1084,6 +1239,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1091,6 +1247,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1098,6 +1255,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1105,6 +1263,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1112,6 +1271,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1119,6 +1279,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1126,6 +1287,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1133,6 +1295,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1140,6 +1303,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1147,6 +1311,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1154,6 +1319,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1161,6 +1327,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 @@ -1168,6 +1335,7 @@ config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b + arch: GptOssForCausalLM gpu: H200_SXM isl: 8192 osl: 1024 diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 7289fc5cdeab..9876e8c8a3b6 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); From 861caaba651eefb27425553dfade010ff06ffaac Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Wed, 25 Feb 2026 12:00:36 +0100 Subject: [PATCH 10/12] [None][test] Forcing entries in db lookup.yaml to specify architecture. - Added a unit test in tests/unittest/llmapi/test_config_database.py. - Fixed kimi k2 curated config. Signed-off-by: Fadi Saady --- .../configs/curated/kimi-k2-thinking.yaml | 1 - examples/configs/curated/lookup.yaml | 3 +++ tests/unittest/llmapi/test_config_database.py | 26 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/examples/configs/curated/kimi-k2-thinking.yaml b/examples/configs/curated/kimi-k2-thinking.yaml index c3da19a74967..70fe82da7c15 100644 --- a/examples/configs/curated/kimi-k2-thinking.yaml +++ b/examples/configs/curated/kimi-k2-thinking.yaml @@ -4,7 +4,6 @@ max_seq_len: 8212 tensor_parallel_size: 8 moe_expert_parallel_size: 8 enable_attention_dp: true -pipeline_parallel_size: 1 print_iter_log: true kv_cache_config: free_gpu_memory_fraction: 0.75 diff --git a/examples/configs/curated/lookup.yaml b/examples/configs/curated/lookup.yaml index 72b50940d980..3e66a4d4028e 100644 --- a/examples/configs/curated/lookup.yaml +++ b/examples/configs/curated/lookup.yaml @@ -29,3 +29,6 @@ - model: nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 arch: Llama4ForConditionalGeneration config_path: examples/configs/curated/llama-4-scout.yaml +- model: nvidia/Kimi-K2-Thinking-NVFP4 + arch: DeepseekV3ForCausalLM + config_path: examples/configs/curated/kimi-k2-thinking.yaml diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 9876e8c8a3b6..4a6a584881af 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -153,6 +153,32 @@ def test_all_configs_have_lookup_entry(): ) +@pytest.mark.part0 +def test_all_lookup_entries_have_arch(): + """Every entry in the lookup files must have the required 'arch' field.""" + entries_missing_arch = [] + for lookup_path in ALL_LOOKUP_PATHS: + if not lookup_path.exists(): + continue + with open(lookup_path, encoding="utf-8") as f: + entries = yaml.safe_load(f) + if not entries: + continue + for entry in entries: + if not isinstance(entry, dict) or "config_path" not in entry: + continue + config_path = entry.get("config_path", "") + if not entry.get("arch"): + entries_missing_arch.append( + f" {lookup_path.relative_to(REPO_ROOT)}: {config_path}" + ) + assert not entries_missing_arch, ( + "The following lookup entries are missing the required 'arch' field. " + "Add 'arch: ' to each entry:\n" + + "\n".join(sorted(entries_missing_arch)) + ) + + def _serve_cli_args(config_path: Path, port: int = 17999): """CLI argv for serve, like: trtllm-serve --config --port .""" return [ From 28ec9dd686ee135996616e5dd7cad1329e614bd5 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Wed, 25 Feb 2026 18:29:23 +0100 Subject: [PATCH 11/12] [None][chore] removing unneeded mock in test_config_database. Signed-off-by: Fadi Saady --- tests/unittest/llmapi/test_config_database.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 4a6a584881af..9e63ef4d1dd1 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -14,7 +14,6 @@ # limitations under the License. """L0 tests for validating curated and database YAML configs against TorchLlmArgs.""" -import asyncio from pathlib import Path from unittest import mock @@ -190,7 +189,7 @@ def _serve_cli_args(config_path: Path, port: int = 17999): ] -async def _noop_serve(_host, _port, sockets=None): +async def _noop_serve(_host, _port, _sockets=None): pass @@ -210,21 +209,11 @@ def test_database_yaml_config_serve_cli(config_path: Path): mock_llm = mock.Mock() mock_pytorch_llm = mock.Mock(return_value=mock_llm) - # Run the coroutine (mock server's _noop_serve) so it is awaited and we avoid - # "coroutine was never awaited". Must capture the real asyncio.run before - # patching: we patch tensorrt_llm.commands.serve.asyncio.run, which is the - # same asyncio module this test uses, so asyncio.run would recurse otherwise. - _real_asyncio_run = asyncio.run - - def _run_coroutine(coroutine): - return _real_asyncio_run(coroutine) - with ( mock.patch("tensorrt_llm.commands.serve.get_is_diffusion_model", return_value=False), mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1), mock.patch("tensorrt_llm.commands.serve.PyTorchLLM", mock_pytorch_llm), mock.patch("tensorrt_llm.commands.serve.OpenAIServer", _MockOpenAIServer), - mock.patch("tensorrt_llm.commands.serve.asyncio.run", side_effect=_run_coroutine), ): serve_main(args=_serve_cli_args(config_path), standalone_mode=False) mock_pytorch_llm.assert_called_once() From 58aeb592fdde49b848a1041b5f8eccfe189b364b Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Wed, 25 Feb 2026 18:46:36 +0100 Subject: [PATCH 12/12] [None][chore] minor fix. Signed-off-by: Fadi Saady --- tests/unittest/llmapi/test_config_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 9e63ef4d1dd1..aa1906b352e4 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -216,6 +216,6 @@ def test_database_yaml_config_serve_cli(config_path: Path): mock.patch("tensorrt_llm.commands.serve.OpenAIServer", _MockOpenAIServer), ): serve_main(args=_serve_cli_args(config_path), standalone_mode=False) - mock_pytorch_llm.assert_called_once() + mock_pytorch_llm.assert_called_once() call_kwargs = mock_pytorch_llm.call_args[1] llm_args_module.TorchLlmArgs(**call_kwargs)