From da6123cd5e8cec128efa7573596334e6abb13721 Mon Sep 17 00:00:00 2001 From: chenhany Date: Fri, 5 Jun 2026 19:04:54 +0000 Subject: [PATCH 1/8] [OMNIML-4962] specdec_bench cell t0_d3 Signed-off-by: chenhany --- .../_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml | 4 ++ .../specdec_bench_dflash_vllm_t0_d3.yaml | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml create mode 100644 tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml diff --git a/tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml b/tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml new file mode 100644 index 00000000000..fcf893c989e --- /dev/null +++ b/tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml @@ -0,0 +1,4 @@ +sampling_kwargs: + temperature: 0 +engine_args: + max_model_len: 40960 diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml new file mode 100644 index 00000000000..f35d00b4385 --- /dev/null +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml @@ -0,0 +1,58 @@ +job_name: Qwen3.5-4B_specdec_bench_dflash_vllm_t0_d3 + +pipeline: + global_vars: + hf_model: /hf-local/Qwen/Qwen3.5-4B + + task_0: + script: common/specdec_bench/run.sh + args: + - --dataset speed + - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/qualitative + - --engine VLLM + - --speculative_algorithm DFlash + - --draft_length 3 + - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml + - --tp_size 2 + - --ep_size 1 + - --concurrency 32 + - --output_length 4096 + - --aa_timing + - --show_progress + - --save_dir /scratchspace/qwen3.5-4b_dflash_vllm_t0_d3/qualitative + environment: + - HF_MODEL_CKPT: <> + - HF_LOCAL: /hf-local + slurm_config: + _factory_: "slurm_factory" + nodes: 1 + ntasks_per_node: 1 + gpus_per_node: 2 + container: vllm/vllm-openai:qwen3_5-cu130 + + task_1: + script: common/specdec_bench/run.sh + args: + - --dataset speed + - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/throughput_32k + - --engine VLLM + - --speculative_algorithm DFlash + - --draft_length 3 + - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml + - --tp_size 2 + - --ep_size 1 + - --concurrency 8 + - --num_requests 80 + - --output_length 4096 + - --aa_timing + - --show_progress + - --save_dir /scratchspace/qwen3.5-4b_dflash_vllm_t0_d3/throughput_32k + environment: + - HF_MODEL_CKPT: <> + - HF_LOCAL: /hf-local + slurm_config: + _factory_: "slurm_factory" + nodes: 1 + ntasks_per_node: 1 + gpus_per_node: 2 + container: vllm/vllm-openai:qwen3_5-cu130 From 0cafa1ff7a2c4d3233c775745a381a73a685c24b Mon Sep 17 00:00:00 2001 From: chenhany Date: Fri, 5 Jun 2026 14:57:35 -0700 Subject: [PATCH 2/8] fix: DFlash -> DFLASH (case-sensitive algorithm name) Signed-off-by: chenhany --- .../Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml index f35d00b4385..0dd383756dd 100644 --- a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml @@ -10,7 +10,7 @@ pipeline: - --dataset speed - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/qualitative - --engine VLLM - - --speculative_algorithm DFlash + - --speculative_algorithm DFLASH - --draft_length 3 - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml - --tp_size 2 @@ -36,7 +36,7 @@ pipeline: - --dataset speed - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/throughput_32k - --engine VLLM - - --speculative_algorithm DFlash + - --speculative_algorithm DFLASH - --draft_length 3 - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml - --tp_size 2 From c97972f9d21c83b3bcef5be51edaebbdb3e72bfb Mon Sep 17 00:00:00 2001 From: chenhany Date: Fri, 5 Jun 2026 15:04:20 -0700 Subject: [PATCH 3/8] fix: add --draft_model_dir for DFLASH (required arg) Signed-off-by: chenhany --- .../Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml index 0dd383756dd..809ca8b2652 100644 --- a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml @@ -11,6 +11,7 @@ pipeline: - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/qualitative - --engine VLLM - --speculative_algorithm DFLASH + - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - --draft_length 3 - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml - --tp_size 2 @@ -37,6 +38,7 @@ pipeline: - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/throughput_32k - --engine VLLM - --speculative_algorithm DFLASH + - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - --draft_length 3 - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml - --tp_size 2 From 5a425bfd0ac26e913e45b9ee555c7fe0cd9165cf Mon Sep 17 00:00:00 2001 From: chenhany Date: Fri, 5 Jun 2026 15:09:05 -0700 Subject: [PATCH 4/8] fix: use vllm v0.22.1 container (dflash support) Signed-off-by: chenhany --- .../Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml index 809ca8b2652..f69cf3b2193 100644 --- a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml @@ -29,7 +29,7 @@ pipeline: nodes: 1 ntasks_per_node: 1 gpus_per_node: 2 - container: vllm/vllm-openai:qwen3_5-cu130 + container: vllm/vllm-openai:v0.22.1 task_1: script: common/specdec_bench/run.sh @@ -57,4 +57,4 @@ pipeline: nodes: 1 ntasks_per_node: 1 gpus_per_node: 2 - container: vllm/vllm-openai:qwen3_5-cu130 + container: vllm/vllm-openai:v0.22.1 From 3b04b6b6e03fe5f08ff756fa0cb21fd6d3ff9507 Mon Sep 17 00:00:00 2001 From: chenhany Date: Fri, 5 Jun 2026 20:30:50 -0700 Subject: [PATCH 5/8] align cell YAML with #1564 sweep-name convention + add explanatory comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit establishes the inaugural pattern for these YAMLs: sweep-name convention `__[_]` (no dots, S3-path-safe), two-task pipeline (qualitative + throughput_32k), header docstring + per-task rationale comments, S3 upload deferred to wrap_up stage. This cell was authored before #1564 merged, so it took the SPEC's literal-string rendering of sweep_name (kept the dots and case from `Qwen3.5-4B`). Align with the merged convention so the matrix of DFlash + MTP + NONE cells lands in S3 under a single readable namespace. Changes: - sweep_name: qwen3.5-4b_dflash_vllm_t0_d3 → qwen35_4b_dflash_vllm_t0_d3 - rename _cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml accordingly - --save_dir paths updated in both tasks - --runtime_params references updated - Add header docstring mirroring #1564 (pipeline shape, sweep-name convention, S3 vault destination, container note for DFlash's v0.22.1+ requirement, slurm launch command) - Add per-task inline comments explaining tp_size / concurrency / num_requests choices - Add header to _cells/...yaml explaining why max_model_len=40960 lives in the per-cell file (both tasks share one --runtime_params) The previous successful experiment cicd_1780697785 wrote to /lustre/.../qwen3.5-4b_dflash_vllm_t0_d3// on disk; the next re-fire under this YAML will write to qwen35_4b_dflash_vllm_t0_d3/ instead. The cell ticket's INTERN-ARTIFACTS stamp will need to be re-emitted from a fresh run. Surfaced on OMNIML-4962. Signed-off-by: chenhany --- .../_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml | 4 -- .../_cells/qwen35_4b_dflash_vllm_t0_d3.yaml | 16 +++++ .../specdec_bench_dflash_vllm_t0_d3.yaml | 66 +++++++++++++++++-- 3 files changed, 76 insertions(+), 10 deletions(-) delete mode 100644 tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml create mode 100644 tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml diff --git a/tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml b/tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml deleted file mode 100644 index fcf893c989e..00000000000 --- a/tools/launcher/common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml +++ /dev/null @@ -1,4 +0,0 @@ -sampling_kwargs: - temperature: 0 -engine_args: - max_model_len: 40960 diff --git a/tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml b/tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml new file mode 100644 index 00000000000..addb33b1a9a --- /dev/null +++ b/tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml @@ -0,0 +1,16 @@ +# Per-cell runtime_params for the SPEED-Bench DFlash cell +# qwen35_4b_dflash_vllm_t0_d3 (temperature=0, draft_length=3 → +# block_size=4). Companion to +# tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml. +# +# Both pipeline tasks (qualitative + throughput_32k) load this file via +# `--runtime_params`. The temperature override is cell-specific. The +# `max_model_len: 40960` mirrors common/specdec_bench/runtime_params_throughput_32k.yaml +# for task_1 (32K input + 4K output + 4K headroom) — harmless on task_0 +# where prompts top out at ~8K. We duplicate it here instead of chaining +# two `--runtime_params` files because run.py takes a single path. + +sampling_kwargs: + temperature: 0 +engine_args: + max_model_len: 40960 diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml index f69cf3b2193..7ae70a5281c 100644 --- a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml @@ -1,9 +1,47 @@ +# SPEED-bench DFlash speculative-decoding run for Qwen3.5-4B via vLLM, +# matrix cell t0_d3 (temperature=0, draft_length=3 → block_size=4). +# +# Companion to specdec_bench_mtp.yaml. This variant exercises the +# z-lab/Qwen3.5-4B-DFlash external draft model. DFlash ignores +# --draft_length (which maps to vLLM's speculative_num_steps); it reads +# `speculative_num_draft_tokens` instead, which we pass via --block_size +# = draft_length + 1. +# +# Two-task pipeline: +# task_0 Quantitative quality split (nvidia/SPEED-Bench-Internal/qualitative) +# task_1 Long-context throughput split (nvidia/SPEED-Bench-Internal/throughput_32k) +# +# Results write to /scratchspace/qwen35_4b_dflash_vllm_t0_d3//. +# The pensieve-intern `specdec_bench` workflow's wrap_up stage owns +# publishing these to s3://team-specdec-workgroup/results/qwen35_4b_dflash_vllm_t0_d3// +# with provenance stamps (jira_ticket + huggingface_model_id). +# Sweep-name convention: ___ so +# multi-model / multi-engine / multi-cell records don't collide in S3. +# +# Container: vllm/vllm-openai:v0.22.1+ is required. The `dflash` +# speculative method landed in vLLM v0.22.0; the qwen3_5-cu130 image +# used by sibling MTP/NONE YAMLs predates this and rejects +# `--speculative_algorithm DFLASH` with "Input should be 'ngram', ..., +# 'mtp'". +# +# Slurm run on cw_dfw: +# uv run slurm.py --yaml modules/Model-Optimizer/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml --yes + job_name: Qwen3.5-4B_specdec_bench_dflash_vllm_t0_d3 pipeline: global_vars: hf_model: /hf-local/Qwen/Qwen3.5-4B + # Step 1: qualitative split — quality / acceptance-rate numbers with + # DFlash block_size=4 (draft_length=3 + 1). tp_size=2 + concurrency=32 + # trades aa_timing fidelity for ~30x wall-clock speedup; + # acceptance-length (AL) is concurrency-independent and is the primary + # metric we care about for this split. + # + # --runtime_params is the per-cell file (sampling_kwargs.temperature + # for this cell + the throughput_32k engine_args pin that task_1 + # needs — harmless on qualitative since prompts top out at ~8K). task_0: script: common/specdec_bench/run.sh args: @@ -12,15 +50,15 @@ pipeline: - --engine VLLM - --speculative_algorithm DFLASH - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - - --draft_length 3 - - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml + - --block_size 4 + - --runtime_params common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml - --tp_size 2 - --ep_size 1 - --concurrency 32 - --output_length 4096 - --aa_timing - --show_progress - - --save_dir /scratchspace/qwen3.5-4b_dflash_vllm_t0_d3/qualitative + - --save_dir /scratchspace/qwen35_4b_dflash_vllm_t0_d3/qualitative environment: - HF_MODEL_CKPT: <> - HF_LOCAL: /hf-local @@ -31,6 +69,14 @@ pipeline: gpus_per_node: 2 container: vllm/vllm-openai:v0.22.1 + # Step 2: throughput_32k split — long-context throughput with DFlash + # block_size=4. `--num_requests 80` caps the run at 80 samples (split + # has 1,536) so it fits in the 4h Slurm time-limit; each 32K-input + # sample takes ~60-90s. tp_size=2 doubles the KV-cache budget across + # 2 GPUs; concurrency=8 keeps 8 * 32K = 256K tokens of in-flight KV + # under that doubled budget. Same per-cell --runtime_params file as + # task_0 — its max_model_len=40960 pin is what this split actually + # needs. task_1: script: common/specdec_bench/run.sh args: @@ -39,8 +85,8 @@ pipeline: - --engine VLLM - --speculative_algorithm DFLASH - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - - --draft_length 3 - - --runtime_params common/specdec_bench/_cells/qwen3.5-4b_dflash_vllm_t0_d3.yaml + - --block_size 4 + - --runtime_params common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml - --tp_size 2 - --ep_size 1 - --concurrency 8 @@ -48,7 +94,7 @@ pipeline: - --output_length 4096 - --aa_timing - --show_progress - - --save_dir /scratchspace/qwen3.5-4b_dflash_vllm_t0_d3/throughput_32k + - --save_dir /scratchspace/qwen35_4b_dflash_vllm_t0_d3/throughput_32k environment: - HF_MODEL_CKPT: <> - HF_LOCAL: /hf-local @@ -58,3 +104,11 @@ pipeline: ntasks_per_node: 1 gpus_per_node: 2 container: vllm/vllm-openai:v0.22.1 + + +# S3 upload is intentionally not a task in this YAML — the bench +# pipeline only writes results to /scratchspace/qwen35_4b_dflash_vllm_t0_d3//. +# The pensieve-intern specdec_bench workflow's wrap_up stage owns +# harvesting these from lustre and publishing them to the team S3 vault +# with provenance stamps (jira_ticket + huggingface_model_id) for the +# "official record" tracking. From 89fd50687f1ae3b52f66289885e1374b38b06ded Mon Sep 17 00:00:00 2001 From: chenhany Date: Fri, 5 Jun 2026 20:58:02 -0700 Subject: [PATCH 6/8] drop redundant per-cell runtime_params for t0_d3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-cell file at common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml contained: sampling_kwargs: {temperature: 0} engine_args: {max_model_len: 40960} Both are redundant for this cell: - run.py's default sampling_kwargs is {"temperature": 0} when no --runtime_params is supplied (see run.py:149) — t0_* cells just don't pass --runtime_params for task_0. - max_model_len=40960 already lives in the shared common/specdec_bench/runtime_params_throughput_32k.yaml (#1564), which is exactly what specdec_bench_mtp.yaml task_1 uses. Point task_1 at the shared file instead of duplicating it per cell. Per-cell runtime_params become necessary only when temperature != 0 (future t1_d3 / t1_d7 cells); at that point we can either spawn a per-cell file or add a --temperature CLI arg to run.py. Cross that bridge when those cells land. PR is now just the single example YAML (no _cells/ file). Signed-off-by: chenhany --- .../_cells/qwen35_4b_dflash_vllm_t0_d3.yaml | 16 ---------------- .../specdec_bench_dflash_vllm_t0_d3.yaml | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 24 deletions(-) delete mode 100644 tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml diff --git a/tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml b/tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml deleted file mode 100644 index addb33b1a9a..00000000000 --- a/tools/launcher/common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Per-cell runtime_params for the SPEED-Bench DFlash cell -# qwen35_4b_dflash_vllm_t0_d3 (temperature=0, draft_length=3 → -# block_size=4). Companion to -# tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml. -# -# Both pipeline tasks (qualitative + throughput_32k) load this file via -# `--runtime_params`. The temperature override is cell-specific. The -# `max_model_len: 40960` mirrors common/specdec_bench/runtime_params_throughput_32k.yaml -# for task_1 (32K input + 4K output + 4K headroom) — harmless on task_0 -# where prompts top out at ~8K. We duplicate it here instead of chaining -# two `--runtime_params` files because run.py takes a single path. - -sampling_kwargs: - temperature: 0 -engine_args: - max_model_len: 40960 diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml index 7ae70a5281c..b6514b4ba5a 100644 --- a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml @@ -39,9 +39,11 @@ pipeline: # acceptance-length (AL) is concurrency-independent and is the primary # metric we care about for this split. # - # --runtime_params is the per-cell file (sampling_kwargs.temperature - # for this cell + the throughput_32k engine_args pin that task_1 - # needs — harmless on qualitative since prompts top out at ~8K). + # No --runtime_params: run.py defaults sampling_kwargs to + # {"temperature": 0} when none is supplied, which is exactly what this + # cell (t0_*) wants. Cells that need non-default temperature + # (t1_d3 / t1_d7) will supply --runtime_params (or take a future + # --temperature CLI arg). task_0: script: common/specdec_bench/run.sh args: @@ -51,7 +53,6 @@ pipeline: - --speculative_algorithm DFLASH - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - --block_size 4 - - --runtime_params common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml - --tp_size 2 - --ep_size 1 - --concurrency 32 @@ -74,9 +75,11 @@ pipeline: # has 1,536) so it fits in the 4h Slurm time-limit; each 32K-input # sample takes ~60-90s. tp_size=2 doubles the KV-cache budget across # 2 GPUs; concurrency=8 keeps 8 * 32K = 256K tokens of in-flight KV - # under that doubled budget. Same per-cell --runtime_params file as - # task_0 — its max_model_len=40960 pin is what this split actually - # needs. + # under that doubled budget. + # + # --runtime_params is the shared throughput_32k pin (engine_args.max_model_len=40960) + # mirroring specdec_bench_mtp.yaml task_1 — vLLM's auto-derivation can + # otherwise cap max_model_len below the 32K input we need. task_1: script: common/specdec_bench/run.sh args: @@ -86,7 +89,7 @@ pipeline: - --speculative_algorithm DFLASH - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - --block_size 4 - - --runtime_params common/specdec_bench/_cells/qwen35_4b_dflash_vllm_t0_d3.yaml + - --runtime_params common/specdec_bench/runtime_params_throughput_32k.yaml - --tp_size 2 - --ep_size 1 - --concurrency 8 From 15c5170443410106b015d987da860ad5c48129f0 Mon Sep 17 00:00:00 2001 From: chenhany Date: Fri, 5 Jun 2026 21:12:13 -0700 Subject: [PATCH 7/8] switch to --max_seq_len 40960 (matches the --max_seq_len CLI flag added in #1564) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1564 replaced common/specdec_bench/runtime_params_throughput_32k.yaml with a generic --max_seq_len CLI flag that maps to the engine-specific kwarg at run.py's seam (VLLM → max_model_len). Following suit on this cell so the runtime_params indirection disappears entirely. Also reflows the inline comments to match the new convention. Signed-off-by: chenhany --- .../specdec_bench_dflash_vllm_t0_d3.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml index b6514b4ba5a..8333f50c67b 100644 --- a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml @@ -39,11 +39,10 @@ pipeline: # acceptance-length (AL) is concurrency-independent and is the primary # metric we care about for this split. # - # No --runtime_params: run.py defaults sampling_kwargs to - # {"temperature": 0} when none is supplied, which is exactly what this - # cell (t0_*) wants. Cells that need non-default temperature - # (t1_d3 / t1_d7) will supply --runtime_params (or take a future - # --temperature CLI arg). + # No --temperature: run.py defaults sampling_kwargs to + # {"temperature": 0} when --temperature is not supplied, which is + # exactly what this cell (t0_*) wants. Cells with non-zero temperature + # (t1_d3 / t1_d7) will pass `--temperature 1` on the args list. task_0: script: common/specdec_bench/run.sh args: @@ -77,9 +76,11 @@ pipeline: # 2 GPUs; concurrency=8 keeps 8 * 32K = 256K tokens of in-flight KV # under that doubled budget. # - # --runtime_params is the shared throughput_32k pin (engine_args.max_model_len=40960) - # mirroring specdec_bench_mtp.yaml task_1 — vLLM's auto-derivation can - # otherwise cap max_model_len below the 32K input we need. + # --max_seq_len 40960 pins the engine's sequence-length cap for the + # 32K input + 4K output + 4K headroom; vLLM's auto-derivation from + # gpu_memory_utilization can otherwise cap below the 32K input we + # need. Generic CLI flag (run.py maps it to engine-specific kwarg — + # max_model_len for vLLM here). task_1: script: common/specdec_bench/run.sh args: @@ -89,7 +90,7 @@ pipeline: - --speculative_algorithm DFLASH - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - --block_size 4 - - --runtime_params common/specdec_bench/runtime_params_throughput_32k.yaml + - --max_seq_len 40960 - --tp_size 2 - --ep_size 1 - --concurrency 8 From 1f264fc306a21c00e3e1948b3b48987d67fe5b9a Mon Sep 17 00:00:00 2001 From: Chenhan Yu Date: Wed, 10 Jun 2026 10:55:12 -0700 Subject: [PATCH 8/8] specdec_bench: refactor Qwen3.5-4B DFlash example to parent-only (post-#1564) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the cell-suffix (`_t0_d3`) from the YAML filename + content so the file is a per-`(model, algorithm, engine)` parent that all four cells of OMNIML-4961 (t0_d3 / t0_d7 / t1_d3 / t1_d7) share. Matches the post-#1564 convention used by the canonical parents on main: tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_mtp_vllm.yaml tools/launcher/examples/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ specdec_bench_dflash_vllm.yaml Cells now override the per-cell knobs (temperature, save_dir, num_requests, max_seq_len) via `pipeline.task_N.args+=[...]` at slurm-invoke time. No per-cell YAML file is committed. Rename: specdec_bench_dflash_vllm_t0_d3.yaml → specdec_bench_dflash_vllm.yaml Content changes: - Drop `t0_d3` from `job_name` - Drop `t0_d3` suffix from `--save_dir` paths (uses `{sweep_name_default}` placeholder so cells override per-split) - Add `global_vars.draft_model` for `<>` substitution on both tasks (was hardcoded `/hf-local/z-lab/...`) - Drop the `--max_seq_len 40960` knob from task_1 — cells pass `--max_seq_len 65536` at slurm-invoke time (sized for SPEED-bench's longest ~46k-token prompts, per the cell.md sizing guidance) - Rewrite header comments to document the post-#1564 CLI-override pattern + cite the reference run (cicd_1780697785 / OMNIML-4962) Reference run unchanged — qualitative AL=5.0395, throughput_32k AL=6.8958 from OMNIML-4962's INTERN-ARTIFACTS, produced from the same slurm pipeline shape this parent now expresses. Signed-off-by: Chenhan Yu --- .../Qwen3.5-4B/specdec_bench_dflash_vllm.yaml | 99 +++++++++++++++ .../specdec_bench_dflash_vllm_t0_d3.yaml | 118 ------------------ 2 files changed, 99 insertions(+), 118 deletions(-) create mode 100644 tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm.yaml delete mode 100644 tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm.yaml new file mode 100644 index 00000000000..0620b4c78c9 --- /dev/null +++ b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm.yaml @@ -0,0 +1,99 @@ +# SPEED-bench DFlash speculative-decoding run for Qwen3.5-4B via vLLM. +# +# Companion to specdec_bench_mtp_vllm.yaml. This variant exercises the +# z-lab/Qwen3.5-4B-DFlash external draft model. DFlash uses +# vLLM's `speculative_num_draft_tokens`, which the specdec_bench wrapper +# computes from `--block_size`. `--block_size = draft_length + 1` +# (e.g. draft_length=3 → block_size=4; draft_length=7 → block_size=8). +# +# Container: `vllm/vllm-openai:v0.22.1`+ is required. The `dflash` +# speculative method landed in vLLM v0.22.0; older tags reject +# `--speculative_algorithm DFLASH` with "Input should be 'ngram', ..., +# 'mtp'". +# +# Cells in OMNIML-4961 (t0_d3 / t0_d7 / t1_d3 / t1_d7) override per-cell +# knobs (temperature, block_size, save_dir, num_requests) via +# pipeline.task_N.args+=[...] at slurm-invoke time — no per-cell file is +# committed (post-#1564 convention). +# +# Slurm run on cw_dfw: +# uv run slurm.py \ +# --yaml modules/Model-Optimizer/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm.yaml \ +# --yes detach=true \ +# pipeline.task_0.args+=["--temperature 0","--block_size 4","--save_dir /scratchspace//qualitative"] \ +# pipeline.task_1.args+=["--temperature 0","--block_size 4","--save_dir /scratchspace//throughput_32k","--num_requests 80","--max_seq_len 65536"] +# +# Reference run: cicd_1780697785 (OMNIML-4962, cw_dfw) produced +# qualitative Average_AL = 5.0395 +# throughput_32k Average_AL = 6.8958 +# See OMNIML-4962 INTERN-ARTIFACTS worklog for the full payload. + +job_name: Qwen3.5-4B_specdec_bench_dflash_vllm + +pipeline: + global_vars: + hf_model: /hf-local/Qwen/Qwen3.5-4B + draft_model: /hf-local/z-lab/Qwen3.5-4B-DFlash + + # task_0: SPEED qualitative split — quality / acceptance-rate numbers. + # tp_size=2 + concurrency=32 trades aa_timing fidelity for ~30x + # wall-clock speedup; acceptance-length (AL) is concurrency-independent + # and is the primary metric we care about for this split. + task_0: + script: common/specdec_bench/run.sh + args: + - --dataset speed + - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/qualitative + - --engine VLLM + - --speculative_algorithm DFLASH + - --draft_model_dir <> + - --block_size 4 + - --tp_size 2 + - --ep_size 1 + - --concurrency 32 + - --output_length 4096 + - --aa_timing + - --show_progress + - --save_dir /scratchspace/{sweep_name_default}/qualitative + environment: + - HF_MODEL_CKPT: <> + - HF_LOCAL: /hf-local + slurm_config: + _factory_: "slurm_factory" + nodes: 1 + ntasks_per_node: 1 + gpus_per_node: 2 + container: vllm/vllm-openai:v0.22.1 + + # task_1: SPEED throughput_32k split — long-context throughput. + # `--num_requests 80` caps the run at 80 samples (split has 1,536) so + # it fits in the 4h Slurm time-limit; each 32K-input sample takes + # ~60-90s. tp_size=2 doubles the KV-cache budget across 2 GPUs; + # concurrency=8 keeps 8 * 32K = 256K tokens of in-flight KV under that + # doubled budget. + task_1: + script: common/specdec_bench/run.sh + args: + - --dataset speed + - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/throughput_32k + - --engine VLLM + - --speculative_algorithm DFLASH + - --draft_model_dir <> + - --block_size 4 + - --tp_size 2 + - --ep_size 1 + - --concurrency 8 + - --num_requests 80 + - --output_length 4096 + - --aa_timing + - --show_progress + - --save_dir /scratchspace/{sweep_name_default}/throughput_32k + environment: + - HF_MODEL_CKPT: <> + - HF_LOCAL: /hf-local + slurm_config: + _factory_: "slurm_factory" + nodes: 1 + ntasks_per_node: 1 + gpus_per_node: 2 + container: vllm/vllm-openai:v0.22.1 diff --git a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml b/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml deleted file mode 100644 index 8333f50c67b..00000000000 --- a/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# SPEED-bench DFlash speculative-decoding run for Qwen3.5-4B via vLLM, -# matrix cell t0_d3 (temperature=0, draft_length=3 → block_size=4). -# -# Companion to specdec_bench_mtp.yaml. This variant exercises the -# z-lab/Qwen3.5-4B-DFlash external draft model. DFlash ignores -# --draft_length (which maps to vLLM's speculative_num_steps); it reads -# `speculative_num_draft_tokens` instead, which we pass via --block_size -# = draft_length + 1. -# -# Two-task pipeline: -# task_0 Quantitative quality split (nvidia/SPEED-Bench-Internal/qualitative) -# task_1 Long-context throughput split (nvidia/SPEED-Bench-Internal/throughput_32k) -# -# Results write to /scratchspace/qwen35_4b_dflash_vllm_t0_d3//. -# The pensieve-intern `specdec_bench` workflow's wrap_up stage owns -# publishing these to s3://team-specdec-workgroup/results/qwen35_4b_dflash_vllm_t0_d3// -# with provenance stamps (jira_ticket + huggingface_model_id). -# Sweep-name convention: ___ so -# multi-model / multi-engine / multi-cell records don't collide in S3. -# -# Container: vllm/vllm-openai:v0.22.1+ is required. The `dflash` -# speculative method landed in vLLM v0.22.0; the qwen3_5-cu130 image -# used by sibling MTP/NONE YAMLs predates this and rejects -# `--speculative_algorithm DFLASH` with "Input should be 'ngram', ..., -# 'mtp'". -# -# Slurm run on cw_dfw: -# uv run slurm.py --yaml modules/Model-Optimizer/tools/launcher/examples/Qwen/Qwen3.5-4B/specdec_bench_dflash_vllm_t0_d3.yaml --yes - -job_name: Qwen3.5-4B_specdec_bench_dflash_vllm_t0_d3 - -pipeline: - global_vars: - hf_model: /hf-local/Qwen/Qwen3.5-4B - - # Step 1: qualitative split — quality / acceptance-rate numbers with - # DFlash block_size=4 (draft_length=3 + 1). tp_size=2 + concurrency=32 - # trades aa_timing fidelity for ~30x wall-clock speedup; - # acceptance-length (AL) is concurrency-independent and is the primary - # metric we care about for this split. - # - # No --temperature: run.py defaults sampling_kwargs to - # {"temperature": 0} when --temperature is not supplied, which is - # exactly what this cell (t0_*) wants. Cells with non-zero temperature - # (t1_d3 / t1_d7) will pass `--temperature 1` on the args list. - task_0: - script: common/specdec_bench/run.sh - args: - - --dataset speed - - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/qualitative - - --engine VLLM - - --speculative_algorithm DFLASH - - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - - --block_size 4 - - --tp_size 2 - - --ep_size 1 - - --concurrency 32 - - --output_length 4096 - - --aa_timing - - --show_progress - - --save_dir /scratchspace/qwen35_4b_dflash_vllm_t0_d3/qualitative - environment: - - HF_MODEL_CKPT: <> - - HF_LOCAL: /hf-local - slurm_config: - _factory_: "slurm_factory" - nodes: 1 - ntasks_per_node: 1 - gpus_per_node: 2 - container: vllm/vllm-openai:v0.22.1 - - # Step 2: throughput_32k split — long-context throughput with DFlash - # block_size=4. `--num_requests 80` caps the run at 80 samples (split - # has 1,536) so it fits in the 4h Slurm time-limit; each 32K-input - # sample takes ~60-90s. tp_size=2 doubles the KV-cache budget across - # 2 GPUs; concurrency=8 keeps 8 * 32K = 256K tokens of in-flight KV - # under that doubled budget. - # - # --max_seq_len 40960 pins the engine's sequence-length cap for the - # 32K input + 4K output + 4K headroom; vLLM's auto-derivation from - # gpu_memory_utilization can otherwise cap below the 32K input we - # need. Generic CLI flag (run.py maps it to engine-specific kwarg — - # max_model_len for vLLM here). - task_1: - script: common/specdec_bench/run.sh - args: - - --dataset speed - - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/throughput_32k - - --engine VLLM - - --speculative_algorithm DFLASH - - --draft_model_dir /hf-local/z-lab/Qwen3.5-4B-DFlash - - --block_size 4 - - --max_seq_len 40960 - - --tp_size 2 - - --ep_size 1 - - --concurrency 8 - - --num_requests 80 - - --output_length 4096 - - --aa_timing - - --show_progress - - --save_dir /scratchspace/qwen35_4b_dflash_vllm_t0_d3/throughput_32k - environment: - - HF_MODEL_CKPT: <> - - HF_LOCAL: /hf-local - slurm_config: - _factory_: "slurm_factory" - nodes: 1 - ntasks_per_node: 1 - gpus_per_node: 2 - container: vllm/vllm-openai:v0.22.1 - - -# S3 upload is intentionally not a task in this YAML — the bench -# pipeline only writes results to /scratchspace/qwen35_4b_dflash_vllm_t0_d3//. -# The pensieve-intern specdec_bench workflow's wrap_up stage owns -# harvesting these from lustre and publishing them to the team S3 vault -# with provenance stamps (jira_ticket + huggingface_model_id) for the -# "official record" tracking.