From 6f3a3d05492c6f4f7ff05ed3ef5cea4158505e0b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 8 Aug 2026 22:13:51 +0000 Subject: [PATCH] fix(qwen3_5): load quantized lm_head, not just BF16 (#164) Reported by rohitpaul: both public Qwen3.6-27B NVFP4 checkpoints die at load with "dense loader: expected BF16 for lm_head.weight". Confirmed in-tree and, more usefully, confirmed WHY. Our benchmarks are not wrong, and the checkpoint is not wrong: the repo was re-quantized under us. Reading the safetensors headers of both snapshots we hold on the lab box: unsloth/Qwen3.6-27B-NVFP4 @890bdef7 lm_head.weight BF16 [248320, 5120] unsloth/Qwen3.6-27B-NVFP4 @ccdaab7e lm_head.weight F8_E4M3 [248320, 5120] lm_head.weight_scale BF16 [248320, 1] @890bdef7 is the snapshot every recorded 27B-NVFP4 number ran on, which is why an unconditional BF16 head survived this long: the body has had real NVFP4 support all along, the head was simply never quantized under us. The reporter used @ccdaab7e. nvidia/Qwen3.6-27B-NVFP4 ships a third form, a ModelOpt NVFP4 head. So the support claim was silently revision-locked. LoadLmHeadAnyDtype now dispatches on the stored dtype: BF16 weight [V,H] -> transpose (unchanged) F8_E4M3 weight [V,H] + weight_scale [V,1] or scalar -> per-row/per-tensor U8 weight [V,H/2] + weight_scale F8 [V,H/16] + weight_scale_2 | weight_global_scale -> nvfp4 Note the scale is PER-OUTPUT-CHANNEL in @ccdaab7e, not per-tensor as the issue guessed, so the existing scalar-scale FP8 loader would have been quietly wrong rather than loud. All three forms land on the same bf16 [in, out] Matmul-B operand the logits GEMM already consumes, so the forward is untouched and a BF16 head takes the identical call as before -- byte-exact, no dequant. ModelOpt spells the global scale weight_scale_2 and stores the scale itself; compressed-tensors spells it weight_global_scale and stores the reciprocal. Both are accepted, converted to the CT divisor convention so the shared DequantCtNvfp4WeightToF32 computes each exactly. An unsupported dtype now names what it saw and what is accepted, instead of claiming BF16 was expected. Keeping the head QUANTIZED end to end would save ~2.3 GiB over dequantizing to bf16; that needs an lm_head_fp4-style field plus a forward branch and is a follow-up, not this fix. This restores loadability at the memory profile we already benchmark. Test: tests/vllm/models/test_qwen3_5_lm_head_dtypes.cpp, 5 cases / 32 assertions, synthetic tensors only (no checkpoint, no GPU): BF16 unchanged, FP8 per-output-channel, FP8 per-tensor, unsupported dtype names itself, and FP8 missing its scale fails loudly. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Claude Opus 5 (1M context) --- docs/BENCHMARKS.md | 2 +- docs/FEATURES.md | 2 +- docs/STATUS.md | 2 +- docs/USAGE.md | 15 ++ .../model_executor/models/qwen3_5_dense.h | 18 +- .../models/qwen3_5_dense_weights.cpp | 129 +++++++++++- tests/CMakeLists.txt | 3 + .../models/test_qwen3_5_lm_head_dtypes.cpp | 190 ++++++++++++++++++ 8 files changed, 355 insertions(+), 6 deletions(-) create mode 100644 tests/vllm/models/test_qwen3_5_lm_head_dtypes.cpp diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 894c4b3d0..15ab81f30 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -30,7 +30,7 @@ The binding comparison. vLLM runs its **production graphed config**, never | Model | Quant | vLLM pin | Axes passing | Disposition | |---|---|---|---:|---| -| Qwen3.6-27B | NVFP4 | 0.25.0 | **115/124** | Effective parity-or-better, two-grid totality | +| Qwen3.6-27B | NVFP4 | 0.25.0 | **115/124** | Effective parity-or-better, two-grid totality. Measured on `unsloth/Qwen3.6-27B-NVFP4` @`890bdef7` (BF16 head); @`ccdaab7e` re-quantized the head to FP8 | | Qwen3.6-35B-A3B | NVFP4 `modelopt_mixed` | 0.25.0 | 2/18 | 3-rep grid 2026-08-05 @`1ea26427`: 0.93-1.03x (c4 wins), c16 0.93x. Both c16 levers A/B'd NEG: drain event -1.9%, mirror 0.999x. ★ probe found a prod async batch-1 greedy DEGENERATION bug the mirror fixes | | DeepSeek-V2-Lite | bf16 MLA | 0.25.0 | 4/25 | Attributed miss, row stays `ACTIVE` | | Qwen3.5-4B | bf16 direct-load | 0.26.0.dev0 | throughput + host PSS | Exact chunks ON: total **1.021x PASS**; TTFT **1.086x**, TPOT **1.025x**, VRAM **1.018x OPEN**; local A/B **+2.152%** ([evidence](bench-evidence/qwen35-4b-sm120-main-20260807.md)) | diff --git a/docs/FEATURES.md b/docs/FEATURES.md index d5fe4fc14..d88f1027a 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -94,7 +94,7 @@ speed-pending, which [BENCHMARKS.md](BENCHMARKS.md) tracks. | Architecture | Tested checkpoint(s) | Correctness gate | Speed vs reference | |---|---|---|---| -| `Qwen3_5ForConditionalGeneration` | Qwen3.6-27B NVFP4; Qwen3.5-4B BF16 | 27B strict 235/235 text + 32/32 image/video; 4B cached 3/3 | 27B at/above vLLM; 4B throughput 1.021x, latency/VRAM pending. CUDA/CPU only; the off-CUDA host-pointer bug (#125) is fixed but unrun | +| `Qwen3_5ForConditionalGeneration` | Qwen3.6-27B NVFP4; Qwen3.5-4B BF16 | 27B strict 235/235 text + 32/32 image/video; 4B cached 3/3 | 27B at/above vLLM; 4B throughput 1.021x, latency/VRAM pending. `lm_head` loads BF16, FP8 or NVFP4 (#164). CUDA/CPU only; the off-CUDA host-pointer bug (#125) is fixed but unrun | | `Qwen3_5MoeForConditionalGeneration` | Qwen3.6-35B-A3B (NVFP4, GDN MoE) | strict 315/315 text vs vLLM 0.25.0 | gate model: 0.93x to 1.03x grid | | `Qwen3ForCausalLM` | Qwen3 dense 0.6B/1.7B/4B/32B, NVFP4A16 | near-tie strict 16/16 vs vLLM 0.25.0 | c1 every-axis parity, c8 decode residual | | `Qwen3MoeForCausalLM` | Qwen3-Coder-30B-A3B | strict 6/6 vs vLLM 0.25.0 | 11/16 grid cells at or above graphed vLLM | diff --git a/docs/STATUS.md b/docs/STATUS.md index 6a1438c6d..090d32d07 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -69,7 +69,7 @@ token-for-token correctness against the pinned oracle. | Capability | State | Notes | |---|---|---| -| Qwen3.6-27B (NVFP4) text generation | Correctness-complete, at/above vLLM speed | Token-exact greedy on GB10; beats vLLM 0.25.0 total throughput at every concurrency (1.007-1.045x), effective parity 115/124 axes | +| Qwen3.6-27B (NVFP4) text generation | Correctness-complete, at/above vLLM speed | Token-exact greedy on GB10; beats vLLM 0.25.0 tput at every concurrency (1.007-1.045x), parity 115/124 axes. FP8/NVFP4 heads load (#164) | | Qwen3.6-35B-A3B (NVFP4, GDN MoE) | Correctness-complete; 3-rep grid 0.93-1.03x. Async batch-1 token-0 degeneration FIXED: `VT_ASYNC_DEVICE_MIRROR` default ON | Token-exact SYNC+ASYNC (RED→GREEN); c16 0.93x; `VT_ASYNC_EXECUTOR` Option A (H2D out of capture) GREEN+RED but A/B NEUTRAL → OFF; c16 residual is prefill glue | | Qwen3 / Qwen2 dense (BF16) | Correctness-complete, speed-pending. Async-serving P0 FIXED (`ROW-SERVE-ASYNC-DENSE-MIRROR`): classic-dense `Qwen3ForCausalLM` now honors the async device token-ids mirror; CPU-only -Werror test-guard fixes x2 | Near-tie-robust token-exact vs vLLM (Qwen3-0.6B, Qwen3-4B); c1 effective parity, c8 decode residual. **Async device-mirror (`ROW-SERVE-ASYNC-DENSE-MIRROR`, `f9c969ae`): the #31 fix ported to the classic dense family, dgx-VERIFIED.** The shared dense `EmbedInto` (qwen3.cpp) raced the async combine's device input-ids write against a stale host upload → token-0 degeneration on the depth-2 AsyncLLM serving path (quant-independent). `EmbedInto` now consumes the device override published by `ForwardQwen3ForCausalLM`'s `DeviceTokenIdsScope` (27B-dense template); gate `test_qwen3_dense_async_serving` RED on `VT_ASYNC_DEVICE_MIRROR=0`, GREEN default, byte-identical mirror-off. dgx GB10: async gate RED→GREEN 0.6B+4B, SACRED 0.6B+4B 184/184 unchanged (byte-neutral sync path), memcheck 0 errors; Yi30/Qwen3-8B-MXFP4 default-config e2e coherent + 3/4 token-exact (p2 = oracle-ratified near-tie, gap 0.0000), closing the QUANT-CT-MXFP4 async-default residual. RESIDUAL: sibling InternLM2/Mistral/Llama scope one-liner; W4 bench RAN; FA2 GQA-swap default-ON, c2-c8 <1.0x. `FLASH-PTXAS` #82: codegen at PARITY (no ptxas lever); gap=engine context. **D1 (2026-07-31, `CLAIM-D1-BF16-MERGED-QKV`): the bf16 merged-QKV path (`Qwen3QkvMergeEnabled`/`VT_QWEN3_QKV_MERGE`) is now default-ON** — one `vt::MatmulBT` over the merged `[qdim+2kdim,H]` owner + a contiguous `vt::QkvSplit` (OLMo-2 exemplar), replacing three per-shard GEMMs. Bit-exact GEMM math (A/B unit `test_ops_qkv_merge` byte-identical, RED-first); the wider-N cuBLASLt K-reduction flips the 0.6B genuine bf16 near-tie so the SACRED 0.6B golden was regenerated (all tokens within the near-tie band, max 0.125 nats), while Qwen3-4B is byte-neutral (0 diffs, stays STRICT). Re-gated 0.6B 16/16 + 4B 16/16; consistency/launch-count fold (measured NEUTRAL on 4B decode), no new throughput owed | | Qwen3.5-4B plain BF16 direct loading on discrete CUDA | Correctness-complete; throughput passes, latency/VRAM open | Exact GDN chunks default ON and byte-identical to rollback. Local A/B: total/output +2.152%, TTFT -2.945%, TPOT/ITL -1.920%; sealed-vLLM comparison 1.021x throughput, 1.086x TTFT, 1.025x TPOT, +233 MiB VRAM ([evidence](bench-evidence/qwen35-4b-sm120-main-20260807.md)) | diff --git a/docs/USAGE.md b/docs/USAGE.md index 32c3240f3..c5575e697 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -50,6 +50,21 @@ Two more example binaries ship alongside it: - `tokenize` ([`examples/tokenize/main.cpp`](../examples/tokenize/main.cpp)), a tokenizer smoke tool taking ` `. +### Quantized checkpoints: which `lm_head` forms load + +Publishers do not agree on how the output head is stored, and a single repo can +change it between revisions. For the Qwen3.6 dense family we accept all three +forms in use, so pick a checkpoint by its quality, not by its head: + +| `lm_head.weight` | Companion tensors | Seen in | +|---|---|---| +| `BF16` | none | `unsloth/Qwen3.6-27B-NVFP4` @`890bdef7` | +| `F8_E4M3` | `lm_head.weight_scale` (per-output-channel or per-tensor) | `unsloth/Qwen3.6-27B-NVFP4` @`ccdaab7e` | +| `U8` NVFP4 | `lm_head.weight_scale` + `weight_scale_2` (ModelOpt) or `weight_global_scale` (compressed-tensors) | `nvidia/Qwen3.6-27B-NVFP4` | + +The head is dequantized to BF16 at load, so all three cost the same memory once +running. Any other dtype fails at load with a message naming what it saw. + ## OpenAI-compatible server `server` is a small HTTP server speaking the OpenAI API. Source: diff --git a/include/vllm/model_executor/models/qwen3_5_dense.h b/include/vllm/model_executor/models/qwen3_5_dense.h index 051e37dc8..f470bf9dd 100644 --- a/include/vllm/model_executor/models/qwen3_5_dense.h +++ b/include/vllm/model_executor/models/qwen3_5_dense.h @@ -31,6 +31,8 @@ #include #include "vllm/model_executor/models/qwen3_5.h" // PagedKvCache, GdnStateCache + v1 attention metadata +#include + #include "vllm/model_executor/models/qwen3_5_weights.h" // OwnedTensor, Gdn/FullAttn weights, TensorResolver #include "vllm/transformers_utils/hf_config.h" #include "vt/device.h" @@ -95,11 +97,14 @@ struct Qwen3_5DenseLayerWeights { DenseMlpWeights mlp; // every layer has a dense MLP }; -// Whole dense-model text weights. lm_head is bf16 (unquantized in the 27B). +// Whole dense-model text weights. `lm_head` is always materialized bf16 here, but +// the CHECKPOINT may store it BF16, FP8 (per-channel scale) or ModelOpt NVFP4 — +// the 27B NVFP4 publishers disagree, and revisions of one repo disagree with each +// other (issue #164). LoadLmHeadAnyDtype dequantizes all three to this operand. struct Qwen3_5DenseWeights { OwnedTensor embed_tokens; // bf16 [vocab, H] (NOT transposed; embed lookup) OwnedTensor final_norm; // bf16 [H] - OwnedTensor lm_head; // bf16 [H, vocab] (unquantized -> Matmul-B layout) + OwnedTensor lm_head; // bf16 [H, vocab] (dequantized -> Matmul-B layout) // Mirrors tie_word_embeddings: logits reuse embed_tokens as raw [V,H] // torch-Linear storage, so no second host/device owner is created. bool tied_lm_head = false; @@ -121,6 +126,15 @@ bool IsQwen27QuantizedLinear(const std::string& name); // reciprocates the global scale), rounds to bf16, and transposes. Exposed for // unit testing. The `.input_global_scale` (activation divisor) is ignored // on this bf16-activation correctness path (notes §3.4 / §5 step-6a). +// `lm_head` across the three storage forms the 27B NVFP4 publishers actually ship +// (issue #164): BF16, FP8 `+_scale` (per-output-channel or per-tensor), and NVFP4 +// `+_scale` `+_scale_2`/`+_global_scale`. Always returns bf16 [in, out] Matmul-B, +// so a BF16 head is byte-identical to the previous LoadBf16Transposed call. +// `has` probes optional companion tensors. Exported for the loader gate. +OwnedTensor LoadLmHeadAnyDtype(const TensorResolver& get, + const std::function& has, + const std::string& name); + OwnedTensor MaterializeCtNvfp4Bf16Transposed(const TensorResolver& get, const std::string& proj); diff --git a/src/vllm/model_executor/models/qwen3_5_dense_weights.cpp b/src/vllm/model_executor/models/qwen3_5_dense_weights.cpp index 6645d0cfe..ad396d6c4 100644 --- a/src/vllm/model_executor/models/qwen3_5_dense_weights.cpp +++ b/src/vllm/model_executor/models/qwen3_5_dense_weights.cpp @@ -13,6 +13,7 @@ #include #include "vllm/model_executor/layers/quantization/compressed_tensors/nvfp4_emulation.h" +#include "vllm/model_executor/model_loader/nvfp4_dequant.h" #include "vllm/model_executor/models/dense_weight_loaders.h" #include "vllm/platforms/interface.h" #include "vt/backend.h" @@ -187,6 +188,132 @@ Nvfp4Weight LoadCtNvfp4Raw(const TensorResolver& get, const std::string& proj) { return r; } +// --- lm_head dtype dispatch (issue #164) -------------------------------------- +// The 27B NVFP4 publishers do NOT agree on the OUTPUT HEAD, and the head is not +// a compressed-tensors Linear, so none of the scheme probes above cover it: +// +// BF16 `lm_head.weight` [V,H] (transpose) +// F8_E4M3 `lm_head.weight` [V,H] + `.weight_scale` [V,1] or [] (per-row/scalar) +// U8 `lm_head.weight` [V,H/2] + `.weight_scale` F8 [V,H/16] +// + `.weight_scale_2` f32 (ModelOpt NVFP4) +// +// This loader was written against `unsloth/Qwen3.6-27B-NVFP4` @890bdef7, which +// ships a BF16 head — the snapshot every recorded 27B-NVFP4 benchmark ran on, so +// those numbers are unaffected by this change. @ccdaab7e later re-quantized the +// head to FP8 with a PER-OUTPUT-CHANNEL scale, and nvidia/Qwen3.6-27B-NVFP4 ships +// a ModelOpt NVFP4 head; both hit the old unconditional BF16 assert. +// +// All three land on the SAME bf16 [in, out] Matmul-B operand the logits GEMM +// already consumes, so the forward is untouched and a BF16 head stays byte-exact +// (identical call, no dequant). Keeping the head quantized end-to-end would save +// ~2.3 GiB but needs an `lm_head_fp4`-style field on the dense weights plus a +// forward branch; that is a follow-up, not this fix. +// +// ModelOpt vs compressed-tensors global-scale convention: CT stores the value as +// a DIVISOR and `DequantCtNvfp4WeightToF32` reciprocates it internally, whereas +// ModelOpt's `weight_scale_2` IS the scale (qwen3_5_weights.cpp:272 assigns it to +// `scale2` directly). Passing `1/weight_scale_2` as the "disk divisor" makes the +// shared CT dequant compute the ModelOpt scale exactly. +} // namespace + +OwnedTensor LoadLmHeadAnyDtype(const TensorResolver& get, const TensorExists& has, + const std::string& name) { + const StTensor& w = get(name); + VT_CHECK(w.shape.size() == 2, "qwen3_5 dense: expected 2-D weight for " + name); + + if (w.dtype == "BF16") { + return LoadBf16Transposed(get, name); // unchanged byte-for-byte + } + + if (w.dtype == "F8_E4M3") { + const int64_t out_dim = w.shape[0]; + const int64_t in_dim = w.shape[1]; + // Per-output-channel [V,1] (unsloth @ccdaab7e) or a single per-tensor scalar. + // Stored BF16 there, F32 elsewhere; normalize both to f32 rows. + std::vector row_scale(static_cast(out_dim), 1.0F); + VT_CHECK(has(name + "_scale"), + "qwen3_5 dense: FP8 " + name + " requires " + name + "_scale"); + const StTensor& sc = get(name + "_scale"); + const int64_t n_scale = + static_cast(sc.nbytes) / (sc.dtype == "BF16" ? 2 : 4); + VT_CHECK(n_scale == out_dim || n_scale == 1, + "qwen3_5 dense: " + name + "_scale must be per-tensor or [out,1]"); + for (int64_t r = 0; r < out_dim; ++r) { + const int64_t i = (n_scale == 1) ? 0 : r; + if (sc.dtype == "BF16") { + uint16_t h = 0; + std::memcpy(&h, static_cast(sc.data) + i * 2, 2); + const uint32_t bits = static_cast(h) << 16; + std::memcpy(&row_scale[static_cast(r)], &bits, sizeof(float)); + } else { + std::memcpy(&row_scale[static_cast(r)], + static_cast(sc.data) + i * 4, sizeof(float)); + } + } + std::vector dq(static_cast(out_dim) * in_dim); + for (int64_t r = 0; r < out_dim; ++r) { + DequantFp8ToBf16(static_cast(w.data) + r * in_dim, + row_scale[static_cast(r)], in_dim, + dq.data() + static_cast(r) * in_dim); + } + MaybeReleaseSourcePages(w.data, w.nbytes); + OwnedTensor o = MakeOwned(vt::DType::kBF16, {in_dim, out_dim}); + dense_loaders::TransposeBf16(dq.data(), out_dim, in_dim, + reinterpret_cast(o.bytes.data())); + return o; + } + + if (w.dtype == "U8") { + const int64_t out_dim = w.shape[0]; + const int64_t in_dim = w.shape[1] * 2; + VT_CHECK(in_dim % 16 == 0, + "qwen3_5 dense: NVFP4 in_dim must be a multiple of 16 for " + name); + const StTensor& ws = get(name + "_scale"); + VT_CHECK(ws.dtype == "F8_E4M3", + "qwen3_5 dense: expected F8_E4M3 " + name + "_scale"); + // ModelOpt spells the global scale `weight_scale_2`; compressed-tensors + // spells it `weight_global_scale` and stores the reciprocal. + float disk_divisor = 0.0F; + if (has(name + "_scale_2")) { + const float ws2 = ReadF32Scalar(get(name + "_scale_2")); + VT_CHECK(ws2 != 0.0F, "qwen3_5 dense: zero " + name + "_scale_2"); + disk_divisor = 1.0F / ws2; // ModelOpt scale -> CT divisor convention + } else { + VT_CHECK(has(name + "_global_scale"), + "qwen3_5 dense: NVFP4 " + name + " requires " + name + + "_scale_2 (ModelOpt) or " + name + "_global_scale (CT)"); + disk_divisor = ReadF32Scalar(get(name + "_global_scale")); + VT_CHECK(disk_divisor != 0.0F, + "qwen3_5 dense: zero " + name + "_global_scale (divisor)"); + } + std::vector f32(static_cast(out_dim) * in_dim); + DequantCtNvfp4WeightToF32(static_cast(w.data), + static_cast(ws.data), disk_divisor, + out_dim, in_dim, f32.data()); + MaybeReleaseSourcePages(w.data, w.nbytes); + std::vector dq(static_cast(out_dim) * in_dim); + for (size_t i = 0; i < f32.size(); ++i) { + uint32_t bits = 0; + std::memcpy(&bits, &f32[i], sizeof(bits)); + // round-to-nearest-even f32 -> bf16, matching DequantFp8ToBf16. + const uint32_t lsb = (bits >> 16) & 1U; + bits += 0x7FFFU + lsb; + dq[i] = static_cast(bits >> 16); + } + OwnedTensor o = MakeOwned(vt::DType::kBF16, {in_dim, out_dim}); + dense_loaders::TransposeBf16(dq.data(), out_dim, in_dim, + reinterpret_cast(o.bytes.data())); + return o; + } + + VT_CHECK(false, "qwen3_5 dense: unsupported dtype '" + w.dtype + "' for " + + name + "; supported: BF16, F8_E4M3 (+_scale), " + "U8 NVFP4 (+_scale and _scale_2/_global_scale)"); + return OwnedTensor{}; +} + +namespace { + GdnLayerWeights LoadGdnDense(const TensorResolver& get, const TensorExists& has, const std::string& base) { const std::string la = base + "linear_attn."; @@ -395,7 +522,7 @@ Qwen3_5DenseWeights LoadQwen3_5Dense(const std::vector& shards, // The 27B owns an explicit head; smaller Qwen3.5 checkpoints tie logits to // the embedding table and omit lm_head.weight. if (has("lm_head.weight")) { - w.lm_head = LoadBf16Transposed(get, "lm_head.weight"); + w.lm_head = LoadLmHeadAnyDtype(get, has, "lm_head.weight"); } else { w.tied_lm_head = true; w.embed_tokens.nk = true; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 83f4809a2..acb7a4ddc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1337,6 +1337,9 @@ target_include_directories(test_mistral_paged_engine PRIVATE # 0 multi-valued cells). Checkpoint-gated, dgx-only. vllm_cpp_add_test(test_qwen3_32b_nvfp4a16_load vllm/models/test_qwen3_32b_nvfp4a16_load.cpp) + +vllm_cpp_add_test(test_qwen3_5_lm_head_dtypes + vllm/models/test_qwen3_5_lm_head_dtypes.cpp) vllm_cpp_add_test(test_qwen3_32b_nvfp4a16_paged_engine vllm/models/test_qwen3_32b_nvfp4a16_paged_engine.cpp) target_compile_definitions(test_qwen3_32b_nvfp4a16_paged_engine PRIVATE diff --git a/tests/vllm/models/test_qwen3_5_lm_head_dtypes.cpp b/tests/vllm/models/test_qwen3_5_lm_head_dtypes.cpp new file mode 100644 index 000000000..f0a0e8884 --- /dev/null +++ b/tests/vllm/models/test_qwen3_5_lm_head_dtypes.cpp @@ -0,0 +1,190 @@ +// Loader gate for issue #164: the Qwen3.6-27B NVFP4 publishers do not agree on how +// the OUTPUT HEAD is stored, and revisions of a SINGLE repo disagree with each +// other. Measured on the two snapshots we hold (headers read directly): +// +// unsloth/Qwen3.6-27B-NVFP4 @890bdef7 lm_head.weight BF16 [248320, 5120] +// unsloth/Qwen3.6-27B-NVFP4 @ccdaab7e lm_head.weight F8_E4M3 [248320, 5120] +// lm_head.weight_scale BF16 [248320, 1] +// +// @890bdef7 is the snapshot every recorded 27B-NVFP4 benchmark ran on, which is +// why the BF16-only assert survived: the head was never quantized under us until +// the repo was re-quantized. nvidia/Qwen3.6-27B-NVFP4 ships a third form, a +// ModelOpt NVFP4 head (`weight` U8 + `weight_scale` F8 + `weight_scale_2` f32). +// +// These cases pin the DISPATCH, not the kernels: the dequant math is already +// covered by the nvfp4-emulation and fp8 loader tests. Synthetic tensors only — +// no checkpoint, no GPU, so this runs everywhere. +#include + +#include +#include +#include +#include +#include +#include + +#include "vllm/model_executor/model_loader/safetensors_reader.h" +#include "vllm/model_executor/models/qwen3_5_dense.h" + +using vllm::LoadLmHeadAnyDtype; +using vllm::OwnedTensor; +using vllm::StTensor; + +namespace { + +// Minimal in-memory stand-in for one resolved safetensors entry. The loader only +// reads dtype/shape/data/nbytes, so a backing vector is enough. +struct Fake { + std::string dtype; + std::vector shape; + std::vector bytes; +}; + +class Bag { + public: + void Put(const std::string& name, Fake f) { items_[name] = std::move(f); } + + // One STABLE StTensor per name. A single shared view would be a fixture bug: + // the loader holds `const StTensor& w = get(name)` across the later + // get(name + "_scale") call, exactly as a real safetensors resolver allows, + // so resolving the scale must not disturb the weight it already bound. + vllm::TensorResolver Resolver() { + return [this](const std::string& name) -> const StTensor& { + auto it = items_.find(name); + REQUIRE_MESSAGE(it != items_.end(), "missing tensor: " << name); + Fake& f = it->second; + StTensor& v = views_[name]; + v.dtype = f.dtype; + v.shape = f.shape; + v.data = f.bytes.data(); + v.nbytes = f.bytes.size(); + return v; + }; + } + + std::function Has() { + return [this](const std::string& n) { return items_.count(n) != 0; }; + } + + private: + std::unordered_map items_; + // std::unordered_map never invalidates references to existing elements on + // insert, so every returned reference stays valid for the whole load. + std::unordered_map views_; +}; + +uint16_t F32ToBf16(float v) { + uint32_t bits = 0; + std::memcpy(&bits, &v, sizeof(bits)); + const uint32_t lsb = (bits >> 16) & 1U; + bits += 0x7FFFU + lsb; + return static_cast(bits >> 16); +} + +float Bf16ToF32(uint16_t h) { + const uint32_t bits = static_cast(h) << 16; + float v = 0.0F; + std::memcpy(&v, &bits, sizeof(v)); + return v; +} + +Fake MakeBf16(const std::vector& shape, const std::vector& vals) { + Fake f{"BF16", shape, {}}; + f.bytes.resize(vals.size() * 2); + for (size_t i = 0; i < vals.size(); ++i) { + const uint16_t h = F32ToBf16(vals[i]); + std::memcpy(f.bytes.data() + i * 2, &h, 2); + } + return f; +} + +// e4m3 encode for the small exact powers of two this test uses (no rounding). +uint8_t EncodeE4M3(float v) { + if (v == 0.0F) return 0; + const uint8_t sign = v < 0 ? 0x80 : 0x00; + float a = v < 0 ? -v : v; + int exp = 0; + while (a >= 2.0F) { a /= 2.0F; ++exp; } + while (a < 1.0F) { a *= 2.0F; --exp; } + const uint8_t biased = static_cast(exp + 7); + const uint8_t mant = static_cast((a - 1.0F) * 8.0F + 0.5F); + return static_cast(sign | (biased << 3) | (mant & 0x7U)); +} + +} // namespace + +TEST_CASE("qwen3_5 lm_head: BF16 head is unchanged (the benchmarked @890bdef7 form)") { + Bag bag; + // [out=2, in=4] row-major, transposed to [in=4, out=2]. + bag.Put("lm_head.weight", + MakeBf16({2, 4}, {1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F, 7.0F, 8.0F})); + const OwnedTensor o = LoadLmHeadAnyDtype(bag.Resolver(), bag.Has(), "lm_head.weight"); + + REQUIRE(o.rank == 2); + CHECK(o.shape[0] == 4); // in + CHECK(o.shape[1] == 2); // out + const auto* d = reinterpret_cast(o.bytes.data()); + CHECK(Bf16ToF32(d[0]) == doctest::Approx(1.0F)); // [in0,out0] + CHECK(Bf16ToF32(d[1]) == doctest::Approx(5.0F)); // [in0,out1] + CHECK(Bf16ToF32(d[6]) == doctest::Approx(4.0F)); // [in3,out0] + CHECK(Bf16ToF32(d[7]) == doctest::Approx(8.0F)); // [in3,out1] +} + +TEST_CASE("qwen3_5 lm_head: FP8 with a PER-OUTPUT-CHANNEL scale (@ccdaab7e form)") { + Bag bag; + // weight rows are e4m3 1.0/2.0; the per-row scale differs, which is exactly + // what a per-tensor reader would get wrong. + Fake w{"F8_E4M3", {2, 4}, {}}; + for (int r = 0; r < 2; ++r) { + for (int c = 0; c < 4; ++c) { + w.bytes.push_back(EncodeE4M3(r == 0 ? 1.0F : 2.0F)); + } + } + bag.Put("lm_head.weight", std::move(w)); + bag.Put("lm_head.weight_scale", MakeBf16({2, 1}, {0.5F, 4.0F})); + + const OwnedTensor o = LoadLmHeadAnyDtype(bag.Resolver(), bag.Has(), "lm_head.weight"); + REQUIRE(o.rank == 2); + CHECK(o.shape[0] == 4); + CHECK(o.shape[1] == 2); + const auto* d = reinterpret_cast(o.bytes.data()); + // row0 = 1.0 * 0.5, row1 = 2.0 * 4.0 -> distinct per-channel results. + CHECK(Bf16ToF32(d[0]) == doctest::Approx(0.5F)); + CHECK(Bf16ToF32(d[1]) == doctest::Approx(8.0F)); + CHECK(Bf16ToF32(d[6]) == doctest::Approx(0.5F)); + CHECK(Bf16ToF32(d[7]) == doctest::Approx(8.0F)); +} + +TEST_CASE("qwen3_5 lm_head: FP8 with a single per-tensor scale") { + Bag bag; + Fake w{"F8_E4M3", {2, 4}, {}}; + for (int i = 0; i < 8; ++i) w.bytes.push_back(EncodeE4M3(2.0F)); + bag.Put("lm_head.weight", std::move(w)); + Fake sc{"F32", {1}, std::vector(4)}; + const float s = 3.0F; + std::memcpy(sc.bytes.data(), &s, 4); + bag.Put("lm_head.weight_scale", std::move(sc)); + + const OwnedTensor o = LoadLmHeadAnyDtype(bag.Resolver(), bag.Has(), "lm_head.weight"); + const auto* d = reinterpret_cast(o.bytes.data()); + for (int i = 0; i < 8; ++i) CHECK(Bf16ToF32(d[i]) == doctest::Approx(6.0F)); +} + +TEST_CASE("qwen3_5 lm_head: an unsupported dtype names itself instead of asserting BF16") { + Bag bag; + Fake w{"I32", {2, 4}, std::vector(32)}; + bag.Put("lm_head.weight", std::move(w)); + // The old code raised "expected BF16 for lm_head.weight" for EVERY quantized + // head; the message must now name what was actually seen and what is accepted. + CHECK_THROWS_WITH_AS( + LoadLmHeadAnyDtype(bag.Resolver(), bag.Has(), "lm_head.weight"), + doctest::Contains("unsupported dtype 'I32'"), std::runtime_error); +} + +TEST_CASE("qwen3_5 lm_head: FP8 without a scale fails loudly, not silently") { + Bag bag; + Fake w{"F8_E4M3", {2, 4}, std::vector(8, EncodeE4M3(1.0F))}; + bag.Put("lm_head.weight", std::move(w)); + CHECK_THROWS_AS(LoadLmHeadAnyDtype(bag.Resolver(), bag.Has(), "lm_head.weight"), + std::runtime_error); +}