diff --git a/.agents/NOW.md b/.agents/NOW.md index 11213aa93..464e878d4 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -1,6 +1,6 @@ # NOW — the one-Read resume surface - + Read this FIRST, every session. A SNAPSHOT, rewritten in place: what is live, the gate being chased, what to do next. Never a log — evidence lives in the @@ -14,14 +14,15 @@ checkpoint on `upstream/main` at `59674cf1d`. | Claim / track | State | Next command or step | |---|---|---| -| Laguna NVFP4 decode speed | **Closed: PARITY+ 1.03x** (44.46 vs 43.10, byte-exact, default). Root cause = bf16 weight residency via `VT_LAGUNA_RESIDENT_BF16W` (default-ON). Detail in benchmark record | Residual: formal vLLM K-run set when convenient | -| DeepSeek-V4-Flash decode | **Closed: BEATS ds4 1.144x** (`VT_V4_RESIDENT_W` on, byte-exact). Phase-2 routed-expert residency NEGATIVE 2026-08-05 (−3.4%), HELD default-OFF. See state | — | +| Laguna NVFP4 decode speed | **Closed: PARITY+ 1.03x** (byte-exact, default; `VT_LAGUNA_RESIDENT_BF16W` bf16-residency). Benchmark record | vLLM K-run set when convenient | +| DeepSeek-V4-Flash decode | **Closed: BEATS ds4 1.144x** (`VT_V4_RESIDENT_W`, byte-exact). Phase-2 routed-expert residency NEGATIVE (−3.4%), default-OFF | — | | f32-out GEMV audit | Only laguna + deepseek_v4 bf16 tower affected; gate models & on-framework dense unaffected (bf16-out, e2e-verified) | Re-verify deepseek_v4 bf16 tower same-tool | | Invocation-parity prevention | CI guard (`check-gemv-invocation-consistency.py`) + AGENTS.md checklist landing | Review + merge; CUDA build-verify `kGemvHeuristicAlgos` on dgx | -| MiniMax-H3 lane | Portable path complete; e2e prompt-conditioned video on real weights (Thor). Speed = NVFP4 FP4 device path, sm_121-gated | PR #26 rebase + supports-audit synthesis (workflow ran; integrate) | -| Kimi-Linear-48B (KDA+NoPE-MLA+MoE) | **Full-model GB10 e2e RUNS** (bf16-resident §13, f32-loader block CLEARED): CPU+CUDA 13/13·656; host RSS peak 1.7 GiB, min-avail 21 GiB, no OOM. **Token gate NEAR-TIE 106/128** (6/8 prompts token-exact; numerics near-tie vs deterministic oracle, not a bug) | STRICT path = device GDN/MLA islands + bf16 stream (W7-speed residuals); 1.59 tok/s; default OFF | +| MiniMax-H3 lane | Portable path complete; e2e prompt-conditioned video on real weights (Thor). Speed = NVFP4 FP4 device path, sm_121-gated | PR #26 rebase + supports-audit synthesis | +| Kimi-Linear-48B (KDA+NoPE-MLA+MoE) | **Full-model GB10 e2e RUNS** (bf16-resident §13): CPU+CUDA 13/13·656, no OOM. **Token gate NEAR-TIE 106/128** (6/8 token-exact) | device GDN/MLA islands + bf16 stream; 1.59 tok/s; default OFF | | 35B fresh grid | **BOUND** @`1ea26427`: tput 0.93-1.03x, c16 0.93x. INTAKE + Option A both **RESOLVED NEGATIVE** (H2D-out-of-capture tput WASH) | Real lever left: prefill glue (task #61) | | Qwen3.5-4B revalidation | 0.9971x @`59674cf1` (#35); TTFT/PSS pass, TPOT/ITL open | `docs/bench-evidence/` | +| MXFP4 c2-c8 lever = FA2 decode GQA group-swap | **Ported gated OFF** (`VT_FA2_DECODE_GQA_SWAP`): vLLM decode swap in the d128 varlen launcher; correctness-complete GB10 (op RED/GREEN, memcheck, swap-ON smoke token-exact), OFF byte-identical | bench + flip: state `KERNEL-FA2-GQA-SWAP` | | ROW-SERVE-ASYNC-DENSE-MIRROR | **LANDED+dgx-VERIFIED** (`f9c969ae`): #31 async mirror on classic dense Qwen3; gate RED→GREEN, SACRED 184/184 | Residual: sibling scope one-liner | | QUANT-CT-MXFP4-BENCH | ~0.91x@c2-c8; marlin+glue levers **both REFUTED**; step = flash GQA group-swap | LEVER + detail: state `QUANT-CT-MXFP4-GLUE` (`cuda_flash_attn_fa2.cu:1075`) | diff --git a/.agents/benchmark-record.md b/.agents/benchmark-record.md index 396f2ab26..ae86c8e85 100644 --- a/.agents/benchmark-record.md +++ b/.agents/benchmark-record.md @@ -12376,3 +12376,40 @@ only "27"/"35"; no Yi30/8B key. Retargeting = MODEL_REVISIONS/REPOSITORIES entry max-num-seqs sizing. The fix unblocks the DEFAULT-config bench (no more VT_ASYNC_SCHED=0 workaround); oracle proven to run the model today. Evidence: dgx:/dev/shm/serve-async-dense/ {gates_06b.log,gates2.log,mxfp4_e2e.log,oracle_neartie2.log,oracle_tf.log}. + +## 2026-08-08T18:00 — KERNEL-FA2-GQA-SWAP: d128 varlen decode group-swap gated-OFF, correctness gates GREEN (bench PENDING) + +The #47-localized lever (`row/KERNEL-FA2-GQA-SWAP`, `VT_FA2_DECODE_GQA_SWAP`, default +OFF): vLLM's FA2 `seqlenq_ngroups_swapped` decode grid ported into +`LaunchDecodeVarlenFA2Bf16` so the Qwen3-dense d128 decode launches +`(batch, kv_heads)` not `(batch, hq)`. `benchmark_binding=false` this session — the +c1-c8 x3 binding grid + default flip is the recorded next step; what ran here is the +CORRECTNESS bar that GATES the flip. dgx GB10, CUDA 13.0, sm_121a, RelWithDebInfo, +VLLM_CPP_FLASH_ATTN=ON; the 4 changed files git-archived by SHA onto `~/mxfp4-bench/src` +(md5-matched local), incremental `ninja` (both CUDA TUs recompiled clean on nvcc). + +GATES: +- OP UNIT TEST (RED-first): `test_ops_paged_attn --test-case="*varlen d128*"` = 5/5 + cases, **280/280 assertions GREEN** (OFF-path parity + group-swap-matches-ref both + ratios 16/8+32/8 × batch{1,2,4,8} × len{5,21,1024}, `swap_launches==1` asserted; + swap-vs-plain near-tie max_abs<2e-2; MHA qpk==1 inert `swap_launches==0`). Full + binary **28/28 cases, 454,679 assertions** — no regression. +- RED PROVEN: injecting a wrong swapped `o_head_stride` (drop the ngroups factor) → + group-swap case FAILS (26,528 violations, max_abs 2.60). Restored + rebuilt + (md5 `ba34d5b8…`). +- MEMCHECK: `compute-sanitizer --tool memcheck --leak-check full` on the swap cases = + **0 errors, 0 bytes leaked**. +- #44 MXFP4 SMOKE (Yi30/Qwen3-8B-MXFP4, groups=4, DEFAULT async/graphed): swap-OFF PASS + (3/3 det token-exact + coherent); **swap-ON PASS (3/3 deterministic TOKEN-EXACT vs + golden AND byte-identical to swap-OFF** — capitals/arithmetic/fibonacci char-identical, + story identical). No token flip e2e → near-tie razor not needed; graphed capture-safety + token gate ON. + +NEXT (the flip campaign): `scripts/mxfp4-online-serving-grid.sh --snapshot +--build-dir ~/mxfp4-bench/build --configure-log ` with `VT_FA2_DECODE_GQA_SWAP=1` +on OURS legs, vs the #45 numbers (tput c1 0.989 / c2 0.911 / c4 0.919 / c8 0.913; TPOT +c2 28.27→25.45). #47 projects flash alone closes ~28%@c2 / ~55%@c8 (decode flash +63.7→~41.7us c2 as the grid drops 192→96 CTAs); residual = marlin +7-9% + ~0.7ms host, +so a single lever may not reach ≥1.0x — record the honest per-axis outcome. Then Qwen3 +0.6B/4B e2e SACRED swap-ON, then flip per parity-enablers. Box left clean (both locks +free, GPU idle, worker down, disk 22G). diff --git a/.agents/kernel-matrix.md b/.agents/kernel-matrix.md index 3d70bbc5a..03d41037e 100644 --- a/.agents/kernel-matrix.md +++ b/.agents/kernel-matrix.md @@ -85,6 +85,25 @@ trace-proven at **180.28 us/call**, within 1.1% of the matched vLLM FA2 kernel at 178.40 us/call. Evidence: [2026-07-25 4B repair](../docs/bench-evidence/qwen35-4b-main-repair-20260725.md). +`KERNEL-ATTN-FA2` d128 varlen decode GQA group-swap (2026-08-06, +`row/KERNEL-FA2-GQA-SWAP`, `VT_FA2_DECODE_GQA_SWAP`, default OFF): vLLM's +`seqlenq_ngroups_swapped` decode grid ported into `LaunchDecodeVarlenFA2Bf16` +(`cuda_flash_attn_fa2.cu`) so the Qwen3-dense d128 decode launches +(batch, kv_heads) not (batch, hq) — the ngroups query heads pack into seqlen_q, +KV read once per group, heuristic sees `batch*kv_heads` — halving the CTA count +at batch>=2 (#47: ours over-waved 192 CTAs c2 / ~5 waves c8). Presented WITHOUT a +materialized transpose via kv-major-group-minor strides, mirroring the shipped +d256 `LaunchDecodeFA2Bf16` swap (the vendored `get_lse_tile`/combine already +honor the flag). Gated so OFF is byte-identical to the shipped plain-varlen +reduction; ON is non-byte-exact only when num_splits>1 (split reduction order +→ near-tie, toward vLLM). Correctness-complete on GB10 (CUDA 13.0, sm_121a): +op RED-first test **280/280 assn GREEN** (both ratios 16/8+32/8, batch 1/2/4/8, +short+long context; `swap_launches==1` proves the grid engaged; wrong-stride RED += 26528 violations), compute-sanitizer **0-err/0-leak**, and the #44 MXFP4 e2e +smoke swap-ON **3/3 deterministic TOKEN-EXACT + coherent**, byte-identical to +swap-OFF (no token flip, near-tie razor unneeded). c1-c8 x3 binding re-bench + +default flip is the recorded next step. Detail: state `KERNEL-FA2-GQA-SWAP`. + | ID | Item | Upstream | Our code | Tests/evidence | Spike/spec | State | Owner | |---|---|---|---|---|---|---|---| | `KERNEL-ACCEL-PROVIDER-SELECT` | **WHICH implementation of an op runs, when more than one exists on a device** — the selection layer above every kernel family in this matrix. Distinct from `KERNEL-CUDA-DISPATCH-AOT`, which is about which ARCH a CUDA kernel is compiled/selected for; this is about which PROVIDER (ours, a vendor library, MLX, llama.cpp) serves the op at all | no single upstream file: this is the shape vLLM's runtime chain uses everywhere — flashinfer tactic registries, cuBLASLt/CUTLASS per-call heuristics, and torch's backend selection — rather than compile-time pinning | `vt::OpProvider` [op_provider.h](../include/vt/op_provider.h) + [op_provider.cpp](../src/vt/op_provider.cpp); the flat `[OpId][DeviceType]` `void*` table it replaces is gone from [ops.cpp](../src/vt/ops.cpp) with the ~70 op wrappers untouched. Providers registered today: `vt-native` (every backend kernel in the tree, priority 0, unconditional — behaviour preserved exactly), `mlx` (priority 100, Metal `kMatmul`/`kMatmulBT`, build-gated `VLLM_CPP_MLX`, [metal_mlx_provider.mm](../src/vt/metal/metal_mlx_provider.mm)), and — NEW 2026-07-23 (`CLAIM-BACKEND-SEAM-S5-1`, work row `S5`) — **`vt-cpu-ref` (priority −1000, the portable reference tier)**: the CPU kernel installed LAZILY as a negative-priority fallback on a UNIFIED-MEMORY device's first `GetOp` miss, mirroring `custom_op.py:138 forward_native`, so a partial backend runs an op it lacks natively instead of throwing. Native always wins (priority); gated on `Backend::UnifiedMemory()` (a discrete GPU never gets it — a CPU kernel on true device memory is corruption); observable via `GetReferenceTierHits()` + a one-time loud stderr line | [test_op_provider.cpp](../tests/vt/test_op_provider.cpp) 11 cases / 47 assertions — deterministic selection under REVERSED registration order, name tie-break, duplicate rejection, capability predicate, caps re-resolution, decline-and-fall-back, stats, runtime disable; [test_metal_backend.cpp](../tests/vt/test_metal_backend.cpp) 9 cases / 108 assertions on the M4 with MLX ON, including MLX-vs-MSL-vs-CPU NMSE per op at real shapes and an end-to-end DECLINE; **[test_reference_tier.cpp](../tests/vt/test_reference_tier.cpp) (S5): discrete-device refusal + unified-device zero-native-kernel fallback correctness + native-wins + observability, hardware-free via a fake backend on `kXPU`.** Linux CPU 156/156; dgx regression set ALL UNCHANGED — anchor `tests/vt/test_op_provider.cpp:64` | [Metal/MLX reuse study §6](specs/metal-mlx-reuse-study.md); [accelerator-seam-audit §10](specs/accelerator-seam-audit.md); [drop-in kernel ABI](specs/dropin-kernel-abi.md) (the complementary ARGUMENT half) | `ACTIVE` — mechanism landed and gated with THREE provider kinds (`vt-native`, `mlx`, `vt-cpu-ref`); the CUDA/CPU/Vulkan vendor provider rows it was designed for are not yet populated (so the row is deliberately left open) | `CLAIM-BACKEND-ACCEL-PROVIDER-1` | diff --git a/.agents/parity-ledger.md b/.agents/parity-ledger.md index 606340574..19a7f116c 100644 --- a/.agents/parity-ledger.md +++ b/.agents/parity-ledger.md @@ -887,3 +887,4 @@ Columns: | 2026-07-31 (`CLAIM-CPU-GCC12-WERROR-PORTABILITY`; maintenance, rows `QUANT-GGUF-KEEPQ-LOADER` + `KV-OFFLOAD`; lifecycle unchanged) | Removes two GCC 12 production-library `-Werror` blockers without suppressions: the GGUF prefault keeps the same one-byte-per-page volatile XOR but uses simple assignment, and the KV filesystem tier builds the identical `...tmp` suffix with append operations inside its thread-local initializer. No API, algorithm, default, CUDA, fixture, or golden change. | Behavior remains grounded in the accepted loader and KV-persistence leaf specs: llama.cpp mmap prefault intent and vLLM `tiering/fs/io.py` unique temporary-file publication. This is compiler portability, not a parity-surface change. | RED: GCC 12 failed first at `qwen3_5_gguf_weights.cpp:49` (`-Wvolatile`), then at `fs_io.cpp:66` (`-Wrestrict`). GREEN: production `vllm` and focused test targets build clean; focused CTest 2/2 (`test_gguf_keep_quant`, `test_kv_offload_fs`). Full all-target build is PARTIAL at 42% on unrelated test-only GCC 12 `-Wrestrict` diagnostics in `test_deepseek_v2_paged_engine.cpp` and `test_glm4_moe_lite_paged_engine.cpp`; no full-CTest claim. Benchmark NOT APPLICABLE. | | 2026-08-01 (`SERVE-CLI-CHAT` W0 contract spike; `CLAIM-SERVE-CLI-CHAT-SPIKE`; CPU-only records/spec) | Accepts `.agents/specs/cli-chat-complete.md`, corrects the inventory from “no direct commands” to the actual pinned `chat`/`complete` surface, and decomposes a dual-mode port: exact remote OpenAI HTTP/SSE commands plus preservation of the existing in-process invocation as a compatibility alias. No production, test, CMake, model, kernel, fixture, or generated file changes. | Pinned vLLM `5559679229`: command registration `vllm/entrypoints/cli/main.py:17-37,73-98`; model/auth resolution and stream shaping `vllm/entrypoints/cli/openai.py:30-100`; chat `:155-234`; complete `:237-312`. The local compatibility baseline is `examples/cli/main.cpp:1-207`. | CPU record/doc gates only; benchmark `NOT APPLICABLE`, `benchmark_binding=false`. Implementation remains absent and the row moves `INVENTORIED` -> `SPIKE`. W1-W5 name parse, transport, complete, chat, and packaging gates, including fake-server request/SSE transcript parity, Release `-Werror`, ASan+UBSan, and TSan. | | 2026-08-04 (`HARDEN-DETECTOR-LANES` PR #28 CI repair; `CLAIM-HARDEN-SANITIZER-DISK`; Ordino task `t-e19dc73f`; CPU-only, lifecycle unchanged; closing commit: this checkpoint) | Repairs the hosted ASan+UBSan build's filesystem exhaustion without weakening detector coverage. Sanitizer tests share one internal fully instrumented engine image instead of force-linking another static copy into every executable, and `-g1` retains file/line traces without full type/local-variable DWARF. CI enables the existing `VT_POOL_BYPASS=1` exact-allocation/real-free detector mode. The newly reachable leak survey removes a real minja `MacroNode` ownership cycle by weakly capturing the context that owns the callable. It also closes the two Nix-only suite portability gaps: resolve `true` and the active Python executable instead of assuming `/usr/bin`, and remove inherited `PYTHONHASHSEED` only from the unconfigured control suite. Default build linkage and packaged C ABI exports remain unchanged. | No vLLM behavioral counterpart: this is local build/test infrastructure plus a vendored minja lifetime repair. Anchors: `CMakeLists.txt`, `tests/CMakeLists.txt`, `.github/workflows/ci.yml`, `third_party/minja/minja.hpp`, `tests/tools/test_gdn_packed_component.py`, and `tests/tools/test_online_gate_client.py`. Remote root-cause evidence: GitHub run `30819266647`, job `91704728276`, 99 MiB free then `ld: No space left on device`. | **PASS.** GCC 15.2.0 ASan+UBSan full suite **331/331** under leak detection and pool bypass; TSan full suite **331/331** under pool bypass; affected plain GCC 15 `-Werror` suites **3/3**. ASan+UBSan tree **93 GiB -> 5.6 GiB** (about 94% smaller); TSan tree **1.9 GiB**. `benchmark_binding=false`, performance **NOT APPLICABLE**. Hosted PR confirmation is the next external gate; `continue-on-error` stays until that confirmation. | +| 2026-08-08 (`row/KERNEL-FA2-GQA-SWAP`; `CLAIM-KERNEL-FA2-GQA-SWAP`; kernel `KERNEL-ATTN-FA2`; gated default-OFF, lifecycle unchanged) | Ports vLLM's FA2 `seqlenq_ngroups_swapped` decode optimization into the d128 varlen decode launcher (`LaunchDecodeVarlenFA2Bf16`, gate `VT_FA2_DECODE_GQA_SWAP`): the Qwen3-dense decode grid becomes `(batch, kv_heads)` not `(batch, hq)` — the ngroups query heads pack into seqlen_q, KV read once/group, presented WITHOUT a materialized transpose via kv-major-group-minor strides (a 1:1 mirror of the already-shipped d256 `LaunchDecodeFA2Bf16` swap). OFF path byte-identical to the prior plain-varlen reduction; ON is non-byte-exact only when num_splits>1 (split reduction order → near-tie, toward vLLM's own numerics). | Mirrors `flash-attention @ 2c839c33` `mha_fwd_kvcache` seqlenq_ngroups_swapped + `set_params_splitkv` and vLLM v0.25.0 `flash_attn.py flash_attn_varlen_func` decode (#47 measured vLLM's swapped grid `(1,6,16)` = batch×kv_heads vs ours `(1,3,64)` = batch×query_heads). The vendored `flash_fwd_kernel.h` `get_lse_tile`/combine already honor the flag in both the num_splits==1 direct-write and >1 combine paths (the d256 arm is the proof). | GB10 sm_121a CUDA 13.0: op RED-first test 280/280 (both GQA ratios × batch{1,2,4,8} × short+long ctx; `swap_launches==1` proves the grid engaged; swap-vs-plain near-tie; MHA-inert) — RED proven (wrong swapped stride → 26,528 violations); full binary 28/28·454,679 no regression; compute-sanitizer 0-err/0-leak; #44 MXFP4 e2e smoke swap-ON 3/3 deterministic TOKEN-EXACT + coherent, byte-identical to swap-OFF. `benchmark_binding=false` (c1-c8 x3 re-bench + default flip = recorded next step; #47 projects flash ~28%@c2 / ~55%@c8 of the gap). | diff --git a/.agents/quantization-matrix.md b/.agents/quantization-matrix.md index fd7d23b11..753289b32 100644 --- a/.agents/quantization-matrix.md +++ b/.agents/quantization-matrix.md @@ -127,7 +127,7 @@ Registry source: | `QUANT-FP8-PCPT` | ModelOpt FP8 per-channel/per-token | W8/A8 | capability selected | - | - | - | - | - | `INVENTORIED` | - | leaf spec open | - | | `QUANT-MXFP8-MODELOPT` | ModelOpt MXFP8 | W8/A8 | CUDA/ROCm/XPU dispatch | - | - | - | - | - | `INVENTORIED` | - | leaf spec open | - | | `QUANT-MIXED-MODELOPT` | ModelOpt mixed precision | FP8/NVFP4/MXFP8 groups | per-layer | part | part | part | Y | Y | `PARTIAL` | 35B FP8+NVFP4 slice: [loader](../src/vllm/model_executor/models/qwen3_5_weights.cpp#L118), [FP8 tests](../tests/vt/test_ops_fp8_cutlass.cpp#L188), [NVFP4 tests](../tests/vt/test_ops_moe_grouped.cpp#L453), [gate](../tests/parity/test_qwen36_paged_engine.cpp#L78) | leaf spec open | - | -| `QUANT-CT-MXFP4` | compressed-tensors MXFP4 `mxfp4-pack-quantized` (group 32, E8M0 block scales, NO global) | W4/A16 native **Marlin mxf4 keep-quant** landed (GB10 target; W4A4 cute-dsl crashes sm_121) | CUDA GB10 Marlin W4A16 (E8M0, group_blocks=2); CPU dequant fallback | Y | Y | Y | Y | - | `ANCHOR-BACKFILL` | **W2 native compute + W3 gates + e2e (row/QUANT-CT-MXFP4, `1c5ee09e`):** W0 vehicle `Yi30/Qwen3-8B-MXFP4` runs on the 0.25.0 oracle; W1 traced FlashInfer-W4A4-selected-but-crashes-on-sm_121 -> **Marlin W4A16 is the GB10 target**. Native path: `generate_kernels.py` MXFP4 config (`kFE8M0fnu`, group_blocks 2) + regenerated instances; [`MarlinProcessExpertScalesMxfp4`](../src/vt/cuda/cuda_marlin_repack.cu) (byte-exact vs vLLM at all shapes); `MoeMarlinArgs.{group_size,mxfp4}` launcher branch; `Nvfp4Weight.{group_size,is_mxfp4}` + [`dense_nvfp4_gemm.h`](../include/vllm/model_executor/models/dense_nvfp4_gemm.h) branch + `MatmulMxfp4W4A16D`; [`dense_weight_loaders.h`](../include/vllm/model_executor/models/dense_weight_loaders.h) MXFP4 loaders; `qwen3_weights.cpp` detect+load. **Gates GREEN:** op-level GEMM vs independent CPU dequant 0.36% M=1/M=8 all real shapes ([`test_ops_moe_grouped.cpp`](../tests/vt/test_ops_moe_grouped.cpp)); model-facing `MakeLinearMethod->Apply->BuildMarlinDenseResident` bad=0 K=4096+12288 ([`test_linear_method.cpp`](../tests/vllm/model_executor/layers/test_linear_method.cpp)); **e2e 3/4 token-exact vs oracle golden (async-off)** ([evidence](../docs/bench-evidence/mxfp4-qwen/W3-e2e-result.md)). **Default-async degeneration RESOLVED (`ROW-SERVE-ASYNC-DENSE-MIRROR`, `f9c969ae`):** the pre-existing classic-dense-Qwen3 async bug (device-mirror not wired for `qwen3.cpp`) is fixed; DEFAULT-config (async ON) e2e on dgx is now coherent + 3/4 token-exact vs the golden (p0/p1/p3 exact; p2 story = oracle-ratified near-tie, teacher-forced max gap 0.0000 nats). p3 formal near-tie gate now RATIFIED. W4 THROUGHPUT bench RAN on GB10 (`row/QUANT-CT-MXFP4-BENCH` `33e93608`, c1/c2/c4/c8x3, production graphed vLLM 0.25.0, oracle `VLLM_DISABLED_KERNELS=FlashInferMxFp4LinearKernel` -> Marlin W4A16): **BELOW-FLOOR** (gate NO). Total tok/s ours/vLLM 0.99x@c1 but ~0.91x@c2-c8 (c2 0.911, c4 0.919, c8 0.913); driven by TPOT +10-12%@c2-c8 (parity@c1); TTFT at parity; peak GPU mem **2.6x LESS** (28.3 vs 73.7 GiB). ATTRIBUTION (grounded, per-concurrency curve): the gap is a BATCHED-decode cost absent at batch-1 -> the grouped Marlin W4A16 decode GEMM at M=2..8, NOT batch-1 GEMV nor prefill; NOT a ceiling. NEXT: same-tool nsys decode-window on c8 both engines. Harness (all additive; 27/35 byte-unchanged; CPU tests 45/45): `online_gate` `q3mxfp4` key + `POINTS_BY_MODEL`/`points_for`, [`mxfp4_smoke_gate.py`](../tools/bench/mxfp4_smoke_gate.py) (#44 battery), `dgx-online-serving.sh` q3mxfp4 branches, [`mxfp4-online-serving-grid.sh`](../scripts/mxfp4-online-serving-grid.sh). Full table + build contract (RelWithDebInfo + oracle flashinfer-cutlass + DISK build) in benchmark-record. Earlier CPU weight unpack + E8M0 dequant: NEW [mxfp4_dequant.h](../include/vllm/model_executor/model_loader/mxfp4_dequant.h) + [.cpp](../src/vllm/model_executor/model_loader/mxfp4_dequant.cpp#L14) (`E8M0ToF32` = `2^(byte-127)`, `DequantMxfp4ToBf16`/`ToF32`, group 32, no global; reuses `kE2M1Lut`). Unit gate [test_mxfp4_dequant.cpp](../tests/vllm/test_mxfp4_dequant.cpp#L34) — E8M0 known-byte decode, hand-computed 32-group dequant (bf16+f32), the E8M0-vs-fp8 + group-32-vs-16 RED traps, multi-row/group offsets, randomized rel-error vs a double-precision port of `dq_mxfp4_torch` with bf16==f32 exactness. CPU `-Werror` 0-warn. Ports FROM `compressed_tensors_w4a4_mxfp4.py:20-97` + `mxfp8_utils.py:61-65,222` + golden `tests/quantization/reference_mxfp4.py:28-117`. C/E/P PENDING: GPU W4A4 fp4 GEMM + Marlin W4A16 fallback + MoE expert path + e2e are NAMED later bricks; DeepSeek-V4 + Kimi-K3 loaders consume this once wired | [MXFP4 spike](specs/mxfp4-compressed-tensors.md) | `CLAIM-QUANT-MXFP4` | +| `QUANT-CT-MXFP4` | compressed-tensors MXFP4 `mxfp4-pack-quantized` (group 32, E8M0 block scales, NO global) | W4/A16 native **Marlin mxf4 keep-quant** landed (GB10 target; W4A4 cute-dsl crashes sm_121) | CUDA GB10 Marlin W4A16 (E8M0, group_blocks=2); CPU dequant fallback | Y | Y | Y | Y | - | `ANCHOR-BACKFILL` | **W2 native compute + W3 gates + e2e (row/QUANT-CT-MXFP4, `1c5ee09e`):** W0 vehicle `Yi30/Qwen3-8B-MXFP4` runs on the 0.25.0 oracle; W1 traced FlashInfer-W4A4-selected-but-crashes-on-sm_121 -> **Marlin W4A16 is the GB10 target**. Native path: `generate_kernels.py` MXFP4 config (`kFE8M0fnu`, group_blocks 2) + regenerated instances; [`MarlinProcessExpertScalesMxfp4`](../src/vt/cuda/cuda_marlin_repack.cu) (byte-exact vs vLLM at all shapes); `MoeMarlinArgs.{group_size,mxfp4}` launcher branch; `Nvfp4Weight.{group_size,is_mxfp4}` + [`dense_nvfp4_gemm.h`](../include/vllm/model_executor/models/dense_nvfp4_gemm.h) branch + `MatmulMxfp4W4A16D`; [`dense_weight_loaders.h`](../include/vllm/model_executor/models/dense_weight_loaders.h) MXFP4 loaders; `qwen3_weights.cpp` detect+load. **Gates GREEN:** op-level GEMM vs independent CPU dequant 0.36% M=1/M=8 all real shapes ([`test_ops_moe_grouped.cpp`](../tests/vt/test_ops_moe_grouped.cpp)); model-facing `MakeLinearMethod->Apply->BuildMarlinDenseResident` bad=0 K=4096+12288 ([`test_linear_method.cpp`](../tests/vllm/model_executor/layers/test_linear_method.cpp)); **e2e 3/4 token-exact vs oracle golden (async-off)** ([evidence](../docs/bench-evidence/mxfp4-qwen/W3-e2e-result.md)). **Default-async degeneration RESOLVED (`ROW-SERVE-ASYNC-DENSE-MIRROR`, `f9c969ae`):** the pre-existing classic-dense-Qwen3 async bug (device-mirror not wired for `qwen3.cpp`) is fixed; DEFAULT-config (async ON) e2e on dgx is now coherent + 3/4 token-exact vs the golden (p0/p1/p3 exact; p2 story = oracle-ratified near-tie, teacher-forced max gap 0.0000 nats). p3 formal near-tie gate now RATIFIED. W4 THROUGHPUT bench RAN on GB10 (`row/QUANT-CT-MXFP4-BENCH` `33e93608`, c1/c2/c4/c8x3, production graphed vLLM 0.25.0, oracle `VLLM_DISABLED_KERNELS=FlashInferMxFp4LinearKernel` -> Marlin W4A16): **BELOW-FLOOR** (gate NO). Total tok/s ours/vLLM 0.99x@c1 but ~0.91x@c2-c8 (c2 0.911, c4 0.919, c8 0.913); driven by TPOT +10-12%@c2-c8 (parity@c1); TTFT at parity; peak GPU mem **2.6x LESS** (28.3 vs 73.7 GiB). ATTRIBUTION (grounded, per-concurrency curve): the gap is a BATCHED-decode cost absent at batch-1 -> the grouped Marlin W4A16 decode GEMM at M=2..8, NOT batch-1 GEMV nor prefill; NOT a ceiling. Refined by nsys c1-vs-c2 (`row/QUANT-CT-MXFP4-GLUE`): the c2-c8 STEP is the FLASH decode launch (ours grid `(batch,hq)` over-waves at batch>=2; vLLM applies the FA2 `seqlenq_ngroups_swapped` decode swap → `(batch,kv_heads)`), NOT the marlin GEMM (per-shape parity) nor glue. **Lever LANDED gated default-OFF (`row/KERNEL-FA2-GQA-SWAP`, `VT_FA2_DECODE_GQA_SWAP`): vLLM's decode group-swap ported into the d128 varlen launcher, correctness-complete on GB10 (op RED/GREEN 280 assn, memcheck 0-err, #44 smoke swap-ON 3/3 det TOKEN-EXACT byte-identical to OFF); OFF byte-identical. c1-c8 x3 re-bench + default flip = next step (#47: flash ~28%@c2 / ~55%@c8 of the gap).** NEXT: same-tool nsys decode-window on c8 both engines. Harness (all additive; 27/35 byte-unchanged; CPU tests 45/45): `online_gate` `q3mxfp4` key + `POINTS_BY_MODEL`/`points_for`, [`mxfp4_smoke_gate.py`](../tools/bench/mxfp4_smoke_gate.py) (#44 battery), `dgx-online-serving.sh` q3mxfp4 branches, [`mxfp4-online-serving-grid.sh`](../scripts/mxfp4-online-serving-grid.sh). Full table + build contract (RelWithDebInfo + oracle flashinfer-cutlass + DISK build) in benchmark-record. Earlier CPU weight unpack + E8M0 dequant: NEW [mxfp4_dequant.h](../include/vllm/model_executor/model_loader/mxfp4_dequant.h) + [.cpp](../src/vllm/model_executor/model_loader/mxfp4_dequant.cpp#L14) (`E8M0ToF32` = `2^(byte-127)`, `DequantMxfp4ToBf16`/`ToF32`, group 32, no global; reuses `kE2M1Lut`). Unit gate [test_mxfp4_dequant.cpp](../tests/vllm/test_mxfp4_dequant.cpp#L34) — E8M0 known-byte decode, hand-computed 32-group dequant (bf16+f32), the E8M0-vs-fp8 + group-32-vs-16 RED traps, multi-row/group offsets, randomized rel-error vs a double-precision port of `dq_mxfp4_torch` with bf16==f32 exactness. CPU `-Werror` 0-warn. Ports FROM `compressed_tensors_w4a4_mxfp4.py:20-97` + `mxfp8_utils.py:61-65,222` + golden `tests/quantization/reference_mxfp4.py:28-117`. C/E/P PENDING: GPU W4A4 fp4 GEMM + Marlin W4A16 fallback + MoE expert path + e2e are NAMED later bricks; DeepSeek-V4 + Kimi-K3 loaders consume this once wired | [MXFP4 spike](specs/mxfp4-compressed-tensors.md) | `CLAIM-QUANT-MXFP4` | | `QUANT-CT-W4A8-FP8` | compressed-tensors W4A8 FP8 | W4/A8 | CUTLASS | - | - | - | - | - | `INVENTORIED` | - | leaf spec open | - | | `QUANT-CT-W4A8-INT8` | compressed-tensors W4A8 INT8 | W4/A8 | platform selected | - | - | - | - | - | `INVENTORIED` | - | leaf spec open | - | | `QUANT-CT-W8A8-FP8` | compressed-tensors W8A8 FP8 | W8/A8 | platform selected | - | - | - | - | - | `INVENTORIED` | - | leaf spec open | - | diff --git a/.agents/state.md b/.agents/state.md index 7824df7fb..bb00114bd 100644 --- a/.agents/state.md +++ b/.agents/state.md @@ -36917,3 +36917,87 @@ bigger delicate lift) and the ~0.7ms host/sched slice. No single lever reaches > Evidence: `dgx:~/mxfp4-nsys/{kern_sum_c1.txt,kern_sum_c2.txt,vllm_torch_c2.txt,vllm_kern_nsys_c2.txt, vllm_kern_sum.txt}` + reps `ours_c1/ours_c2/vllm_c2.nsys-rep`. Box: both locks free, GPU idle, disk 22G, local-ai-worker left down, tmux clean. + +## KERNEL-FA2-GQA-SWAP: the #47 lever LANDS — vLLM decode group-swap ported into the d128 varlen launcher, correctness-complete gated default-OFF (op RED/GREEN + memcheck + swap-ON smoke TOKEN-EXACT); bench + flip = next step + + +Implemented the #47-localized lever on `row/KERNEL-FA2-GQA-SWAP` off `origin/main` +`2f235beb`. **The port is DONE and correctness-complete on GB10; it ships gated +default-OFF, so production is byte-unchanged.** The c1-c8 x3 binding re-bench and +the default flip are the recorded next step (not run this session — the 72 GiB- +oracle grid is the box-safety-heavy arm; the correctness bar that GATES the flip +is fully met). Box left clean (both locks free, GPU idle, worker down, disk 22G). + +WHAT SHIPPED (the port, file:line): +- `src/vt/cuda/cuda_flash_attn_fa2.cu` `LaunchDecodeVarlenFA2Bf16` (now takes + `bool gqa_swap`): when TRUE-GQA pure decode (`max_seqlen_q==1`, `hq%kv==0`, + ngroups>1) and `gqa_swap`, presents q/o logically as `[b, ngroups, kv_heads, d]` + via kv-major-group-minor strides (`q_head_stride = ngroups*stride[1]`, no + materialized transpose), clears `cu_seqlens_q` (batched layout), sets + `h=kv_heads / h_h_k_ratio=1 / seqlen_q=ngroups / total_q=b*ngroups`, recomputes + `num_splits` on `batch*kv_heads*num_m_blocks`, normalizes causal→non-causal + (every seqlen_q row is the SAME decode token across a group's heads), sets + `seqlenq_ngroups_swapped=true`, dispatches ``. This is a 1:1 + mirror of the SHIPPED d256 `LaunchDecodeFA2Bf16` swap (which was the proof the + vendored `flash_fwd_kernel.h` get_lse_tile/combine already honor the flag in + BOTH the num_splits==1 direct-write and >1 combine paths). Scratch BYTES are + identical to the plain arm (`b*hq` LSE elements either way) so capture-safe; the + `DecodeShapeKey.groups` field keys swap-on vs swap-off apart. OFF path is the + UNTOUCHED prior code (byte-identical, proven). +- Gate `Fa2DecodeGqaSwapEnabled()` (`VT_FA2_DECODE_GQA_SWAP`, default OFF, `=1` + opt-in) + call-site `gqa_swap = qpk>1 && enabled` in `cuda_paged_attn.cu` + `LaunchPaged` (the `fa2_decode_qwen3` branch). Fwd-decl + internal header + (`Fa2DecodeSwapLaunchesForTesting`) updated; `RecordDecodeLaunch` gained a + `swapped` arg (+`swap_launches` counter; the d256 arms pass true). + +BLAST RADIUS (enumerated from the dispatcher): the d128 varlen decode +(`fa2_decode_qwen3` → `LaunchDecodeVarlenFA2Bf16`) is the SHARED dense FA2 decode +launcher. Under the toggle the swap engages for the Qwen3-dense family — 0.6B +(16/8, ngroups 2), 4B (32/8, ngroups 4), 8B / MXFP4-8B (32/8, ngroups 4) — and +ONLY those (d128 + bf16 + groups>1). The 27B/35B gate models decode on their OWN +d256 `LaunchDecodeFA2Bf16` (already swapped) — NOT touched. Voxtral's audio decode +also routes the d128 varlen launcher but is MHA (qpk==1 where applicable) → inert. + +GATE VERDICTS (dgx GB10, CUDA 13.0, sm_121a, RelWithDebInfo, VLLM_CPP_FLASH_ATTN=ON; +build in `~/mxfp4-bench` overlay of the 4 files, git-archived by SHA, md5-matched): +- BUILD: clean (`ninja test_ops_paged_attn` + `vllm-cli`, EXIT 0; the two CUDA TUs + compile on nvcc). One typo fixed (`std::max(double,float)` in the new test). +- OP UNIT TEST (RED-first): `test_ops_paged_attn --test-case="*varlen d128*"` = + **5/5 cases, 280/280 assertions GREEN** — the shipped OFF-path parity + 3 NEW + swap cases: (a) group-swap matches the f32 ComposedPagedRef across both ratios + ×batch{1,2,4,8}×len{5,21,1024} (short num_splits==1 + long split) with + `swap_launches==1` ASSERTED (grid engaged), (b) swap-ON near-ties the plain arm + directly (max_abs < 2e-2), (c) swap inert for MHA qpk==1 (`swap_launches==0`). + **RED PROVEN:** injecting a wrong swapped `o_head_stride` (drop the ngroups + factor) on dgx → the group-swap case FAILS loudly (26528 violations, max_abs + 2.60), so a GREEN means the layout is read correctly (not a vacuous fallback). + Full binary 28/28 cases / 454,679 assertions (no regression from the + RecordDecodeLaunch signature change; d256 arms unmoved). +- MEMCHECK: `compute-sanitizer --tool memcheck --leak-check full` on the swap + cases = **0 errors, 0 bytes leaked**. +- #44 SMOKE (Yi30/Qwen3-8B-MXFP4, the blast-radius model, groups=4, DEFAULT async + config = graphed decode) BOTH arms PASS: swap-OFF 3/3 det token-exact + coherent; + **swap-ON 3/3 deterministic TOKEN-EXACT vs golden AND byte-identical to the OFF + arm** (capitals/arithmetic/fibonacci char-identical; the near-tie story + identical too) — the swap does NOT flip a token e2e at these lengths, so the + near-tie razor was not needed. This is the capture-safety graphed token gate ON. +- SACRED e2e for Qwen3-0.6B/4B (16/8 + 32/8): the OP test covers both ratios at + the kernel level; the model-level e2e SACRED with those checkpoints' oracle + goldens is PENDING (bundled with the flip campaign below). + +DEFAULT DECISION: keep **OFF** this session — the flip is conditioned +(parity-enablers) on the speed bench showing ≥1.0x on every axis with no c1 +regression, which is NOT yet measured. The correctness half of that bar is met. + +REMAINING (the flip campaign, exact recipe): (1) `scripts/mxfp4-online-serving-grid.sh +--snapshot --build-dir ~/mxfp4-bench/build +--configure-log ` with `VT_FA2_DECODE_GQA_SWAP=1` on the OURS legs, vs the +#45 numbers (c1 0.989 / c2 0.911 / c4 0.919 / c8 0.913 tput; TPOT c2 28.27→25.45). +#47 projects flash alone closes ~28%@c2 / ~55%@c8 of the gap (decode flash +63.7→~41.7us c2 as the grid drops 192→96 CTAs); residual = marlin +7-9% +(grouped→dense-direct-A, refuted as a config lever) + ~0.7ms host/sched, so a +single lever may not reach ≥1.0x — record the honest per-axis outcome either way. +(2) Qwen3-0.6B/4B e2e SACRED with swap ON (near-tie razor if a token flips — the +smoke suggests it won't at short contexts). (3) then flip default per (f). +Evidence: dgx smoke `~/smoke.log` (transient), op/memcheck run inline (above). +Box: both locks free, GPU idle, disk 22G, local-ai-worker left down, tmux clean. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 98ec5c4bd..b5055d5af 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -307,7 +307,7 @@ built on it rather than keeping the flattering one. | Qwen3-dense decode CUDA-graph | Token-exact pass, ~4.3% e2e directional | Steady-state per-step tok/s | | Kimi-Linear-48B-A3B (KDA+MLA+MoE) | Full-model GB10 e2e RUNS (bf16-resident §13), NEAR-TIE 106/128, pool math CLOSES; default OFF | Full model RUNS on GB10 (bf16-resident, RSS peak 1.7 GiB, min-avail 21 GiB, no OOM). Token NEAR-TIE 106/128 (6/8 prompts exact, numerics vs deterministic oracle). 1.59 tok/s. Detail: spec §13 | | vLLM 0.26 re-benchmark | Pending | Re-run the binding grids on the advanced pin | -| MXFP4 Qwen3-8B (W4A16 Marlin) | Compute proven (#38); **DEFAULT-config (async ON) e2e coherent + 3/4 token-exact** on dgx after `ROW-SERVE-ASYNC-DENSE-MIRROR`; degenerate without the fix | p2/p3 near-tie RATIFIED. W4 grid (`33e93608`): 0.99x@c1 / ~0.91x@c2-c8, mem 2.6x LESS. Marlin-config + glue levers **both REFUTED**; c2-c8 step = flash decode GQA group-swap. Detail + lever: state `QUANT-CT-MXFP4-GLUE` | +| MXFP4 Qwen3-8B (W4A16 Marlin) | Compute proven (#38); **DEFAULT-config (async ON) e2e coherent + 3/4 token-exact** on dgx after `ROW-SERVE-ASYNC-DENSE-MIRROR`; degenerate without the fix | p2/p3 near-tie RATIFIED. W4 grid: 0.99x@c1 / ~0.91x@c2-c8, mem 2.6x LESS. c2-c8 lever = FA2 decode GQA group-swap **LANDED gated-OFF** (`VT_FA2_DECODE_GQA_SWAP`); bench+flip pending. state `KERNEL-FA2-GQA-SWAP` | | SGLang floor arms | Never ran | Both arms of the SGLang comparison | | cuBLAS invocation-parity guard | CI guard landed (CPU); `kGemvHeuristicAlgos` refactor build-verify owed | `nvcc` rebuild + SACRED gate on dgx | diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 38709f8a3..2c2369d3d 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -71,6 +71,7 @@ portable/reference path. In normal operation leave them unset. | `VT_FA2_PREFILL` | on (CUDA) | The portable prefill attention instead of the vendored FA2 | | `VT_FA2_DECODE` | on (CUDA) | The portable decode attention instead of the vendored FA2 | | `VT_FA2_DECODE_4B` | on (CUDA, Qwen3.5-4B) | The portable paged decode attention instead of the ratio-4 vendored FA2 path; the 27B and 35B selectors are unchanged | +| `VT_FA2_DECODE_GQA_SWAP` | off (CUDA, Qwen3-dense d128 decode) | `=1` applies vLLM's FA2 `seqlenq_ngroups_swapped` decode optimization in the d128 varlen launcher (launches (batch, kv_heads) not (batch, hq), packing the ngroups query heads into seqlen_q); default off is byte-identical to the plain-varlen reduction, on is a split-reduction near-tie | | `VT_CPU_REF` | off | Set on to force the portable reference path (dequantize-everything oracle), the standard "is this a kernel bug?" bisect switch | | `VT_DFLASH_PAGED` | on (CUDA, DFlash spec-decode) | The materialized `[context;block]` draft forward instead of the fixed-capacity paged draft-KV store read through `vt::DFlashPagedBlockAttention` (bit-identical; only the DFlash single-request propose path) | | `VT_DFLASH_GRAPH` | on (CUDA, DFlash spec-decode) | The eager paged draft step instead of the captured/replayed draft-step CUDA graph (replayed==eager bit-identical; only the DFlash single-request propose path) | diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 6c5d2ddb6..0a13b4929 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -65,7 +65,7 @@ are our reading of their documented behavior, not measurements. | GGUF k-quants and i-quants | ✅ | ☐ | ☐ | ✅ | | AWQ | ◐ CPU dequant | ✅ | ✅ | ☐ | | GPTQ | ◐ CPU dequant | ✅ | ✅ | ☐ | -| MXFP4 compressed-tensors | ◐ W4A16 Marlin compute proven; e2e 3/4 token-exact; W4 throughput grid ran on GB10: ~0.91x vLLM c2-c8 / 0.99x c1, peak GPU mem 2.6x less; below-floor (batched-decode gap) | ✅ | ✅ | ☐ | +| MXFP4 compressed-tensors | ◐ W4A16 Marlin compute proven; e2e 3/4 token-exact; W4 grid GB10 ~0.91x c2-c8 / 0.99x c1, mem 2.6x less. c2-c8 lever = FA2 decode GQA group-swap, LANDED gated-OFF (`VT_FA2_DECODE_GQA_SWAP`); bench+flip pending | ✅ | ✅ | ☐ | | fp8 weights | ✅ | ✅ | ✅ | ☐ | | bf16 / fp16 | ✅ | ✅ | ✅ | ✅ | | Safetensors direct load, no conversion | ✅ | ✅ | ✅ | ☐ | diff --git a/docs/STATUS.md b/docs/STATUS.md index 38eb120d6..a18371edb 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -84,7 +84,7 @@ token-for-token correctness against the pinned oracle. | Safetensors loading | Supported | Both gate models plus every registered dense/MoE family | | GGUF loading (F32/F16/BF16/Q4_0/Q8_0/Q2_K/Q3_K/Q4_K/Q5_K/Q6_K/IQ2_XXS/IQ3_XXS/IQ2_S/MXFP4/NVFP4) | Supported; compute-in-quant (keep-quant) on CPU AND now CUDA for the six K-block encodings PLUS Q2_K/IQ2_XXS/IQ3_XXS (DeepSeek-V4 W8, 2026-07-29 - the FIRST CUDA keep-quant GGUF k-quant GEMM `KERNEL-QUANT-CIQ-GEMM-CUDA`, MMVQ-style dequant-in-kernel, GB10-gated 92401/92401 vs the CPU oracle, so a CUDA runner keeps blocks compressed and dots them on the GPU instead of the ARM cores); **NVFP4 now COMPUTES IN FP4 on CUDA for the dense-MLP and full-attention projections (2026-07-29, `CLAIM-GGUF-NVFP4-COMPUTE`), no longer materialize-only** | Weights in six block encodings stay compressed from file to matmul on CPU (no BF16 expansion). NVFP4 (ggml type 40) DEQUANTIZES, including the per-tensor (per-expert) `.scale` sidecar the container keeps outside the blocks; gated BIT-EXACT against the compressed-tensors NVFP4 path on real Qwen3.6-27B bytes from both containers. **It no longer expands to bf16 on CUDA:** an NVFP4 matmul/expert weight is REPACKED at load into the same (`weight_packed [N,K/2]`, `weight_scale [N,K/16]`) operand pair the compressed-tensors path produces - a pure byte permutation, gated BYTE-IDENTICAL against that container - and the existing `vt::MatmulNvfp4*` kernels run on it, so no new kernel exists and no numerics are re-derived. Covers the dense MLP + full-attention q/k/v/o and the MoE shared/routed experts; the GDN `in_proj_*` family and `ssm_out` still expand (the V-head reorder rewrites their layout) and a CPU build still expands everything - the documented `part` subset. **MEASURED on GB10 (2026-07-29), same-binary A/B, one `flock`, idle box, 2 reps per arm:** peak RSS **50.8 -> 25.7 GiB**, load-and-generate **1:58 -> 0:41**, and the 256 projections that move cost 35 840 MiB expanded against 10 080 MiB fp4-resident (3.56x). **The divergence against the safetensors sibling CLOSES:** the fp4 arm is token-IDENTICAL over 24 greedy tokens where the bf16 arm of the same binary diverges at index 4, which retires the reading that that divergence was permanent. It is REPORTED, not gated: the two containers are not the same model - the GGUF NVFP4-quantizes 192 GDN `in_proj` tensors the safetensors keeps BF16 (mean relative weight error ~0.18) and their activation global scales differ - so identity is not guaranteed and a cross-container throughput arm is not valid. SACRED gates unmoved: `test_qwen27_paged_engine` 235/235, `test_qwen36_paged_engine` 315/315. **The MoE (35B) stacked-expert arm is now HARDWARE-GATED too (2026-07-29)**, superseding the gap recorded here: the real 35B A3B NVFP4 GGUF loads and generates through the fp4 path, its 120 routed-expert stacks x 256 experts repack to the modelopt safetensors' own operands with ZERO differing bytes over 840 sampled (tensor, expert) slabs, and all 840 per-expert `.scale[e]` are bit-identical to that expert's `weight_scale_2` - the per-expert scale INDEXING, mutation-proved against both a `scales[0]`-for-every-expert and an expert-0-slab-for-every-expert mutant. Same-binary A/B: peak RSS 68.5 -> 22.7 GiB (3.01x), load-and-generate 1:51.9 -> 0:28.8, tokens IDENTICAL (correct here, since the 35B routed experts run the W4A16 grouped GEMM in both arms). Recorded as OPEN, not smoothed over: this case's 24-token greedy stream is NOT run-to-run stable (one of three `use_a16` runs and one of four safetensors-reference runs differed), so the binding results are the weight-level byte identity and the residency audit, not a token-exactness claim; `test_qwen36_paged_engine` is token-exact at ITS engine params, so the instability belongs to this case's configuration and attributing it is owed work. That run also found and FIXED a latent defect the MoE arm made reachable: the two fp4 fused MoE blocks issued the router GEMM assuming the safetensors `[K,N]` gate layout and threw `matmul: inner dims mismatch` on the GGUF's `[N,K]` one; `MoeRouterLogits` now branches on `nk` (inert for the safetensors path, SACRED gates unmoved). **Q2_K (id 10) + IQ2_XXS (id 16) DEQUANTIZE (2026-07-29, `CLAIM-DSV4-GGUF-LOADER`):** the ~2-bit types the single-Spark `DeepSeek-V4-Flash-GGUF UD-IQ2_XXS`/`UD-Q2_K_XL` vehicles use, ported 1:1 from llama.cpp `ggml-quants.c` (`iq2xxs_grid` codebook + signs; Q2_K nibble sub-scale/min), unit-gated on hand-derived bytes (`test_gguf_dequant` 15/15). Dequant-only (no vec_dot -> expand-bf16). A V4-GGUF model still cannot RUN: the V4-GGUF name map (tensor-manifest-blocked) + the V4 forward (W3-W8) remain. **Multi-shard split GGUF READING landed (2026-08-03, `CLAIM-GGUF-SPLIT-SHARDS`):** `GgufFile::Open` now transparently stitches llama.cpp `gguf-split` shards (`...-00001-of-00003.gguf`) — every shard mmap'd, tensor tables merged, KV metadata taken from shard `00001`, and the sibling shard mappings kept alive by the primary so keep-quant mmap-borrows stay valid across shards (`OwnsSpan` is shard-aware); `VT_GGUF_NO_SPLIT=1` opts out; unit-gated (`test_gguf` split-merge / no-split / count-mismatch cases, 33/33 local). This unblocks the real 3-shard `unsloth/DeepSeek-V4-Flash-0731 UD-IQ2_M` (~91 GiB), whose layout is the NATIVE `deepseek4` arch — per-block `ffn_gate_tid2eid` hash tables (hash layers 0/1/2) + `hc_*` MHC + DSA compressor/indexer are all PRESENT (name-map 1328/1328), `vocab_size` derives from `token_embd` — NOT a standard llama.cpp conversion, so no loader-layout change is owed. It now loads THROUGH 1324/1328 tensors; the sole remaining gap is 4 routed-expert slabs quantized with IQ2_S (id 22, ×2) + MXFP4 (id 39, ×2) — encodings we have GGUF block traits for but no keep-quant vec_dot, so they hit the expand→dequant path which lacks them. Dequant-expanding those 4 big expert tensors to bf16 would add ~17 GiB (~106 GiB total → GB10 OOM-reboot risk), so the memory-safe fix is an IQ2_S+MXFP4 keep-quant kernel (CPU dequant dispatch + the `iq2s_grid` codebook + a CUDA `DotSuperblock`), spec'd as the next brick **IQ2_S (id 22) + MXFP4 (id 39) DEQUANTIZE + KEEP-QUANT on CPU (2026-08-03, `CLAIM-DSV4-UDIQ2M-QUANT`, off-GPU):** the extra per-tensor "dynamic" encodings the `unsloth/DeepSeek-V4-Flash-GGUF UD-IQ2_M` checkpoint mixes into its last routed-expert slabs (IQ2_S `ffn_gate/up` dotting Q8_K, MXFP4 `ffn_down` dotting Q8_0) — ported 1:1 from llama.cpp `ggml-quants.c` @ 237ad9b96 (`iq2s_grid` 1024-entry codebook + DIRECT sign bytes; MXFP4 `kvalues_mxfp4` + `e8m0_to_fp32_half` micro-scaling, distinct from the compressed-tensors `E8M0ToF32` NVFP4 path). CPU dequant + keep-quant `vec_dot`, unit-gated on hand-derived golden bytes (`test_gguf_dequant` 17/17), an INDEPENDENT f64 dequant-then-dot + GEMM NMSE (`test_ops_quant_dot` 19/19), and keep-quant routing (`test_gguf_keep_quant` 37/37) — all CPU-green, so UD-IQ2_M's four previously-`unsupported ggml type 22/39` slabs now load COMPRESSED (no ~17 GiB bf16 expansion that OOM-reboots the box). CUDA: the IQ2_S device `DotSuperblock` is wired into the Q8_K grouped-MoE GEMM and now **CUDA-BUILT + LINKED on GB10 (sm_121a, CUDA 13.0, `-Werror`, 2026-08-03 integration)** — it compiles clean and the merged binary links; MXFP4's device dot (`DotMXFP4`) is written but NOT wired (Q8_0-activation needs a separate 32-block GEMM) so it is marked `[[maybe_unused]]` to keep the ready math without tripping nvcc #177-D, and on GPU MXFP4 CPU-fallbacks like Q4_0/Q8_0. The V4-GGUF forward + a real UD-IQ2_M GPU load/coherence run are owed | | AWQ / GPTQ quantization | W0 spike + W1 CPU INT4 dequant primitive; not yet loadable end to end | INT4 unpack+dequant-to-bf16 for BOTH community formats, mirroring vLLM 1:1 (AWQ reverse-order `awq_triton.py`; GPTQ `qdq_4.cuh` with zero_offset v1/v2 + act-order g_idx). Unit-gated RED-first (hand-computed known bytes + double-precision roundtrip). NOT wired to a loader, no GPU Marlin compute, no model run yet: config recognizer (W2), Marlin GPU GEMM riding the vendored NVFP4 Marlin (W4), CPU e2e (W3), GPTQ 8/2/3-bit (W5) and MoE (W6) are named next bricks. See [.agents/specs/awq-gptq-quant.md](../.agents/specs/awq-gptq-quant.md) | -| MXFP4 (compressed-tensors `mxfp4-pack-quantized`) | Native Marlin W4A16 keep-quant compute PROVEN correct (#38); W4 throughput grid RAN on GB10 (`33e93608`): ~0.91x vLLM c2-c8 / 0.99x c1, peak GPU mem 2.6x LESS; BELOW-FLOOR (batched-decode gap). **Marlin-config + glue-fusion levers both REFUTED by measurement** (`row/QUANT-CT-MXFP4-M28-LEVER`, `row/QUANT-CT-MXFP4-GLUE`); the c2-c8 step is the flash decode GQA group-swap (lever site `cuda_flash_attn_fa2.cu:1075`). Detail in state.md | Shared with DeepSeek-V4-Flash + Kimi-K3 MXFP4-expert paths. CPU E8M0 dequant (`2^(byte-127)`, group 32, no global scale) unit-gated 5/5·1142 vs `dq_mxfp4_torch`. GPU W4A4 fp4 GEMM + MoE-expert e2e remain later bricks | +| MXFP4 (compressed-tensors `mxfp4-pack-quantized`) | Native Marlin W4A16 keep-quant compute PROVEN correct (#38); W4 throughput grid RAN on GB10 (`33e93608`): ~0.91x vLLM c2-c8 / 0.99x c1, peak GPU mem 2.6x LESS; BELOW-FLOOR (batched-decode gap). **Marlin-config + glue-fusion levers both REFUTED by measurement** (`row/QUANT-CT-MXFP4-M28-LEVER`, `row/QUANT-CT-MXFP4-GLUE`); the c2-c8 step is the flash decode GQA group-swap. **Lever now IMPLEMENTED gated default-OFF (`VT_FA2_DECODE_GQA_SWAP`, `row/KERNEL-FA2-GQA-SWAP`): correctness-complete GB10 (op RED/GREEN, memcheck 0-err, swap-ON smoke token-exact); bench+flip pending.** Detail in state.md | Shared with DeepSeek-V4-Flash + Kimi-K3 MXFP4-expert paths. CPU E8M0 dequant (`2^(byte-127)`, group 32, no global scale) unit-gated 5/5·1142 vs `dq_mxfp4_torch`. GPU W4A4 fp4 GEMM + MoE-expert e2e remain later bricks | | CPU backend vs llama.cpp | At or ahead on every axis (GGUF) | Prefill 1.18x ahead, decode at parity, peak memory 1.01x, byte-identical greedy tokens. Single-stream only; no concurrent-serving comparison has been measured | | Paged KV cache + prefix caching | Supported | Block-paged full attention, hybrid full-attention + GDN state groups, automatic prefix caching (APC) on by default for dense models (cache-ON gated end to end: token-identical output, cache hits, faster TTFT) | | fp8 KV cache (`cache_dtype=fp8`) | In progress (W1 CPU brick), not yet usable end-to-end | HIGH-priority memory/throughput lever (halves the KV footprint). W0 spike + W1 CPU brick landed (`KV-FP8` ACTIVE): fp8-e4m3 K/V STORE (`Quantize(hp/scale)`) + the paged-attention READ dequant (`Dequant(fp8)*scale`) + the `cache_dtype` config parse, all CPU-gated RED-first (`test_ops_fp8_kv_cache` 8/8·511; a wrong store direction fails 3/480). Storage is 1-byte fp8 (`DType::kI8`) + a `Fp8KVCacheDataType` interpretation enum, per-tensor k/v scales (mirroring vLLM `BaseKVCacheMethod`). The CUDA store + fp8 paged-attention read (the GPU memory-halving path, DGX-blocked), the runner/spec integration (half-sized KV blocks + checkpoint-scale threading + `--kv-cache-dtype`/`--calculate-kv-scales`), fp8_e5m2 and per-head scales are named W2-W5 in [.agents/specs/fp8-kv-cache.md](../.agents/specs/fp8-kv-cache.md). No model can run with an fp8 KV cache yet | diff --git a/src/vt/cuda/cuda_flash_attn_fa2.cu b/src/vt/cuda/cuda_flash_attn_fa2.cu index 69b2f0c24..620ac4f8e 100644 --- a/src/vt/cuda/cuda_flash_attn_fa2.cu +++ b/src/vt/cuda/cuda_flash_attn_fa2.cu @@ -250,6 +250,10 @@ struct Fa2DebugCounters { std::atomic no_split_launches{0}; std::atomic scratch_allocations{0}; std::atomic scratch_reuses{0}; + // Decode launches that presented q via the seqlenq_ngroups_swapped layout + // (the d256 group-swap arms are always swapped; the d128 varlen arm swaps only + // under VT_FA2_DECODE_GQA_SWAP). Lets the op test PROVE the swap grid engaged. + std::atomic swap_launches{0}; }; Fa2DebugCounters& DebugCounters() { @@ -257,7 +261,7 @@ Fa2DebugCounters& DebugCounters() { return counters; } -void RecordDecodeLaunch(bool split, bool allocated) { +void RecordDecodeLaunch(bool split, bool allocated, bool swapped) { Fa2DebugCounters& counters = DebugCounters(); if (!counters.enabled.load(std::memory_order_relaxed)) return; counters.decode_launches.fetch_add(1, std::memory_order_relaxed); @@ -265,6 +269,7 @@ void RecordDecodeLaunch(bool split, bool allocated) { .fetch_add(1, std::memory_order_relaxed); (allocated ? counters.scratch_allocations : counters.scratch_reuses) .fetch_add(1, std::memory_order_relaxed); + if (swapped) counters.swap_launches.fetch_add(1, std::memory_order_relaxed); } } // namespace @@ -867,36 +872,56 @@ void LaunchDecodeFA2Bf16(cudaStream_t stream, Tensor& out, const Tensor& query, p.seqlenq_ngroups_swapped = true; p.num_splits = num_splits; - RecordDecodeLaunch(num_splits > 1, inserted); + RecordDecodeLaunch(num_splits > 1, inserted, /*swapped=*/true); FLASH_NAMESPACE::run_mha_fwd_splitkv_dispatch(p, stream); Check(cudaGetLastError(), "decode splitkv dispatch launch"); } -// Launch the pinned FA2 VARLEN decode — the EXACT reduction vLLM's -// flash_attn_varlen_func runs for a paged bf16 KV cache at head_dim 128 -// (Qwen3-dense). Unlike LaunchDecodeFA2Bf16 (the d256 group-swap -// seqlenq_ngroups_swapped route), this presents the decode batch as PLAIN -// varlen: one query row per request (cu_seqlens_q = query_start_loc), full -// query heads (h = hq, h_h_k_ratio = groups), per-request K length via -// seqused_k, causal = true, NO group swap. Because a block_table is present, -// upstream forces the split-KV kernel even at num_splits==1, so we dispatch -// run_mha_fwd_splitkv_dispatch<..., 128, causal> exactly like the varlen -// prefill launcher (Split=false when num_splits==1). num_splits is the exact -// port of upstream num_splits_heuristic (=1 for the short gate contexts), and -// for num_splits>1 the split combine addresses O via batch_idx*o_batch_stride; -// with seqlen_q==1 per request that equals o_row_stride, so we set -// o_batch_stride = o_row_stride and the combine writes the packed [total_q,Hq,D] -// output correctly (this is why the packed-prefill combine restriction — -// seqlen_q>1, irregular row spacing — does NOT apply to decode). -// Ported from vllm-project/flash-attention @ 2c839c33 (mha_varlen_fwd paged -// path + set_params_splitkv) and vLLM v0.25.0 flash_attn.py flash_attn_varlen_func. +// Launch the pinned FA2 VARLEN decode for a paged bf16 KV cache at head_dim 128 +// (Qwen3-dense). Two presentations of the same vendored split-KV kernel, chosen +// by gqa_swap (VT_FA2_DECODE_GQA_SWAP): +// +// gqa_swap == false (DEFAULT, byte-identical to the shipped path): PLAIN +// varlen — one query row per request (cu_seqlens_q = query_start_loc), full +// query heads (h = hq, h_h_k_ratio = groups), per-request K length via +// seqused_k, causal = true, NO group swap. Because a block_table is present, +// upstream forces the split-KV kernel even at num_splits==1, so we dispatch +// run_mha_fwd_splitkv_dispatch<..., 128, causal> exactly like the varlen +// prefill launcher (Split=false when num_splits==1). num_splits is the exact +// port of upstream num_splits_heuristic (=1 for the short gate contexts), and +// for num_splits>1 the split combine addresses O via batch_idx*o_batch_stride; +// with seqlen_q==1 per request that equals o_row_stride, so we set +// o_batch_stride = o_row_stride and the combine writes the packed [total_q,Hq,D] +// output correctly (this is why the packed-prefill combine restriction — +// seqlen_q>1, irregular row spacing — does NOT apply to decode). +// +// gqa_swap == true: the exact seqlenq_ngroups_swapped decode optimization vLLM +// runs (mha_fwd_kvcache / set_params_splitkv). vLLM's varlen decode grid is +// (b, kv_heads) not (b, hq): the ngroups query heads of a KV group are packed +// into the seqlen_q dimension, KV is read once per group, and the heuristic +// sees batch*kv_heads instead of batch*hq — halving the CTA count at batch>=2 +// (#47: ours over-waved at c2/c8). Presented WITHOUT a materialized transpose, +// exactly like LaunchDecodeFA2Bf16 (d256): the physical query head layout is +// kv-major group-minor (head = kv*ngroups + g, the same mapping the non-swap +// kernel reads via bidh/h_h_k_ratio), so independent strides expose the logical +// [b, ngroups, kv_heads, d] view. cu_seqlens_q is cleared (batched, non-varlen), +// causal is normalized to non-causal (every swapped seqlen_q row is the SAME +// decode token across query heads, so a causal mask across them would be +// wrong), and the vendored get_lse_tile/combine already honor +// seqlenq_ngroups_swapped for the LSE/O strides. NON-byte-exact vs the plain +// arm only when num_splits>1 (the split reduction order differs) — a near-tie +// that moves TOWARD vLLM's own numerics. +// +// Ported from vllm-project/flash-attention @ 2c839c33 (mha_varlen_fwd paged path +// + mha_fwd_kvcache seqlenq_ngroups_swapped + set_params_splitkv) and vLLM +// v0.25.0 flash_attn.py flash_attn_varlen_func. void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, const Tensor& k_cache, const Tensor& v_cache, const Tensor& block_table, const Tensor& seq_lens, const Tensor& query_start_loc, const PagedAttentionArgs& args, int64_t hq, int64_t d, int64_t num_reqs, int64_t num_kv_heads, - int64_t block_size) { + int64_t block_size, bool gqa_swap) { const int64_t total_q = query.shape[0]; if (total_q == 0 || num_reqs == 0 || hq == 0 || d == 0) return; if (query.dtype != DType::kBF16 || out.dtype != DType::kBF16 || @@ -945,23 +970,38 @@ void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, const int kv_heads = static_cast(num_kv_heads); const int head_dim = static_cast(d); const int max_blocks = static_cast(block_table.shape[1]); + const int query_groups = (kv_heads > 0) ? heads / kv_heads : 0; + + // seqlenq_ngroups_swapped decode presentation (VT_FA2_DECODE_GQA_SWAP). Only for + // TRUE GQA pure decode: one query token per request (max_seqlen_q==1) and + // ngroups>1. groups==1 (MHA) has nothing to pack; multi-token would break the + // "all seqlen_q rows are the same decode token" normalization. + const bool do_swap = gqa_swap && kv_heads > 0 && (heads % kv_heads == 0) && + query_groups > 1 && max_seqlen_q == 1; const auto stream_scratch = Fa2ScratchFor(query.device.index, s); std::lock_guard submit_lock(stream_scratch->submit_mu); // Exact port of set_params_splitkv for head_dim 128: block-N is 128, block-M // is 64, and the heuristic receives 2*numSM for the 128-thread CTA occupancy - // model. For the gate contexts (<=2 key blocks, batch 1) this yields 1. + // model. For the gate contexts (<=2 key blocks, batch 1) this yields 1. Under + // the swap the heuristic sees (b, kv_heads, ngroups-as-seqlen_q) instead of + // (b, hq, 1) — the exact grid dims vLLM's mha_fwd_kvcache feeds it. constexpr int kBlockN = 128; // Headdim <= 128 constexpr int kBlockM = 64; + const int p_seqlen_q = do_swap ? query_groups : max_seqlen_q; + const int heuristic_heads = do_swap ? kv_heads : heads; const int num_n_blocks = (max_seqlen_k + kBlockN - 1) / kBlockN; - const int num_m_blocks = (max_seqlen_q + kBlockM - 1) / kBlockM; + const int num_m_blocks = (p_seqlen_q + kBlockM - 1) / kBlockM; const int num_splits = NumSplitsHeuristic( - batch * heads * num_m_blocks, stream_scratch->num_sms * 2, num_n_blocks, 128); - - // softmax_lse is [nheads, total_q] f32 (unpadded/varlen LSE, written+consumed - // by the combine). oaccum/lseaccum only when the KV dimension is split. - const DecodeShapeKey key{batch, heads, kv_heads, 0, + batch * heuristic_heads * num_m_blocks, stream_scratch->num_sms * 2, num_n_blocks, 128); + + // softmax_lse is f32, batch*hq elements either way (swap: b*kv_heads*ngroups == + // b*hq; plain: hq*total_q == hq*b). The swap only reinterprets the stride + // (seqlenq_ngroups_swapped), so the scratch BYTES are identical — capture-safe. + // `groups` keys swap-on vs swap-off apart so one process can A/B without alias. + // oaccum/lseaccum only when the KV dimension is split. + const DecodeShapeKey key{batch, heads, kv_heads, do_swap ? query_groups : 0, head_dim, max_blocks, static_cast(block_size), num_splits}; auto [it, inserted] = stream_scratch->varlen_decode.try_emplace(key); DecodeScratch& scratch = it->second; @@ -1002,15 +1042,30 @@ void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, p.v_ptr = v_cache.data; p.o_ptr = out.data; - // Varlen q/o: cu_seqlens_q drives the row offset; batch strides are ignored on - // the non-split path but MUST equal o_row_stride so the split combine (which - // writes O via batch_idx*o_batch_stride, seqlen_q==1) lands each request's row. - p.q_batch_stride = query.stride[0]; - p.q_row_stride = query.stride[0]; - p.q_head_stride = query.stride[1]; - p.o_batch_stride = out.stride[0]; - p.o_row_stride = out.stride[0]; - p.o_head_stride = out.stride[1]; + if (do_swap) { + // Group-swap presentation (mirrors LaunchDecodeFA2Bf16): expose the logical + // [b, ngroups, kv_heads, d] view over the physical kv-major group-minor query + // [total_q=b, hq, d] via independent strides, no materialized transpose. + // physical q[req, kv*ngroups + g, :] -> logical q[b=req, row=g, head=kv, :] + // cu_seqlens_q is cleared so the kernel reads the batched (b, seqlen_q=ngroups) + // layout; per-request K length still comes from seqused_k. + p.q_batch_stride = query.stride[0]; + p.q_row_stride = query.stride[1]; + p.q_head_stride = static_cast(query_groups) * query.stride[1]; + p.o_batch_stride = out.stride[0]; + p.o_row_stride = out.stride[1]; + p.o_head_stride = static_cast(query_groups) * out.stride[1]; + } else { + // Varlen q/o: cu_seqlens_q drives the row offset; batch strides are ignored on + // the non-split path but MUST equal o_row_stride so the split combine (which + // writes O via batch_idx*o_batch_stride, seqlen_q==1) lands each request's row. + p.q_batch_stride = query.stride[0]; + p.q_row_stride = query.stride[0]; + p.q_head_stride = query.stride[1]; + p.o_batch_stride = out.stride[0]; + p.o_row_stride = out.stride[0]; + p.o_head_stride = out.stride[1]; + } // Paged k/v [num_blocks, block_size, num_kv_heads, d]. p.k_batch_stride = k_cache.stride[0]; p.k_row_stride = k_cache.stride[1]; @@ -1019,7 +1074,9 @@ void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, p.v_row_stride = v_cache.stride[1]; p.v_head_stride = v_cache.stride[2]; - p.cu_seqlens_q = query_start_loc.Ptr(); // [0,1,2,...,num_reqs] + // Swap presents the batched (non-varlen) layout: cu_seqlens_q == nullptr makes + // BlockInfo read seqlen_q = params.seqlen_q (=ngroups) and q_offset = b*batch_stride. + p.cu_seqlens_q = do_swap ? nullptr : query_start_loc.Ptr(); // else [0,1,...,num_reqs] p.cu_seqlens_k = nullptr; // paged: block_table + seqused_k p.seqused_k = seq_lens.Ptr(); p.softmax_lse_ptr = scratch.softmax_lse.ptr; @@ -1027,16 +1084,16 @@ void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, p.oaccum_ptr = scratch.out_accum.ptr; p.b = batch; - p.h = heads; + p.h = do_swap ? kv_heads : heads; // swap packs groups into seqlen_q p.h_k = kv_heads; - p.h_h_k_ratio = static_cast(hq / num_kv_heads); - p.seqlen_q = max_seqlen_q; // 1 per decode request + p.h_h_k_ratio = do_swap ? 1 : static_cast(hq / num_kv_heads); + p.seqlen_q = p_seqlen_q; // ngroups (swap) or 1 (plain) p.seqlen_k = max_seqlen_k; - p.seqlen_q_rounded = RoundMultiple(max_seqlen_q, 128); + p.seqlen_q_rounded = RoundMultiple(p_seqlen_q, 128); p.seqlen_k_rounded = RoundMultiple(max_seqlen_k, 128); p.d = head_dim; p.d_rounded = RoundMultiple(head_dim, 64); - p.total_q = static_cast(total_q); + p.total_q = do_swap ? batch * query_groups : static_cast(total_q); // Attention logit soft-cap fold (vLLM Attention(logits_soft_cap=...), // gemma2.py:202). flash-attn convention: p.softcap = scale/cap, scale_softmax = @@ -1058,12 +1115,23 @@ void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, p.scale_softmax_rp_dropout = args.scale; p.philox_args = at::PhiloxCudaState(0, 0); - // Plain causal decode (NO group-swap normalization): vLLM passes causal=True to - // flash_attn_varlen_func; with seqlen_q==1 the single query still sees the full - // context, and the causal template selects the exact n_block geometry vLLM runs. - p.is_causal = args.causal; - p.window_size_left = -1; - p.window_size_right = args.causal ? 0 : -1; + if (do_swap) { + // Upstream normalizes max_seqlen_q==1 causal decode to NON-causal before the + // swap: the ngroups seqlen_q rows are the SAME decode token replicated across + // a KV group's query heads, so a causal mask ACROSS those rows would be wrong. + // The single decode token already sees the full context, so non-causal + + // per-request seqused_k is exact. Matches LaunchDecodeFA2Bf16. + p.is_causal = false; + p.window_size_left = -1; + p.window_size_right = -1; + } else { + // Plain causal decode (NO group-swap normalization): vLLM passes causal=True to + // flash_attn_varlen_func; with seqlen_q==1 the single query still sees the full + // context, and the causal template selects the exact n_block geometry vLLM runs. + p.is_causal = args.causal; + p.window_size_left = -1; + p.window_size_right = args.causal ? 0 : -1; + } p.is_seqlens_k_cumulative = true; // ignored while cu_seqlens_k == nullptr p.is_rotary_interleaved = false; p.rotary_dim = 0; @@ -1071,14 +1139,16 @@ void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, p.block_table = block_table.Ptr(); p.block_table_batch_stride = block_table.stride[0]; p.page_block_size = static_cast(block_size); - p.unpadded_lse = true; // LSE is [nheads, total_q] - p.seqlenq_ngroups_swapped = false; // the whole point: no group swap + p.unpadded_lse = true; // LSE is [nheads, total_q] (get_lse_tile honors the swap) + p.seqlenq_ngroups_swapped = do_swap; p.num_splits = num_splits; - RecordDecodeLaunch(num_splits > 1, inserted); - if (args.causal) { + RecordDecodeLaunch(num_splits > 1, inserted, do_swap); + if (!do_swap && args.causal) { FLASH_NAMESPACE::run_mha_fwd_splitkv_dispatch(p, s); } else { + // Swap decode is normalized non-causal (like the d256 arm); a plain non-causal + // caller also lands here. FLASH_NAMESPACE::run_mha_fwd_splitkv_dispatch(p, s); } Check(cudaGetLastError(), "varlen decode splitkv dispatch launch"); @@ -1133,6 +1203,7 @@ void ResetFa2DecodeDebugCounters() { counters.no_split_launches.store(0, std::memory_order_relaxed); counters.scratch_allocations.store(0, std::memory_order_relaxed); counters.scratch_reuses.store(0, std::memory_order_relaxed); + counters.swap_launches.store(0, std::memory_order_relaxed); counters.enabled.store(true, std::memory_order_release); } @@ -1152,6 +1223,10 @@ uint64_t Fa2DecodeNoSplitLaunchesForTesting() { return DebugCounters().no_split_launches.load(std::memory_order_relaxed); } +uint64_t Fa2DecodeSwapLaunchesForTesting() { + return DebugCounters().swap_launches.load(std::memory_order_relaxed); +} + uint64_t Fa2DecodeScratchAllocationsForTesting() { return DebugCounters().scratch_allocations.load(std::memory_order_relaxed); } diff --git a/src/vt/cuda/cuda_flash_attn_fa2_internal.h b/src/vt/cuda/cuda_flash_attn_fa2_internal.h index d466ce22a..2b8c65595 100644 --- a/src/vt/cuda/cuda_flash_attn_fa2_internal.h +++ b/src/vt/cuda/cuda_flash_attn_fa2_internal.h @@ -22,6 +22,11 @@ void DisableFa2DecodeDebugCounters(); uint64_t Fa2DecodeLaunchesForTesting(); uint64_t Fa2DecodeSplitLaunchesForTesting(); uint64_t Fa2DecodeNoSplitLaunchesForTesting(); +// Decode launches that used the seqlenq_ngroups_swapped presentation. Lets a test +// PROVE the swapped grid engaged (VT_FA2_DECODE_GQA_SWAP), not just that a launch +// happened. The d256 group-swap arms always count; the d128 varlen arm counts +// only under the swap toggle. +uint64_t Fa2DecodeSwapLaunchesForTesting(); uint64_t Fa2DecodeScratchAllocationsForTesting(); uint64_t Fa2DecodeScratchReusesForTesting(); diff --git a/src/vt/cuda/cuda_paged_attn.cu b/src/vt/cuda/cuda_paged_attn.cu index d693f439d..0da873302 100644 --- a/src/vt/cuda/cuda_paged_attn.cu +++ b/src/vt/cuda/cuda_paged_attn.cu @@ -62,15 +62,16 @@ void LaunchDecodeFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, const PagedAttentionArgs& args, int64_t hq, int64_t d, int64_t num_reqs, int64_t num_kv_heads, int64_t block_size); -// VARLEN d128 decode — the exact non-swap flash_attn_varlen_func reduction vLLM -// runs for Qwen3-dense DECODE (bf16 paged KV, head_dim 128). Toggle -// VT_FA2_DECODE_QWEN3 (see Fa2DecodeQwen3Enabled()). +// VARLEN d128 decode for Qwen3-dense DECODE (bf16 paged KV, head_dim 128). Toggle +// VT_FA2_DECODE_QWEN3 (see Fa2DecodeQwen3Enabled()). gqa_swap selects the vLLM +// seqlenq_ngroups_swapped grid (VT_FA2_DECODE_GQA_SWAP, see Fa2DecodeGqaSwapEnabled()); +// gqa_swap==false is byte-identical to the shipped plain-varlen reduction. void LaunchDecodeVarlenFA2Bf16(cudaStream_t s, Tensor& out, const Tensor& query, const Tensor& k_cache, const Tensor& v_cache, const Tensor& block_table, const Tensor& seq_lens, const Tensor& query_start_loc, const PagedAttentionArgs& args, int64_t hq, int64_t d, int64_t num_reqs, int64_t num_kv_heads, - int64_t block_size); + int64_t block_size, bool gqa_swap); #endif // VLLM_CPP_FLASH_ATTN namespace { @@ -2557,6 +2558,21 @@ bool Fa2DecodeQwen3Enabled() { const char* e = std::getenv("VT_FA2_DECODE_QWEN3"); return e == nullptr || e[0] != '0'; } + +// vLLM's decode GQA group-swap (seqlenq_ngroups_swapped) for the d128 varlen arm. +// vLLM's paged decode grid is (batch, kv_heads) — the ngroups query heads are +// packed into seqlen_q and KV is read once per group — while our plain-varlen arm +// launches (batch, hq), over-waving at batch>=2 (#47: decode flash 63.7us c2 / +// 218us c8 vs vLLM 41.7/151). This routes the SAME vendored split-KV kernel through +// the swapped presentation (already proven by the d256 LaunchDecodeFA2Bf16 arm). +// DEFAULT OFF: it is non-byte-exact vs the plain arm when num_splits>1 (the split +// reduction order changes -> near-tie, moving toward vLLM's numerics) and touches +// the CUDA-graph-captured decode, so it flips ON only after the near-tie token +// gates hold. =1 opts in for a same-binary A/B. Read fresh (host path per step). +bool Fa2DecodeGqaSwapEnabled() { + const char* e = std::getenv("VT_FA2_DECODE_GQA_SWAP"); + return e != nullptr && e[0] != '0'; +} #endif // VLLM_CPP_FLASH_ATTN // TQ = query dtype, TKV = KV-cache dtype (decoupled: Phase-1 bf16 KV cache keeps @@ -2696,9 +2712,13 @@ void LaunchPaged(cudaStream_t s, Tensor& out, const Tensor& query, const Tensor& LaunchDecodeFA2Bf16(s, out, query, k_cache, v_cache, block_table, seq_lens, args, hq, d, num_reqs, num_kv_heads, block_size); } else if (fa2_decode_qwen3) { + // Group-swap (VT_FA2_DECODE_GQA_SWAP) only when it can matter: true GQA + // (qpk>1). The launcher re-checks max_seqlen_q==1; groups==1 (MHA) has + // nothing to pack and stays on the plain-varlen reduction. + const bool gqa_swap = qpk > 1 && Fa2DecodeGqaSwapEnabled(); LaunchDecodeVarlenFA2Bf16(s, out, query, k_cache, v_cache, block_table, seq_lens, query_start_loc, args, hq, d, num_reqs, num_kv_heads, - block_size); + block_size, gqa_swap); } else // NOLINT(readability/braces) — chains into the flash2vec ladder below #endif if (flash2vec) { diff --git a/tests/vt/test_ops_paged_attn.cpp b/tests/vt/test_ops_paged_attn.cpp index 8e0fb1caa..41751dda0 100644 --- a/tests/vt/test_ops_paged_attn.cpp +++ b/tests/vt/test_ops_paged_attn.cpp @@ -45,6 +45,7 @@ void DisableFa2DecodeDebugCounters(); uint64_t Fa2DecodeLaunchesForTesting(); uint64_t Fa2DecodeSplitLaunchesForTesting(); uint64_t Fa2DecodeNoSplitLaunchesForTesting(); +uint64_t Fa2DecodeSwapLaunchesForTesting(); uint64_t Fa2DecodeScratchAllocationsForTesting(); uint64_t Fa2DecodeScratchReusesForTesting(); size_t Fa2DecodeScratchShapeCountForTesting(int device, void* stream); @@ -1448,6 +1449,7 @@ struct Fa2DecodeRunStats { uint64_t launches = 0; uint64_t split_launches = 0; uint64_t no_split_launches = 0; + uint64_t swap_launches = 0; }; Fa2DecodeRunStats RunFa2DecodeCase(Fa2DecodeCase& c, const char* toggle, @@ -1498,12 +1500,17 @@ Fa2DecodeRunStats RunFa2DecodeCase(Fa2DecodeCase& c, const char* toggle, return stats; } -// VARLEN d128 decode harness (Qwen3-dense): the exact non-swap -// flash_attn_varlen_func path (VT_FA2_DECODE_QWEN3). Same host/device fixture, -// but head_dim 128 and the plain-varlen launcher (cu_seqlens_q = qsl, causal, -// no group swap). Validated against the same f32 ComposedPagedRef. +// VARLEN d128 decode harness (Qwen3-dense): the flash_attn_varlen_func path +// (VT_FA2_DECODE_QWEN3), head_dim 128. `swap_toggle` drives VT_FA2_DECODE_GQA_SWAP +// so one harness covers both the plain-varlen reduction and vLLM's decode +// seqlenq_ngroups_swapped grid. `expect_swap` asserts the swapped launcher engaged +// (or did NOT, for the eligibility guards). Both validated against the same f32 +// ComposedPagedRef. When `out_bits` is non-null the downloaded bf16 output is +// copied out so callers can cross-check swap-ON vs swap-OFF (the near-tie razor). Fa2DecodeRunStats RunFa2VarlenDecodeCase(Fa2DecodeCase& c, const char* toggle, - bool expect_fa2) { + bool expect_fa2, const char* swap_toggle = "0", + bool expect_swap = false, + std::vector* out_bits = nullptr) { Backend& gpu = vt::GetBackend(DeviceType::kCUDA); QueueGuard guard(gpu); DeviceTensor query(gpu, guard.q, DType::kBF16, {c.batch, c.hq, c.d}, @@ -1520,6 +1527,7 @@ Fa2DecodeRunStats RunFa2VarlenDecodeCase(Fa2DecodeCase& c, const char* toggle, DeviceTensor out(gpu, guard.q, DType::kBF16, {c.batch, c.hq, c.d}); EnvGuard qwen3_toggle("VT_FA2_DECODE_QWEN3", toggle); + EnvGuard swap_env("VT_FA2_DECODE_GQA_SWAP", swap_toggle); vt::cuda::testing::ResetFa2DecodeDebugCounters(); PagedAttentionArgs args{c.scale, true}; args.query_start_loc_host = c.qsl.data(); @@ -1532,11 +1540,16 @@ Fa2DecodeRunStats RunFa2VarlenDecodeCase(Fa2DecodeCase& c, const char* toggle, const Fa2DecodeRunStats stats{ vt::cuda::testing::Fa2DecodeLaunchesForTesting(), vt::cuda::testing::Fa2DecodeSplitLaunchesForTesting(), - vt::cuda::testing::Fa2DecodeNoSplitLaunchesForTesting()}; + vt::cuda::testing::Fa2DecodeNoSplitLaunchesForTesting(), + vt::cuda::testing::Fa2DecodeSwapLaunchesForTesting()}; vt::cuda::testing::DisableFa2DecodeDebugCounters(); CheckBf16AgainstReference(got, c.Reference(c.seq_lens), expect_fa2 ? "FA2 varlen decode d128" : "paged fallback"); CHECK(stats.launches == (expect_fa2 ? 1U : 0U)); + // The swap grid MUST have engaged (or not) as expected — a plain-path fallback + // that still matched the reference would otherwise pass the port vacuously. + CHECK(stats.swap_launches == (expect_swap ? 1U : 0U)); + if (out_bits != nullptr) *out_bits = std::move(got); return stats; } @@ -1583,6 +1596,96 @@ TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode toggle-off uses fallback RunFa2VarlenDecodeCase(c, "0", /*expect_fa2=*/false); } +// Qwen3-dense VARLEN d128 decode GQA GROUP-SWAP (VT_FA2_DECODE_GQA_SWAP): vLLM's +// seqlenq_ngroups_swapped decode grid ported into the varlen launcher (#47). The +// port must (a) match the same f32 composed reference within the near-tie band at +// both gate ratios (0.6B 16/8 => ngroups 2, 4B 32/8 => ngroups 4), across batch +// and both num_splits regimes, AND (b) PROVE the swapped grid actually engaged +// (swap_launches==1) — a silent fallback to the plain arm that still matched the +// reference would pass the port vacuously. Grid-sensitivity (the RED anchor): a +// wrong swapped stride / seqlen_q / head count corrupts the output and trips +// CheckBf16AgainstReference, so a GREEN here means the layout is read correctly. +TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode GQA group-swap matches composed reference") { + if (!HasCuda()) { + MESSAGE("no CUDA backend; skipping FA-2 varlen d128 group-swap parity (dgx-pending)"); + return; + } + for (const auto& ratio : {std::pair{16, 8}, + std::pair{32, 8}}) { + for (const int batch : {1, 2, 4, 8}) { + for (const int base_len : {5, 21, 1024}) { // short => num_splits==1; long => split + CAPTURE(ratio.first); + CAPTURE(batch); + CAPTURE(base_len); + std::vector lengths(static_cast(batch)); + for (int i = 0; i < batch; ++i) + lengths[static_cast(i)] = base_len + i * 3; + Fa2DecodeCase c(ratio.first, ratio.second, std::move(lengths), + 7700U + static_cast(batch * 31 + base_len), + /*capacity_blocks=*/0, /*head_dim=*/128); + RunFa2VarlenDecodeCase(c, "1", /*expect_fa2=*/true, /*swap_toggle=*/"1", + /*expect_swap=*/true); + } + } + } +} + +// Near-tie razor: the swap arm and the plain-varlen arm read the SAME q/k/v, so +// their bf16 outputs must agree within the split-reduction near-tie band (they are +// only non-byte-exact when num_splits>1). Directly comparing the two engine +// outputs (not just each vs the host reference) pins that the swap is a numerics- +// preserving reshape of vLLM's own reduction, not a different computation. +TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode swap near-ties the plain-varlen arm") { + if (!HasCuda()) { + MESSAGE("no CUDA backend; skipping FA-2 varlen d128 swap-vs-plain near-tie (dgx-pending)"); + return; + } + for (const auto& ratio : {std::pair{16, 8}, + std::pair{32, 8}}) { + for (const int batch : {2, 8}) { + for (const int base_len : {21, 1024}) { + CAPTURE(ratio.first); + CAPTURE(batch); + CAPTURE(base_len); + std::vector lengths(static_cast(batch)); + for (int i = 0; i < batch; ++i) + lengths[static_cast(i)] = base_len + i * 3; + Fa2DecodeCase c(ratio.first, ratio.second, std::move(lengths), + 7900U + static_cast(batch * 31 + base_len), + /*capacity_blocks=*/0, /*head_dim=*/128); + std::vector plain, swapped; + RunFa2VarlenDecodeCase(c, "1", /*expect_fa2=*/true, /*swap_toggle=*/"0", + /*expect_swap=*/false, &plain); + RunFa2VarlenDecodeCase(c, "1", /*expect_fa2=*/true, /*swap_toggle=*/"1", + /*expect_swap=*/true, &swapped); + REQUIRE(plain.size() == swapped.size()); + double max_abs = 0.0; + for (size_t i = 0; i < plain.size(); ++i) { + const double diff = static_cast(Bf16BitsToF32(plain[i])) - + static_cast(Bf16BitsToF32(swapped[i])); + max_abs = std::max(max_abs, std::abs(diff)); + } + INFO("swap vs plain max_abs = " << max_abs); + CHECK(max_abs < 2.0e-2); + } + } + } +} + +// Eligibility guard: with the swap env ON but a NON-GQA topology (MHA, qpk==1) the +// launcher must stay on the plain-varlen path — there are no query groups to pack. +// Proves the dispatcher/launcher gate, not just the happy path. +TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode swap is inert for MHA (qpk==1)") { + if (!HasCuda()) { + MESSAGE("no CUDA backend; skipping FA-2 varlen d128 swap MHA-ineligibility (dgx-pending)"); + return; + } + Fa2DecodeCase c(/*Hq=*/8, /*Hkv=*/8, {21, 24, 27}, 7811, + /*capacity_blocks=*/0, /*head_dim=*/128); + RunFa2VarlenDecodeCase(c, "1", /*expect_fa2=*/true, /*swap_toggle=*/"1", + /*expect_swap=*/false); +} + TEST_CASE("paged_attention CUDA FA-2 split heuristic mirrors upstream") { using vt::cuda::testing::Fa2DecodeNumSplitsForTesting; CHECK(Fa2DecodeNumSplitsForTesting(32, 40, 18, 128) == 1);