[#10345][perf] Enable multi-stream MOE for super. Also adds multi-stream MLA attn

examples/auto_deploy/model_registry/configs/glm-4.7-flash.yaml

@@ -4,5 +4,19 @@ max_seq_len: 4096
 enable_chunked_prefill: true
 cuda_graph_batch_sizes: [1, 2, 4, 8, 16, 32, 64]
 transforms:
+  match_swiglu_pattern:
+    enabled: true
+  match_nvfp4_swiglu_pattern:
+    enabled: true
   fuse_nvfp4_moe:
     allow_different_input_scales: true
+  fuse_nvfp4_swiglu:
+    enabled: true
+  fuse_swiglu:
+    enabled: true
+  multi_stream_moe:
+    stage: compile
+    enabled: true
+  multi_stream_mla_attn:
+    stage: compile
+    enabled: true

examples/auto_deploy/super_v3.yaml

@@ -37,7 +37,7 @@ transforms:
         "fc2_latent_proj": "gather"
   multi_stream_moe:
     stage: compile
-    enabled: false
+    enabled: true
   gather_logits_before_lm_head:
     # TODO: fix https://github.com/NVIDIA/TensorRT-LLM/issues/9878 to enable by default
     enabled: true

tensorrt_llm/_torch/auto_deploy/config/default.yaml

@@ -50,6 +50,7 @@ transforms:
     expected_layout: bsnd
   match_rmsnorm_pattern:
     stage: pattern_matcher
+    run_shape_prop: true
   match_l2norm_pattern:
     stage: pattern_matcher
   ############################################################################################
@@ -75,6 +76,18 @@ transforms:
     stage: pattern_matcher
   quantize_nvfp4_from_graph:
     stage: pattern_matcher
+  # SwiGLU pattern matching must run AFTER quantization transforms. For pre-quantized
+  # checkpoints (e.g., NVFP4), quantization converts torch_linear_simple ops to quantized
+  # ops first, and then match_nvfp4_swiglu_pattern captures the NVFP4 SwiGLU pattern.
+  # For non-quantized models, quantization transforms are no-ops, so match_swiglu_pattern
+  # proceeds normally.
+  match_swiglu_pattern:
+    stage: pattern_matcher
+    enabled: false
+  match_nvfp4_swiglu_pattern:
+    stage: pattern_matcher
+    requires_shape_prop: true
+    enabled: false
   quantize_fp8_moe:
     stage: pattern_matcher
   quantize_nvfp4_moe:
@@ -126,6 +139,8 @@ transforms:
   fuse_nvfp4_linear:
     stage: post_load_fusion
     backend: trtllm
+  fuse_nvfp4_swiglu:
+    stage: post_load_fusion
   fuse_moe:
     stage: post_load_fusion
     expect_mem_change: true
@@ -149,6 +164,9 @@ transforms:
   fuse_l2norm:
     stage: post_load_fusion
     backend: fla
+  fuse_swiglu:
+    stage: post_load_fusion
+    enabled: false
   fuse_add_rms_norm:
     stage: post_load_fusion
     enabled: true
@@ -200,6 +218,9 @@ transforms:
   multi_stream_moe:
     stage: compile
     enabled: false
+  multi_stream_mla_attn:
+    stage: compile
+    enabled: false
   compile_model:
     stage: compile
     expect_mem_change: true

tensorrt_llm/_torch/auto_deploy/custom_ops/linear/__init__.py

@@ -18,9 +18,11 @@
 This module provides linear layer implementations:
 - linear: Linear layer operations
 - torch_router: MoE router operations
+- swiglu: SwiGLU MLP custom operations
 """
 
 __all__ = [
     "linear",
     "torch_router",
+    "swiglu",
 ]