Skip to content

Commit 91acac7

Browse files
committed
Revert "Add stash_type=1 to RMSNormalization for FP32 variance accumulation"
This reverts commit ea4e95c.
1 parent ea4e95c commit 91acac7

1 file changed

Lines changed: 1 addition & 5 deletions

File tree

src/mobius/components/_rms_norm.py

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ def forward(self, op: builder.OpBuilder, hidden_states: ir.Value):
4343
effective_weight,
4444
epsilon=self.variance_epsilon,
4545
axis=-1,
46-
stash_type=1,
4746
)
4847

4948

@@ -190,7 +189,4 @@ def apply_rms_norm(op: builder.OpBuilder, x, weight, eps):
190189
Returns:
191190
Normalized tensor with the same shape as input.
192191
"""
193-
# stash_type=1 (FLOAT) ensures the variance computation uses FP32
194-
# internally even when the input is FP16/BF16, preventing overflow
195-
# when squaring large values.
196-
return op.RMSNormalization(x, weight, epsilon=eps, axis=-1, stash_type=1)
192+
return op.RMSNormalization(x, weight, epsilon=eps, axis=-1)

0 commit comments

Comments
 (0)