Fix Layernorm when num_cols not a power of 2 (#1867)

* fix

* Update layernorm.py

---------

Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
Mohamed Mekkouri 2025-03-01 23:23:22 +01:00 committed by GitHub
commit 8180e9803c

View file

@ -49,7 +49,8 @@ def layernorm_forward(
b_row = tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)
mean_X = tl.sum(X_row, axis = 0) / n_cols
XX = X_row - mean_X
# (X[0] - mean) == -mean so we need to mask it out
XX = tl.where(mask, X_row - mean_X, 0)
row_var = tl.sum(XX * XX, axis = 0) / n_cols
inv_var = tl.math.rsqrt(row_var + eps)
tl.store (r, inv_var)