ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update swiglu.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-27 19:05:02 +11:00
commit 2e4c59deaf
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/kernels/swiglu.py
View file

@ -71,7 +71,7 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE : tl.constexpr,):
# se = 1.0 / (1.0 + torch.exp(-e))
se_row = 1.0 / (1.0 + tl.exp(-e_row))
# f = (se * e).to(dtype)
f_row = se_row * e_roe
f_row = se_row * e_row
f_row = f_row.to(DW_row.dtype)
# h = f * g
h_row = f_row * g_row