From 2e4c59deaf8c53d6632bb1364c33a9d7e0aeb63e Mon Sep 17 00:00:00 2001
From: Daniel Han-Chen <danielhanchen@gmail.com>
Date: Sat, 27 Jan 2024 19:05:02 +1100
Subject: [PATCH] Update swiglu.py

---
 unsloth/kernels/swiglu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py
index 116ad04904..f7ba5995c5 100644
--- a/unsloth/kernels/swiglu.py
+++ b/unsloth/kernels/swiglu.py
@@ -71,7 +71,7 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE : tl.constexpr,):
     # se = 1.0 / (1.0 + torch.exp(-e))
     se_row = 1.0 / (1.0 + tl.exp(-e_row))
     # f = (se * e).to(dtype)
-    f_row = se_row * e_roe
+    f_row = se_row * e_row
     f_row = f_row.to(DW_row.dtype)
     # h = f * g
     h_row  =  f_row * g_row