From 485d54fd70211c6ad94667e595fb12650cd95874 Mon Sep 17 00:00:00 2001
From: Daniel Han-Chen <danielhanchen@gmail.com>
Date: Fri, 26 Jan 2024 23:48:17 +1100
Subject: [PATCH] Update fast_lora.py

---
 unsloth/kernels/fast_lora.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 12cecb5a9c..20618bcac0 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -127,7 +127,7 @@ class LoRA_MLP(torch.autograd.Function):
         f = torch.nn.functional.silu(e)
         h = f * g
         DW_f   = DW * f
-        DW_dfg = DW * se * g * (1 + e * (1 - se))
+        DW_dfg = DW * g * se * (1.0 + e - f)
         # f = e * se
         # h = f * g
         # df = se * (1 - f) + f