From 694da2dba3c48ab268497772277a2d801351b8f3 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 20:06:52 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index ecae146bdb..6f8ea2a68e 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -125,6 +125,7 @@ class LoRA_MLP(torch.autograd.Function): DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) se = torch.nn.functional.sigmoid(e) f = torch.nn.functional.silu(e) + h = f * g DW_f = DW * f DW_dfg = DW * se * g * (1 + f * (1 - se)) # f = e * se