From 38bff800b64c40abd60707fcb54f772d18657a57 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 27 Jan 2024 01:16:31 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 20618bcac0..6f25949656 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -123,11 +123,12 @@ class LoRA_MLP(torch.autograd.Function): dtype = X.dtype DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) - se = torch.nn.functional.sigmoid(e) + se = 1 / (1 + torch.exp(-e.float())) f = torch.nn.functional.silu(e) h = f * g DW_f = DW * f - DW_dfg = DW * g * se * (1.0 + e - f) + DW_dfg = DW.float() * g.float() * se * (1.0 + e.float() - f.float()) + DW_dfg = DW_dfg.to(dtype) # f = e * se # h = f * g # df = se * (1 - f) + f