From f7760719f0a48de63030ca61c6b754a5c6571d70 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 27 Jan 2024 01:28:03 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 6f25949656..7c09db9c40 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -126,8 +126,8 @@ class LoRA_MLP(torch.autograd.Function): se = 1 / (1 + torch.exp(-e.float())) f = torch.nn.functional.silu(e) h = f * g - DW_f = DW * f - DW_dfg = DW.float() * g.float() * se * (1.0 + e.float() - f.float()) + DW_f = (DW * f) + DW_dfg = (DW * g).float() * se * (1.0 + e.float() * (1.0 - se)) DW_dfg = DW_dfg.to(dtype) # f = e * se # h = f * g