From ab449453365177ac7afa41bce24469de27222914 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 14:06:25 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 4f91254e6c..e76173b479 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -123,8 +123,8 @@ class LoRA_MLP(torch.autograd.Function): g = g .view(-1, g .shape[-1]) dtype = X.dtype - DW_f = (D @ W.T * f) - DW_dfg = (D @ W.T * df * g) + DW_f = (dY @ W.T * f) + DW_dfg = (dY @ W.T * df * g) se = torch.nn.functional.sigmoid(e) f = e * se h = f * g