From c4ac728d4c87b10823499433cd0465d0766b5068 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 16:08:10 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index e76173b479..01aba8052d 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -122,9 +122,8 @@ class LoRA_MLP(torch.autograd.Function): e = e .view(-1, e .shape[-1]) g = g .view(-1, g .shape[-1]) dtype = X.dtype - - DW_f = (dY @ W.T * f) - DW_dfg = (dY @ W.T * df * g) + + DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) se = torch.nn.functional.sigmoid(e) f = e * se h = f * g