From 3865c8cd68219257f55ebfd5c4de432c828d4ddd Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 23:06:06 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 6f8ea2a68e..8e57b7bfdf 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -162,15 +162,14 @@ class LoRA_MLP(torch.autograd.Function): # (D @ W.T * f) @ (U.T + B.T @ A.T) dX = torch.matmul(DW_f, upW.t(), out = X) del upW - new_dX = upS * (DW_f @ upB.to(dtype).t() @ (upA.to(dtype).t())) + dX += (DW_f @ upB.to(dtype).t() @ (upS * upA.to(dtype).t())) # And add the derivative for the gate projection gateW = fast_dequantize(gateW.t(), gateW_quant) - new_dX2 = DW_dfg @ gateW.t() - # dX += DW_dfg @ gateW.t() + # new_dX2 = DW_dfg @ gateW.t() + dX += DW_dfg @ gateW.t() del gateW - new_dX2 += gateS * (DW_dfg @ gateB.to(dtype).t() @ (gateA.to(dtype).t())) - dX += (new_dX + new_dX2) + dX += DW_dfg @ gateB.to(dtype).t() @ (gateS * gateA.to(dtype).t()) # gateW, gateW_quant, gateA, gateB, gateS, # upW, upW_quant, upA, upB, upS,