From ebfc8f8a55769c616a18327677275d6c4686bc96 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 23 Feb 2024 17:57:42 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 3ed0d3c914..6ec4bbf141 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -128,27 +128,27 @@ class LoRA_MLP(torch.autograd.Function): h, df, de = DW, e, g # Down projection LoRA weights - d_downA = h.t() @ (dY @ downB.t()) - d_downB = (downA.t() @ h.t()) @ dY + d_downA = (h.t() @ dY) @ downB.t() + d_downB = downA.t() @ (h.t() @ dY) d_downA *= downS d_downB *= downS # Up projection LoRA weights - d_upA = X.t() @ (df @ upB.t()) - d_upB = (upA.t() @ X.t()) @ df + d_upA = (X.t() @ df) @ upB.t() + d_upB = upA.t() @ (X.t() @ df) d_upA *= upS d_upB *= upS # Gate projection LoRA weights - d_gateA = X.t() @ (de @ gateB.t()) - d_gateB = (gateA.t() @ X.t()) @ de + d_gateA = (X.t() @ de) @ gateB.t() + d_gateB = gateA.t() @ (X.t() @ de) d_gateA *= gateS d_gateB *= gateS # dX = matmul_lora(df, upW.t(), upW_quant, upB, upA, upS) # dX += matmul_lora(de, gateW.t(), gateW_quant, gateB, gateA, gateS) upW = fast_dequantize(upW.t(), upW_quant) - dX = torch.matmul(df, upW.t(), out = X) + dX = torch.matmul(df, upW.t())#, out = X) del upW dX += df @ upB.to(dtype).t() @ (upS * upA.to(dtype).t())