From f3da0d2e4a661e799fda429f391454ab8d84e99d Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 27 Jan 2024 02:52:54 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 8321a0b770..cceeac2d64 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -139,19 +139,19 @@ class LoRA_MLP(torch.autograd.Function): # h, DW_f, DW_dfg = DW, e, g # Down projection LoRA weights - d_downA = h.t() @ dY @ downB.t() + d_downA = h.t() @ (dY @ downB.t()) d_downB = (downA.t() @ h.t()) @ dY d_downA *= downS d_downB *= downS # Up projection LoRA weights - d_upA = X.t() @ DW_f @ upB.t() + d_upA = X.t() @ (DW_f @ upB.t()) d_upB = (upA.t() @ X.t()) @ DW_f d_upA *= upS d_upB *= upS # Gate projection LoRA weights - d_gateA = X.t() @ DW_dfg @ gateB.t() + d_gateA = X.t() @ (DW_dfg @ gateB.t()) d_gateB = (gateA.t() @ X.t()) @ DW_dfg d_gateA *= gateS d_gateB *= gateS