From 7fb64e0e31d871ada3c1d3ac8114304f8f62ccea Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 27 Jan 2024 03:58:21 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 997235f82e..550539d927 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -236,11 +236,11 @@ class LoRA_MLP_New(torch.autograd.Function): dtype = X.dtype DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) - df = DW * g # 88us - dg = DW * f # 88us - de = cls._silu_backward(df, e) # 90us + df = DW * f # 88us + dg = DW * g # 88us + de = cls._silu_backward(dg, e) # 90us - dX = matmul_lora(dg, upW.t(), upW_quant, upB, upA, upS) + dX = matmul_lora(df, upW.t(), upW_quant, upB, upA, upS) dX += matmul_lora(de, gateW.t(), gateW_quant, gateB, gateA, gateS) # Down projection LoRA weights