From 6fa0635971e086b954f9302eeb875942c3bba6a5 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 27 Jan 2024 23:20:41 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 3f6d4d56c2..b3a1098355 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -125,15 +125,15 @@ class LoRA_MLP(torch.autograd.Function): dtype = X.dtype DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) - e = e.float() - se = 1.0 / (1.0 + torch.exp(-e)) - f = (se * e).to(dtype) - h = f * g - df = DW * f - dg = DW * g - de = (dg.float() * se * (1.0 + e * (1.0 - se))).to(dtype) - # DW, e, g = swiglu_DWf_DW_dfg_kernel(DW, e, g) - # h, df, de = DW, e, g + # e = e.float() + # se = 1.0 / (1.0 + torch.exp(-e)) + # f = (se * e).to(dtype) + # h = f * g + # df = DW * f + # dg = DW * g + # de = (dg.float() * se * (1.0 + e * (1.0 - se))).to(dtype) + DW, e, g = swiglu_DWf_DW_dfg_kernel(DW, e, g) + h, df, de = DW, e, g # Down projection LoRA weights d_downA = h.t() @ (dY @ downB.t()) @@ -148,8 +148,8 @@ class LoRA_MLP(torch.autograd.Function): d_upB *= upS # Gate projection LoRA weights - d_gateA = X.t() @ (dg @ gateB.t()) - d_gateB = (gateA.t() @ X.t()) @ dg + d_gateA = X.t() @ (de @ gateB.t()) + d_gateB = (gateA.t() @ X.t()) @ de d_gateA *= gateS d_gateB *= gateS