From 4f573494da67fde0036619a12f81314d7edea5fa Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 20:04:51 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 6ea5345714..ecae146bdb 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -124,11 +124,14 @@ class LoRA_MLP(torch.autograd.Function): DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) se = torch.nn.functional.sigmoid(e) - f = e * se - h = f * g - df = se * (1 - f) + f + f = torch.nn.functional.silu(e) DW_f = DW * f - DW_dfg = DW * df * g + DW_dfg = DW * se * g * (1 + f * (1 - se)) + # f = e * se + # h = f * g + # df = se * (1 - f) + f + # DW_f = DW * f + # DW_dfg = DW * df * g # DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) # DW, e, g = swiglu_DWf_DW_dfg_kernel(DW, e, g) # h, DW_f, DW_dfg = DW, e, g