From 0a1aa98d3c6851ece9a48de5ff8c0ee804e6fb0a Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 27 Jan 2024 02:19:00 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 7c09db9c40..cceeac2d64 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -90,8 +90,8 @@ class LoRA_MLP(torch.autograd.Function): e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS) g = matmul_lora(X, upW, upW_quant, upA, upB, upS) - # h = torch.nn.functional.silu(e) * g - h = swiglu_fg_kernel(e, g) + h = torch.nn.functional.silu(e) * g + # h = swiglu_fg_kernel(e, g) i = matmul_lora(h, downW, downW_quant, downA, downB, downS) ctx.custom_saved_tensors = (