From 317bbc57fee180b6881480e460bdefacbdf6590a Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 16:16:32 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 01aba8052d..c1f3e5e02a 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -122,7 +122,7 @@ class LoRA_MLP(torch.autograd.Function): e = e .view(-1, e .shape[-1]) g = g .view(-1, g .shape[-1]) dtype = X.dtype - + DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) se = torch.nn.functional.sigmoid(e) f = e * se @@ -130,6 +130,7 @@ class LoRA_MLP(torch.autograd.Function): df = se * (1 - f) + f DW_f = DW * f DW_dfg = DW * df * g + h, DW_f, DW_dfg = DW, e, g # DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) # DW, e, g = swiglu_DWf_DW_dfg_kernel(DW, e, g) # h, DW_f, DW_dfg = DW, e, g