diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 01aba8052d..c1f3e5e02a 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -122,7 +122,7 @@ class LoRA_MLP(torch.autograd.Function): e = e .view(-1, e .shape[-1]) g = g .view(-1, g .shape[-1]) dtype = X.dtype - + DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) se = torch.nn.functional.sigmoid(e) f = e * se @@ -130,6 +130,7 @@ class LoRA_MLP(torch.autograd.Function): df = se * (1 - f) + f DW_f = DW * f DW_dfg = DW * df * g + h, DW_f, DW_dfg = DW, e, g # DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) # DW, e, g = swiglu_DWf_DW_dfg_kernel(DW, e, g) # h, DW_f, DW_dfg = DW, e, g