Update fast_lora.py

2024-01-26 16:08:10 +11:00 · 2024-01-26 16:08:10 +11:00 · c4ac728d4c
commit c4ac728d4c
parent ab44945336
1 changed files with 2 additions and 3 deletions
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@ -122,9 +122,8 @@ class LoRA_MLP(torch.autograd.Function):
        e  = e .view(-1, e .shape[-1])
        g  = g .view(-1, g .shape[-1])
        dtype = X.dtype
-
-        DW_f   = (dY @ W.T * f)
-        DW_dfg = (dY @ W.T * df * g)
+        
+        DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS)
        se = torch.nn.functional.sigmoid(e)
        f = e * se
        h = f * g