Update fast_lora.py

2024-01-26 14:06:25 +11:00 · 2024-01-26 14:06:25 +11:00 · ab44945336
commit ab44945336
parent 84772de40f
1 changed files with 2 additions and 2 deletions
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@ -123,8 +123,8 @@ class LoRA_MLP(torch.autograd.Function):
        g  = g .view(-1, g .shape[-1])
        dtype = X.dtype

-        DW_f   = (D @ W.T * f)
-        DW_dfg = (D @ W.T * df * g)
+        DW_f   = (dY @ W.T * f)
+        DW_dfg = (dY @ W.T * df * g)
        se = torch.nn.functional.sigmoid(e)
        f = e * se
        h = f * g