Update fast_lora.py

2024-01-26 20:06:52 +11:00 · 2024-01-26 20:06:52 +11:00 · 694da2dba3
commit 694da2dba3
parent 4f573494da
1 changed files with 1 additions and 0 deletions
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@ -125,6 +125,7 @@ class LoRA_MLP(torch.autograd.Function):
        DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS)
        se = torch.nn.functional.sigmoid(e)
        f = torch.nn.functional.silu(e)
+        h = f * g
        DW_f   = DW * f
        DW_dfg = DW * se * g * (1 + f * (1 - se))
        # f = e * se