ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update fast_lora.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-27 02:34:21 +11:00
commit 4ae3ad3bce
1 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
unsloth/kernels/fast_lora.py
View file

@ -161,16 +161,16 @@ class LoRA_MLP(torch.autograd.Function):
# (D @ W.T * f) @ U.T
upW = fast_dequantize(upW.t(), upW_quant)
# (D @ W.T * f) @ (U.T + B.T @ A.T)
dX = torch.matmul(DW_f, upW.t())
dX = torch.matmul(DW_f, upW.t(), out = X)
del upW
dX += upS * ((DW_f @ upB.to(dtype).t() @ (upA.to(dtype).t())))
dX += (DW_f @ upB.to(dtype).t() @ (upS * upA.to(dtype).t()))
# And add the derivative for the gate projection
gateW = fast_dequantize(gateW.t(), gateW_quant)
# new_dX2 = DW_dfg @ gateW.t()
# dX +=
# del gateW
dX += DW_dfg @ gateW.t() + gateS * (DW_dfg @ gateB.to(dtype).t() @ (gateA.to(dtype).t()))
dX += DW_dfg @ gateW.t()
del gateW
dX += DW_dfg @ gateB.to(dtype).t() @ (gateS * gateA.to(dtype).t())
# gateW, gateW_quant, gateA, gateB, gateS,
# upW, upW_quant, upA, upB, upS,