ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update fast_lora.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-27 03:58:21 +11:00
commit 7fb64e0e31
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
unsloth/kernels/fast_lora.py
View file

@ -236,11 +236,11 @@ class LoRA_MLP_New(torch.autograd.Function):
dtype = X.dtype
DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS)
df = DW * g # 88us
dg = DW * f # 88us
de = cls._silu_backward(df, e) # 90us
df = DW * f # 88us
dg = DW * g # 88us
de = cls._silu_backward(dg, e) # 90us
dX = matmul_lora(dg, upW.t(), upW_quant, upB, upA, upS)
dX = matmul_lora(df, upW.t(), upW_quant, upB, upA, upS)
dX += matmul_lora(de, gateW.t(), gateW_quant, gateB, gateA, gateS)
# Down projection LoRA weights