ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update fast_lora.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-26 17:36:40 +11:00
commit 259097bc61
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
unsloth/kernels/fast_lora.py
View file

@ -178,6 +178,11 @@ pass
def apply_lora_mlp(self, X):
gate = self.gate_proj(X)
up = self. up_proj(X)
h = torch.nn.functional.silu(gate) * up
down = self.down_proj(h)
return down
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self. up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)