ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update fast_lora.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-26 18:04:59 +11:00
commit 4bfbccec4b
1 changed files with 2 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

7
unsloth/kernels/fast_lora.py
View file

@ -178,11 +178,8 @@ pass
def apply_lora_mlp(self, X):
gate = self.gate_proj(X)
up = self. up_proj(X)
h = torch.nn.functional.silu(gate) * up
down = self.down_proj(h)
return down
down_proj = self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
return down_proj
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self. up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)