From 259097bc61dc684557b66d7100155dc55d6628e9 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 17:36:40 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 7bf8e1f75f..418a6bfd64 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -178,6 +178,11 @@ pass def apply_lora_mlp(self, X): + gate = self.gate_proj(X) + up = self. up_proj(X) + h = torch.nn.functional.silu(gate) * up + down = self.down_proj(h) + return down gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj) upW, upW_quant, upA, upB, upS = get_lora_parameters(self. up_proj) downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)