From 97658f9a74aa1259ae6c0a879c5fa1e7ca418a24 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 26 Jan 2024 19:12:08 +1100 Subject: [PATCH] Update fast_lora.py --- unsloth/kernels/fast_lora.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 1fd9704266..6d1f65b4ee 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -35,7 +35,7 @@ def matmul_lora(X, W, W_quant, A, B, s, out = None): if A is not None: # LoRA is enabled A, B = A.t(), B.t() - out += (X @ A.to(dtype)) @ (s * B.to(dtype)) + out += s * (X @ A.to(dtype)) @ (B.to(dtype)) pass return out.view(batch, seq_len, -1) if reshape else out @@ -178,12 +178,12 @@ pass from transformers.models.llama.modeling_llama import logger def apply_lora_mlp(self, X): - logger.warning_once("Hello!") - gate = self.gate_proj(X) - up = self. up_proj(X) - h = torch.nn.functional.silu(gate) * up - down = self.down_proj(h) - return down + logger.warning_once("Hello!2") + # gate = self.gate_proj(X) + # up = self. up_proj(X) + # h = torch.nn.functional.silu(gate) * up + # down = self.down_proj(h) + # return down gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj) upW, upW_quant, upA, upB, upS = get_lora_parameters(self. up_proj) downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)