From 085a8e944adb019b167d4372260125ced48e85ac Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Tue, 23 Jan 2024 00:32:57 +1100 Subject: [PATCH] inference --- unsloth/kernels/utils.py | 1 - unsloth/models/llama.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 26d0f6bb4d..141814c332 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -180,7 +180,6 @@ pass def fast_linear_forward(proj, X, temp_lora = None, out = None): W, W_quant, lora_A, lora_B, lora_S = get_lora_parameters(proj) out = fast_gemv(X, W, W_quant, out = out) - print(X.shape, W.quant_state.shape, out.shape) if lora_A is not None: dtype = X.dtype temp_lora = torch.matmul(X, lora_A.to(dtype).t(), out = temp_lora) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 8dfc520327..cba5365233 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -172,7 +172,7 @@ def fast_mlp_inference(self, X): gate *= up # X = self.down_proj(gate) - down = fast_linear_forward(self.down_proj, X) + down = fast_linear_forward(self.down_proj, gate) X = down.view(1, 1, self.hidden_size) return X