diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index d61dbf842b..a51860a14c 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -141,7 +141,7 @@ def LlamaAttention_fast_forward_inference( self.paged_attention_V = self.paged_attention[:,1] self.attention.resize_((bsz, n_heads, 1, self.attention.shape[-1]+KV_CACHE_INCREMENT)) pass - + Qn = fast_linear_forward(self.q_proj, Xn, out = self.temp_QA[0]) Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0]) Vn = fast_linear_forward(self.v_proj, Xn, out = self.temp_KV[1]) @@ -197,7 +197,6 @@ def fast_mlp_inference(self, X): # gate = self.gate_proj(X) # up = self.up_proj(X) bsz, _, hd = X.shape - mlp_size = self.config.intermediate_size temp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = "cuda") gate = fast_linear_forward(self.gate_proj, X, out = temp[0])