From 1065936ecbb8d990dde225d1813d1c8f2ab48603 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Wed, 7 Feb 2024 18:21:36 +1100 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 3cbd8c84a4..8d1ca7b781 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -196,7 +196,7 @@ def LlamaAttention_fast_forward_inference( # pass # Attention - A = torch.matmul(Qn, Knn.transpose(2, 3), out = self.attention[:,:,:,:attention_size]) + A = torch.matmul(Qn, Knn.transpose(2, 3)), #out = self.attention[:,:,:,:attention_size]) A *= self.scalar A[:] = torch.nn.functional.softmax(A, dim = -1, dtype = torch.float32)#.to(A.dtype) A = torch.matmul(A, Vnn, out = Qn)