From abc47836dcac59f90d818b5905a30e62937eb84d Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Thu, 1 Feb 2024 23:41:32 +1100 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 9d267bde6c..feb82be1bb 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -268,8 +268,8 @@ def LlamaAttention_fast_forward_inference( self.paged_attention_K[:,:,:kv_seq_len,:] = K1 self.paged_attention_V[:,:,:kv_seq_len,:] = V1 pass - Kn = self.paged_attention_K[:,:,seq_len,:] = K1 - Vn = self.paged_attention_V[:,:,seq_len,:] = V1 + Kn = self.paged_attention_K[:,:,seq_len,:] = Kn + Vn = self.paged_attention_V[:,:,seq_len,:] = Vn # Grouped query attention if n_groups != 1: