From 1f7fe4631dd3aecf1990c0abc339018f7ee95501 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 3 Sep 2025 19:11:53 -0700 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 68c67c6ce8..cf2ca75f75 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -162,6 +162,10 @@ def _fast_prepare_inputs_for_generation(self, input_ids, attention_mask=None, ** if len(past_key_values) == 0: past_key_values = None kwargs["past_key_values"] = None + # New since 4.56 + elif hasattr(past_key_values, "get_seq_length") and past_key_values.get_seq_length() == 0: + past_key_values = None + kwargs["past_key_values"] = None else: bs, cache_length = input_ids.shape input_ids = input_ids[:,[-1]] @@ -1833,7 +1837,7 @@ class FastLlamaModel: # Solves https://github.com/unslothai/unsloth/issues/168 # Static KV Cache was introduced in 4.38.0, causing training to be much slower. - # Inferene can now be CUDAGraphed, but we shall retain the old rotary embeddings. + # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings. # https://github.com/huggingface/transformers/pull/27931 # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py import transformers.models.llama.modeling_llama