ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update llama.py

Browse source
This commit is contained in:
Daniel Han 2025-09-03 19:11:53 -07:00
commit 1f7fe4631d
1 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/models/llama.py
View file

@ -162,6 +162,10 @@ def _fast_prepare_inputs_for_generation(self, input_ids, attention_mask=None, **
if len(past_key_values) == 0:
past_key_values = None
kwargs["past_key_values"] = None
# New since 4.56
elif hasattr(past_key_values, "get_seq_length") and past_key_values.get_seq_length() == 0:
past_key_values = None
kwargs["past_key_values"] = None
else:
bs, cache_length = input_ids.shape
input_ids = input_ids[:,[-1]]
@ -1833,7 +1837,7 @@ class FastLlamaModel:
# Solves https://github.com/unslothai/unsloth/issues/168
# Static KV Cache was introduced in 4.38.0, causing training to be much slower.
# Inferene can now be CUDAGraphed, but we shall retain the old rotary embeddings.
# Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
# https://github.com/huggingface/transformers/pull/27931
# https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
import transformers.models.llama.modeling_llama