diff --git a/tests/test_past_kv_models.py b/tests/test_past_kv_models.py index bd4aeac1fe..c2ff2a37f2 100644 --- a/tests/test_past_kv_models.py +++ b/tests/test_past_kv_models.py @@ -24,6 +24,7 @@ def _load_model(model_name, load_in_4bit = True): """Load model, raising SkipTest if the model cannot be loaded.""" try: from unsloth import FastLanguageModel + model, tokenizer = FastLanguageModel.from_pretrained( model_name = model_name, max_seq_length = 2048, diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 2c6e0eb4eb..23fccc9fa0 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -227,10 +227,10 @@ def _slice_position_ids(position_ids, input_ids): return None if position_ids.dim() == 2: if position_ids.shape[1] > input_ids.shape[1]: - position_ids = position_ids[:, -input_ids.shape[1]:] + position_ids = position_ids[:, -input_ids.shape[1] :] elif position_ids.dim() == 1: if position_ids.shape[0] > input_ids.shape[1]: - position_ids = position_ids[-input_ids.shape[1]:] + position_ids = position_ids[-input_ids.shape[1] :] return position_ids @@ -1492,7 +1492,11 @@ def CausalLM_fast_forward(fast_forward_inference): attention_mask = attention_mask, **kwargs, ) - elif past_key_values is not None and input_ids is not None and input_ids.shape[1] > 1: + elif ( + past_key_values is not None + and input_ids is not None + and input_ids.shape[1] > 1 + ): # Multi-token prefill with user-provided KV cache. The fast inference # path only supports single-token decoding (q_len == 1), so fall # through to the regular model forward which handles arbitrary lengths. diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 1166685a6a..854a07c23e 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -265,7 +265,9 @@ def MistralForCausalLM_fast_forward( position_ids = position_ids, attention_mask = attention_mask, ) - elif past_key_values is not None and input_ids is not None and input_ids.shape[1] > 1: + elif ( + past_key_values is not None and input_ids is not None and input_ids.shape[1] > 1 + ): # Multi-token prefill with user-provided KV cache self.model._has_no_labels = labels is None outputs = self.model(