From 72de37e3214c6b013c0ea6dc4557a9a9f322b25a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 19:15:40 -0700 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index dd6d805ec4..24f9942119 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1392,17 +1392,17 @@ def _wrap_fast_inference(generate, device_type, dtype, model): # For newer HF # kwargs["cache_implementation"] = "dynamic" # For num_logits_to_keep - kwargs["num_logits_to_keep"] = 1 + # kwargs["num_logits_to_keep"] = 1 - # Remove token_type_ids - kwargs.pop("token_type_ids", None) + # # Remove token_type_ids + # kwargs.pop("token_type_ids", None) - # Check pad_token - model_eos_token_id = getattr(model.config, "eos_token_id", None) - if model_eos_token_id is not None and hasattr(model_eos_token_id, "__iter__"): - model_eos_token_id = model_eos_token_id[0] + # # Check pad_token + # model_eos_token_id = getattr(model.config, "eos_token_id", None) + # if model_eos_token_id is not None and hasattr(model_eos_token_id, "__iter__"): + # model_eos_token_id = model_eos_token_id[0] - kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id) + # kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id) # Set pad token # old_pad_token_id = getattr(model.config, "pad_token_id", None)