From 55bf35be5d07d9125dfd8ee37512a48ccb79dbb4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 7 Jul 2024 15:46:36 -0700 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 64bc6f84f0..7221a19a2a 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1037,9 +1037,9 @@ def _wrap_fast_inference(generate, device_type, dtype, model): kwargs["cache_implementation"] = "dynamic" # Set pad token - old_pad_token_id = getattr(model.config, "pad_token_id", None) - old_eos_token_id = getattr(model.config, "eos_token_id", None) - model.config.pad_token_id = old_eos_token_id + # old_pad_token_id = getattr(model.config, "pad_token_id", None) + # old_eos_token_id = getattr(model.config, "eos_token_id", None) + # model.config.pad_token_id = old_eos_token_id # Autocasted with torch.autocast(device_type = device_type, dtype = dtype): @@ -1047,7 +1047,7 @@ def _wrap_fast_inference(generate, device_type, dtype, model): pass # Revert - model.config.pad_token_id = old_pad_token_id + # model.config.pad_token_id = old_pad_token_id # Unset a flag for generation! internal_model = model