ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix llama tokenizer padding_side when using model.generate in inference mode (#3644)

Browse source 
* Only restore training mode after generation, if the model started out in training mode

Signed-off-by: Dina Suehiro Jones <dina.s.jones@intel.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: Dina Suehiro Jones <dina.s.jones@intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Dina Suehiro Jones 2025-11-25 17:33:28 -08:00 • committed by GitHub
commit 8d72323bd8
2 changed files with 15 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/models/llama.py
View file

@ -1988,6 +1988,9 @@ def unsloth_fast_generate(
*args,
**kwargs,
):
# If the model starts out in training mode, restore training mode after generation
restore_training_mode = self.training
FastLlamaModel.for_inference(self)
dtype = _get_dtype(dtype_from_config(self.config))
@ -2043,7 +2046,8 @@ def unsloth_fast_generate(
# accelerate.utils.operations.send_to_device = accelerate_old_send_to_device
# pass
FastLlamaModel.for_training(self)
if restore_training_mode:
FastLlamaModel.for_training(self)
return output

14
unsloth/models/loader.py
View file

@ -258,7 +258,7 @@ class FastLanguageModel(FastLlamaModel):
if model_name.lower().endswith("-bf16"):
load_in_4bit = False
load_in_8bit = False
load_in_fp8 = False
load_in_fp8 = False
load_in_16bit = True
if USE_MODELSCOPE and not os.path.exists(model_name):
@ -387,7 +387,7 @@ class FastLanguageModel(FastLlamaModel):
if model_name.lower().endswith("-bf16"):
load_in_4bit = False
load_in_8bit = False
load_in_fp8 = False
load_in_fp8 = False
load_in_16bit = True
model_config = AutoConfig.from_pretrained(
@ -711,10 +711,16 @@ class FastModel(FastBaseModel):
)
load_in_4bit = False
load_in_8bit = False
load_in_fp8 = False
load_in_fp8 = False
load_in_16bit = False
if int(load_in_4bit) + int(load_in_8bit) + int(load_in_16bit) + int(load_in_fp8 != False) >= 2:
if (
int(load_in_4bit)
+ int(load_in_8bit)
+ int(load_in_16bit)
+ int(load_in_fp8 != False)
>= 2
):
raise RuntimeError(
"Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!\n"
"Also, we by default set `load_in_4bit = True`.\n"