From fc6ca8dd1afc618cede19f27df62fc6252a9f0e1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 14 Mar 2025 08:17:36 -0700 Subject: [PATCH] Update vision.py --- unsloth/models/vision.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 1404be8b0f..219aba8cb6 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -314,15 +314,16 @@ class FastBaseModel: __tokenizer = tokenizer.tokenizer # Add padding side as well __tokenizer.padding_side = "right" - # Check bos, eos, pad, unk tokens - tokens = ["bos_token", "eos_token", "pad_token", "unk_token"] - for token in tokens: - if hasattr(__tokenizer, token) and not hasattr(tokenizer, token): - _args = {"__tokenizer" : __tokenizer, "tokenizer" : tokenizer} - exec(f"tokenizer.{token} = __tokenizer.{token}", _args) - exec(f"tokenizer.{token}_id = __tokenizer.{token}_id", _args) - pass - pass + # Check bos, eos, pad tokens + if hasattr(tokenizer, "bos_token"): + tokenizer.bos_token = tokenizer.tokenizer.bos_token + tokenizer.bos_token_id = tokenizer.tokenizer.bos_token_id + if hasattr(tokenizer, "eos_token"): + tokenizer.eos_token = tokenizer.tokenizer.eos_token + tokenizer.eos_token_id = tokenizer.tokenizer.eos_token_id + if hasattr(tokenizer, "pad_token"): + tokenizer.pad_token = tokenizer.tokenizer.pad_token + tokenizer.pad_token_id = tokenizer.tokenizer.pad_token_id pass model, tokenizer = patch_tokenizer(model, tokenizer) model = post_patch_loss_function(model)