ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update vision.py

Browse source
This commit is contained in:
Daniel Han 2025-03-14 08:17:36 -07:00
commit fc6ca8dd1a
1 changed files with 10 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

19
unsloth/models/vision.py
View file

@ -314,15 +314,16 @@ class FastBaseModel:
__tokenizer = tokenizer.tokenizer
# Add padding side as well
__tokenizer.padding_side = "right"
# Check bos, eos, pad, unk tokens
tokens = ["bos_token", "eos_token", "pad_token", "unk_token"]
for token in tokens:
if hasattr(__tokenizer, token) and not hasattr(tokenizer, token):
_args = {"__tokenizer" : __tokenizer, "tokenizer" : tokenizer}
exec(f"tokenizer.{token} = __tokenizer.{token}", _args)
exec(f"tokenizer.{token}_id = __tokenizer.{token}_id", _args)
pass
pass
# Check bos, eos, pad tokens
if hasattr(tokenizer, "bos_token"):
tokenizer.bos_token = tokenizer.tokenizer.bos_token
tokenizer.bos_token_id = tokenizer.tokenizer.bos_token_id
if hasattr(tokenizer, "eos_token"):
tokenizer.eos_token = tokenizer.tokenizer.eos_token
tokenizer.eos_token_id = tokenizer.tokenizer.eos_token_id
if hasattr(tokenizer, "pad_token"):
tokenizer.pad_token = tokenizer.tokenizer.pad_token
tokenizer.pad_token_id = tokenizer.tokenizer.pad_token_id
pass
model, tokenizer = patch_tokenizer(model, tokenizer)
model = post_patch_loss_function(model)