ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update _utils.py

Browse source
This commit is contained in:
Daniel Han 2024-10-27 17:34:33 -07:00
commit 6f19b9aecd
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/_utils.py
View file

@ -507,7 +507,7 @@ def patch_tokenizer(model, tokenizer):
"\u00BF", # Inverted question mark
]
for replacement_char in possible_replacements:
char = tokenizer(replacement_char, add_special_tokens = False)
char = tokenizer(replacement_char, add_special_tokens = False).input_ids
if len(char) == 1:
# Get actual token representation
try: char = tokenizer.convert_ids_to_tokens(char[0])