ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2024-07-20 13:25:59 -07:00
commit e41cc4093c
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
unsloth/tokenizer_utils.py
View file

@ -42,12 +42,12 @@ IGNORED_TOKENIZER_CHECKING = frozenset((
IGNORED_TOKENIZER_NAMES = [
"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit",
"unsloth/Mistral-Nemo-Instruct-2407",
"mistralai/Mistral-Nemo-Instruct-2407",
"unsloth/Mistral-Nemo-Base-2407-bnb-4bit",
"unsloth/Mistral-Nemo-Base-2407",
"mistralai/Mistral-Nemo-Base-2407",
# "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit",
# "unsloth/Mistral-Nemo-Instruct-2407",
# "mistralai/Mistral-Nemo-Instruct-2407",
# "unsloth/Mistral-Nemo-Base-2407-bnb-4bit",
# "unsloth/Mistral-Nemo-Base-2407",
# "mistralai/Mistral-Nemo-Base-2407",
]
IGNORED_TOKENIZER_NAMES = frozenset(
[x.lower() for x in IGNORED_TOKENIZER_NAMES]