ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2024-10-01 00:20:01 -07:00
commit cd530425f0
1 changed files with 5 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

12
unsloth/tokenizer_utils.py
View file

@ -42,15 +42,13 @@ IGNORED_TOKENIZER_CHECKING = frozenset((
IGNORED_TOKENIZER_NAMES = [
# "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit",
# "unsloth/Mistral-Nemo-Instruct-2407",
# "mistralai/Mistral-Nemo-Instruct-2407",
# "unsloth/Mistral-Nemo-Base-2407-bnb-4bit",
# "unsloth/Mistral-Nemo-Base-2407",
# "mistralai/Mistral-Nemo-Base-2407",
# Qwen Coder did not train on tool calling. Math did!
"unsloth/Qwen2.5-Coder-1.5B-Instruct",
"unsloth/Qwen2.5-Coder-7B-Instruct",
]
IGNORED_TOKENIZER_NAMES = frozenset(
[x.lower() for x in IGNORED_TOKENIZER_NAMES]
[x.lower() for x in IGNORED_TOKENIZER_NAMES] + \
[x.lower()+"-bnb-4bit" for x in IGNORED_TOKENIZER_NAMES]
)
# Check environments