From cd530425f020bf8d1861df634a14214a215868e8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 1 Oct 2024 00:20:01 -0700 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index df36552b4b..5efe610a10 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -42,15 +42,13 @@ IGNORED_TOKENIZER_CHECKING = frozenset(( IGNORED_TOKENIZER_NAMES = [ - # "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", - # "unsloth/Mistral-Nemo-Instruct-2407", - # "mistralai/Mistral-Nemo-Instruct-2407", - # "unsloth/Mistral-Nemo-Base-2407-bnb-4bit", - # "unsloth/Mistral-Nemo-Base-2407", - # "mistralai/Mistral-Nemo-Base-2407", + # Qwen Coder did not train on tool calling. Math did! + "unsloth/Qwen2.5-Coder-1.5B-Instruct", + "unsloth/Qwen2.5-Coder-7B-Instruct", ] IGNORED_TOKENIZER_NAMES = frozenset( - [x.lower() for x in IGNORED_TOKENIZER_NAMES] + [x.lower() for x in IGNORED_TOKENIZER_NAMES] + \ + [x.lower()+"-bnb-4bit" for x in IGNORED_TOKENIZER_NAMES] ) # Check environments