diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 7b88b0932f..07cd874125 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -41,14 +41,17 @@ IGNORED_TOKENIZER_CHECKING = frozenset(( )) -IGNORED_TOKENIZER_NAMES = frozenset(( +IGNORED_TOKENIZER_NAMES = [ "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", "unsloth/Mistral-Nemo-Instruct-2407", "mistralai/Mistral-Nemo-Instruct-2407", "unsloth/Mistral-Nemo-Base-2407-bnb-4bit", "unsloth/Mistral-Nemo-Base-2407", "mistralai/Mistral-Nemo-Base-2407", -)) +] +IGNORED_TOKENIZER_NAMES = frozenset( + [x.lower() for x in IGNORED_TOKENIZER_NAMES] +) # Check environments keynames = "\n" + "\n".join(os.environ.keys())