diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 0469f4d61b..8474c2c6b7 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -42,12 +42,12 @@ IGNORED_TOKENIZER_CHECKING = frozenset(( IGNORED_TOKENIZER_NAMES = [ - "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", - "unsloth/Mistral-Nemo-Instruct-2407", - "mistralai/Mistral-Nemo-Instruct-2407", - "unsloth/Mistral-Nemo-Base-2407-bnb-4bit", - "unsloth/Mistral-Nemo-Base-2407", - "mistralai/Mistral-Nemo-Base-2407", + # "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", + # "unsloth/Mistral-Nemo-Instruct-2407", + # "mistralai/Mistral-Nemo-Instruct-2407", + # "unsloth/Mistral-Nemo-Base-2407-bnb-4bit", + # "unsloth/Mistral-Nemo-Base-2407", + # "mistralai/Mistral-Nemo-Base-2407", ] IGNORED_TOKENIZER_NAMES = frozenset( [x.lower() for x in IGNORED_TOKENIZER_NAMES]