From e41cc4093c70095e4aef390c8afae85c38aa4eb3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 20 Jul 2024 13:25:59 -0700 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 0469f4d61b..8474c2c6b7 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -42,12 +42,12 @@ IGNORED_TOKENIZER_CHECKING = frozenset(( IGNORED_TOKENIZER_NAMES = [ - "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", - "unsloth/Mistral-Nemo-Instruct-2407", - "mistralai/Mistral-Nemo-Instruct-2407", - "unsloth/Mistral-Nemo-Base-2407-bnb-4bit", - "unsloth/Mistral-Nemo-Base-2407", - "mistralai/Mistral-Nemo-Base-2407", + # "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", + # "unsloth/Mistral-Nemo-Instruct-2407", + # "mistralai/Mistral-Nemo-Instruct-2407", + # "unsloth/Mistral-Nemo-Base-2407-bnb-4bit", + # "unsloth/Mistral-Nemo-Base-2407", + # "mistralai/Mistral-Nemo-Base-2407", ] IGNORED_TOKENIZER_NAMES = frozenset( [x.lower() for x in IGNORED_TOKENIZER_NAMES]