From 6786fa425fe1633140927627c20e9c3052c4fc2e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 19 Jul 2024 01:24:46 -0700 Subject: [PATCH] Nightly (#783) * Update __init__.py * dynamic RoPE * Update mistral.py * Update llama.py * Update tokenizer_utils.py * Update mistral.py * Update llama.py * Update __init__.py * Update flex_attention.py * Update llama.py * Update llama.py * Mistral Nemo * Update tokenizer_utils.py * Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index e0c89e4516..07cd874125 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -41,14 +41,17 @@ IGNORED_TOKENIZER_CHECKING = frozenset(( )) -IGNORED_TOKENIZER_NAMES = frozenset(( +IGNORED_TOKENIZER_NAMES = [ "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", "unsloth/Mistral-Nemo-Instruct-2407", "mistralai/Mistral-Nemo-Instruct-2407", "unsloth/Mistral-Nemo-Base-2407-bnb-4bit", "unsloth/Mistral-Nemo-Base-2407", "mistralai/Mistral-Nemo-Base-2407", -)) +] +IGNORED_TOKENIZER_NAMES = frozenset( + [x.lower() for x in IGNORED_TOKENIZER_NAMES] +) # Check environments keynames = "\n" + "\n".join(os.environ.keys()) @@ -498,7 +501,8 @@ def load_correct_tokenizer( cache_dir = cache_dir, ) - if tokenizer_name in IGNORED_TOKENIZER_NAMES: pass + if tokenizer_name in IGNORED_TOKENIZER_NAMES: + return fast_tokenizer elif slow_tokenizer is not None: if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"): fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token