From 9b464716c7148affce4b1e6bf513d416fc70b7cc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 19 Jul 2024 01:03:51 -0700 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index e0c89e4516..7b88b0932f 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -498,7 +498,8 @@ def load_correct_tokenizer( cache_dir = cache_dir, ) - if tokenizer_name in IGNORED_TOKENIZER_NAMES: pass + if tokenizer_name in IGNORED_TOKENIZER_NAMES: + return fast_tokenizer elif slow_tokenizer is not None: if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"): fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token