From aed494a1ca152ae71ee460ad9c2b4d391239e23d Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 19 Apr 2024 03:05:54 +1000 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index edd9717557..fa536ef29a 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -299,9 +299,9 @@ def load_correct_tokenizer( ) if slow_tokenizer is not None: - if hasattr(fast_tokenizer "add_bos_token") and hasattr(slow_tokenizer "add_bos_token"): + if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"): fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token - if hasattr(fast_tokenizer "add_eos_token") and hasattr(slow_tokenizer "add_eos_token"): + if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"): fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token # Confirm if slow and fast are equivalent!