From 7e1f9118e78213851ea828d241bc4187f768b7fc Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sun, 24 Mar 2024 19:32:54 +1100 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 60c8ceb6fd..9630a0f764 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -76,7 +76,6 @@ pass def get_sorted_dict(dictionary): - # Get sorted dict by values 0, 1, 2, ... sorted_keys = sorted(dictionary.values()) inverted_dictionary = { value : key for key, value in dictionary.items() } @@ -164,7 +163,9 @@ def convert_to_fast_tokenizer( # Now load it! fast_tokenizer = AutoTokenizer.from_pretrained(new_location) - return fast_tokenizer + if assert_same_tokenization(slow_tokenizer, fast_tokenizer): + return fast_tokenizer + return slow_tokenizer pass