From 8fc66addb9999bd95f8649ace0aa3d4c44c903d0 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Tue, 26 Mar 2024 04:35:07 +1100 Subject: [PATCH] Update chat_templates.py --- unsloth/chat_templates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 4c27eacdd4..1bde2884d1 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -322,7 +322,6 @@ def get_chat_template( if map_eos_token and (not stop_word in token_mapping.values()): # Do not map 107 = <|im_end|> and 1 = <|im_end|>. This will reduce the vocab size by 1 logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.") - print(1) string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word) pass @@ -341,7 +340,6 @@ def get_chat_template( pass elif map_eos_token and (stop_word != "eos_token"): - print(2) logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.") # Replaces the old EOS token with a new one. @@ -354,6 +352,9 @@ def get_chat_template( string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word) new_tokenizer = tokenizer._tokenizer.from_str(string_vocab) tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word) + + # Must fix the sentence piece tokenizer since there's no tokenizer.model file! + tokenizer = fix_sentencepiece_tokenizer(tokenizer, new_tokenizer, token_mapping,) pass else: