From 58d1f1e03c65e7dccf42b3d52826a453e4349043 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 9 Mar 2024 20:05:52 +1100 Subject: [PATCH] Update chat_templates.py --- unsloth/chat_templates.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 9675b10fe4..715cf6b3c2 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -298,8 +298,12 @@ def get_chat_template( pass pass - logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.") - string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word) + if not stop_word in token_mapping.values(): + # Do not map 107 = <|im_end|> and 1 = <|im_end|>. This will reduce the vocab size by 1 + logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.") + string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word) + pass + new_tokenizer = tokenizer._tokenizer.from_str(string_vocab) tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word)