ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update chat_templates.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-03-09 20:05:52 +11:00
commit 58d1f1e03c
1 changed files with 6 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

8
unsloth/chat_templates.py
View file

@ -298,8 +298,12 @@ def get_chat_template(
pass
pass
logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")
string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word)
if not stop_word in token_mapping.values():
# Do not map 107 = <|im_end|> and 1 = <|im_end|>. This will reduce the vocab size by 1
logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")
string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word)
pass
new_tokenizer = tokenizer._tokenizer.from_str(string_vocab)
tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word)