ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update chat_templates.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-03-26 04:35:07 +11:00
commit 8fc66addb9
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
unsloth/chat_templates.py
View file

@ -322,7 +322,6 @@ def get_chat_template(
if map_eos_token and (not stop_word in token_mapping.values()):
# Do not map 107 = <|im_end|> and 1 = <|im_end|>. This will reduce the vocab size by 1
logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")
print(1)
string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word)
pass
@ -341,7 +340,6 @@ def get_chat_template(
pass
elif map_eos_token and (stop_word != "eos_token"):
print(2)
logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")
# Replaces the old EOS token with a new one.
@ -354,6 +352,9 @@ def get_chat_template(
string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word)
new_tokenizer = tokenizer._tokenizer.from_str(string_vocab)
tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word)
# Must fix the sentence piece tokenizer since there's no tokenizer.model file!
tokenizer = fix_sentencepiece_tokenizer(tokenizer, new_tokenizer, token_mapping,)
pass
else: