ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update chat_templates.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-03-15 03:37:37 +11:00
commit 5dfe582c96
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/chat_templates.py
View file

@ -371,7 +371,7 @@ def get_chat_template(
new_tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word)
# Must fix the sentence piece tokenizer since there's no tokenizer.model file!
new_tokenizer = fix_sentencepiece_tokenizer(tokenizer, new_tokenizer, token_mapping,)
tokenizer = fix_sentencepiece_tokenizer(tokenizer, new_tokenizer, token_mapping,)
elif stop_word != "eos_token":
logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")