ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update chat_templates.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-02-14 17:31:33 +11:00
commit d40a12852e
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unsloth/chat_templates.py
View file

@ -372,5 +372,6 @@ def test_chat_templates():
correct_tokenizer = AutoTokenizer.from_pretrained("lmsys/vicuna-7b-v1.5")
correct_tokenizer.chat_template = template
our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)
assert(correct_prompt == our_prompt)
# We add </s> ourselves
assert(correct_prompt == our_prompt.replace("</s>", ""))
pass