ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: preserve chat_template BOS check when test_text is None

Browse source 
The has_bos_token_already guard wrapped both test_text.startswith()
and bos_token in chat_template with test_text is not None, which
disabled the chat_template BOS detection for conversational datasets
where test_text is set to None.

Split the guard so test_text is not None only applies to the
startswith() call, while bos_token in chat_template is always checked.
This commit is contained in:
Daniel Han 2026-03-24 10:01:05 +00:00
commit 40bd8b8917
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/rl_replacements.py
View file

@ -164,7 +164,7 @@ def sft_trainer_prepare_dataset(function_name, function):
" test_text = None\n"
"chat_template = getattr(tokenizer, 'chat_template', None)\n"
"chat_template = '' if chat_template is None else chat_template\n"
"has_bos_token_already = (test_text is not None and (test_text.startswith(tokenizer.bos_token) or tokenizer.bos_token in chat_template)) "
"has_bos_token_already = ((test_text is not None and test_text.startswith(tokenizer.bos_token)) or tokenizer.bos_token in chat_template) "
"if getattr(tokenizer, 'bos_token', None) is not None else False\n"
"if 'add_special_tokens' not in locals() and has_bos_token_already:\n"
" from functools import partial\n"

2
unsloth/tokenizer_utils.py
View file

@ -985,7 +985,7 @@ def patch_sft_trainer_tokenizer():
" test_text = None\n"
"chat_template = getattr(tokenizer, 'chat_template', None)\n"
"chat_template = '' if chat_template is None else chat_template\n"
"has_bos_token_already = (test_text is not None and (test_text.startswith(tokenizer.bos_token) or tokenizer.bos_token in chat_template)) "
"has_bos_token_already = ((test_text is not None and test_text.startswith(tokenizer.bos_token)) or tokenizer.bos_token in chat_template) "
"if getattr(tokenizer, 'bos_token', None) is not None else False\n"
"if 'add_special_tokens' not in locals() and has_bos_token_already:\n"
" from functools import partial\n"