From 3dc5893e12cd0e363e7c7dffcf6e0566304ddff4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 11 Feb 2025 01:22:58 -0800 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 7e4baa60e5..dcdd5c662f 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -946,8 +946,9 @@ def patch_sft_trainer_tokenizer(): " tokenizer = partial(tokenizer, add_special_tokens = False)\n"\ " processing_class = tokenizer\n"\ "else:\n"\ - " add_special_tokens = False if has_bos_token_already else add_special_tokens" - + " add_special_tokens = False if has_bos_token_already else add_special_tokens\n\n" + f"{' '*4}" + check_text = check_text.split("\n") check_text = "\n".join(" "*where + x for x in check_text)