From 95029e4163d880c6f5bf50cfc5fc4c95b24919e0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 11 Feb 2025 01:25:17 -0800 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index dcdd5c662f..4c57377884 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -947,8 +947,7 @@ def patch_sft_trainer_tokenizer(): " processing_class = tokenizer\n"\ "else:\n"\ " add_special_tokens = False if has_bos_token_already else add_special_tokens\n\n" - f"{' '*4}" - + check_text = check_text.split("\n") check_text = "\n".join(" "*where + x for x in check_text) @@ -961,6 +960,9 @@ def patch_sft_trainer_tokenizer(): ) if len(replacer) == 0: continue replacer = replacer[0] + print("====") + print(check_text) + print("====") function = function.replace(replacer, replacer + check_text) else: function = function.replace(replacer, check_text + replacer)