From 09e1fb76722b9cbc49d94b6d41f814658e1ad23f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 11 Feb 2025 01:30:02 -0800 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 2062df4808..5226c3c5b7 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -951,7 +951,7 @@ def patch_sft_trainer_tokenizer(): check_text = check_text.split("\n") check_text = "\n".join(" "*where + x for x in check_text) check_text = check_text.rstrip() + "\n" - + if replacer is None: # .*? matches first match. .+? matches final match. replacer = re.findall( @@ -961,14 +961,10 @@ def patch_sft_trainer_tokenizer(): ) if len(replacer) == 0: continue replacer = replacer[0] - print("====") - print(replacer) - print("====") function = function.replace(replacer, replacer + check_text) else: function = function.replace(replacer, check_text + replacer) pass - print(function) x = [x for x in all_imports if x in function] exec(f"from trl.trainer.sft_trainer import ({','.join(x)})", locals())