From 01b312cc5fae9874a72cc1f95b6c4695fca0f128 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 11 Feb 2025 00:36:12 -0800 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 78494f8efa..5f904ad7dd 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -917,7 +917,7 @@ def patch_sft_trainer_tokenizer(): return all_imports = dir(trl.trainer.sft_trainer) - for function_name, replacer in ( + for (function_name, replacer,) in ( ("_prepare_non_packed_dataloader", "def tokenize(element):",), ("_prepare_dataset", None,), # ("_prepare_packed_dataloader", "if dataset_text_field is not None",), @@ -962,12 +962,12 @@ def patch_sft_trainer_tokenizer(): else: function = function.replace(replacer, check_text + replacer) pass + print(function) x = [x for x in all_imports if x in function] exec(f"from trl.trainer.sft_trainer import ({','.join(x)})", locals()) exec(function, locals(), globals()) exec(f"trl.trainer.sft_trainer.SFTTrainer.{function_name} = {function_name}", globals()) - print("Patched") pass # Patch train with fix_untrained_tokens