From f18ac214cd5a15fcfbf310ddd9dfce5d66fd3731 Mon Sep 17 00:00:00 2001 From: Erland366 Date: Wed, 23 Oct 2024 17:32:19 +0400 Subject: [PATCH] fix: correct tokenizer handling in patch_sft_trainer_tokenizer --- unsloth/tokenizer_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 1cad00d44d..788c5b9792 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -909,13 +909,13 @@ def patch_sft_trainer_tokenizer(): ): function = getsource(eval(f"trl.trainer.sft_trainer.SFTTrainer.{function_name}")) where = function.find("def") - function = function.split("\n") - function = "\n".join(x[where:] for x in function) + function_lines = function.split("\n") + function = "\n".join(x[where:] for x in function_lines) check_text = \ "\n"\ "if 'tokenizer' not in locals(): tokenizer = processing_class\n"\ - "test_text = dataset[0][dataset_text_field] if (formatting_func is not None and dataset_text_field is None) else formatting_func(dataset[0])[0]\n"\ + "test_text = dataset[0][dataset_text_field] if (formatting_func is None and dataset_text_field is not None) else formatting_func(dataset[0])[0]\n"\ "chat_template = getattr(tokenizer, 'chat_template', None)\n"\ "chat_template = '' if chat_template is None else chat_template\n"\ "has_bos_token_already = (test_text.startswith(tokenizer.bos_token) or tokenizer.bos_token in chat_template) "\ @@ -928,7 +928,8 @@ def patch_sft_trainer_tokenizer(): function = function.replace(replacer, check_text + replacer) exec(function, globals()) - exec(f"trl.trainer.sft_trainer.SFTTrainer.{function_name} = {function_name}", globals()) + modified_function = eval(function_name) + setattr(trl.trainer.sft_trainer.SFTTrainer, function_name, modified_function) pass # Patch train with fix_untrained_tokens