ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: correct tokenizer handling in patch_sft_trainer_tokenizer

Browse source
This commit is contained in:
Erland366 2024-10-23 17:32:19 +04:00
commit f18ac214cd
1 changed files with 5 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
unsloth/tokenizer_utils.py
View file

@ -909,13 +909,13 @@ def patch_sft_trainer_tokenizer():
):
function = getsource(eval(f"trl.trainer.sft_trainer.SFTTrainer.{function_name}"))
where = function.find("def")
function = function.split("\n")
function = "\n".join(x[where:] for x in function)
function_lines = function.split("\n")
function = "\n".join(x[where:] for x in function_lines)
check_text = \
"\n"\
"if 'tokenizer' not in locals(): tokenizer = processing_class\n"\
"test_text = dataset[0][dataset_text_field] if (formatting_func is not None and dataset_text_field is None) else formatting_func(dataset[0])[0]\n"\
"test_text = dataset[0][dataset_text_field] if (formatting_func is None and dataset_text_field is not None) else formatting_func(dataset[0])[0]\n"\
"chat_template = getattr(tokenizer, 'chat_template', None)\n"\
"chat_template = '' if chat_template is None else chat_template\n"\
"has_bos_token_already = (test_text.startswith(tokenizer.bos_token) or tokenizer.bos_token in chat_template) "\
@ -928,7 +928,8 @@ def patch_sft_trainer_tokenizer():
function = function.replace(replacer, check_text + replacer)
exec(function, globals())
exec(f"trl.trainer.sft_trainer.SFTTrainer.{function_name} = {function_name}", globals())
modified_function = eval(function_name)
setattr(trl.trainer.sft_trainer.SFTTrainer, function_name, modified_function)
pass
# Patch train with fix_untrained_tokens