From 13d59174158a6baffd93510700758fce491e8a29 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 24 May 2024 11:21:29 +1000 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 1cbe49b7ae..a03491669f 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -719,9 +719,9 @@ from inspect import getsource import trl.trainer.sft_trainer from trl.trainer.sft_trainer import * -def fix_sft_trainer_tokenizer(): +def patch_sft_trainer_tokenizer(): """ - Fixes double adding BOS tokens like in llama-3 + Patches the trainer with changes """ for function_name, replacer in ( ("_prepare_non_packed_dataloader", "def tokenize(element):",), @@ -747,10 +747,8 @@ def fix_sft_trainer_tokenizer(): function = function.replace(replacer, check_text + replacer) exec(function, globals()) - # Replace TRL's SFTTrainer exec(f"trl.trainer.sft_trainer.SFTTrainer.{function_name} = {function_name}", globals()) pass pass -# Fixes double adding BOS tokens like in llama-3 -fix_sft_trainer_tokenizer() +patch_sft_trainer_tokenizer()