From 6ae8ef7fdff357d0e5fc57d1568d9e3495e2983b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 11 Feb 2025 00:42:47 -0800 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 5f904ad7dd..c35d990d00 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -918,7 +918,8 @@ def patch_sft_trainer_tokenizer(): all_imports = dir(trl.trainer.sft_trainer) for (function_name, replacer,) in ( - ("_prepare_non_packed_dataloader", "def tokenize(element):",), + # ("_prepare_non_packed_dataloader", "def tokenize(element):",), + ("_prepare_non_packed_dataloader", None,), ("_prepare_dataset", None,), # ("_prepare_packed_dataloader", "if dataset_text_field is not None",), ):