From 1623c3242bb0892cef28fdfbd2b865c9a18ede79 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 22 Oct 2024 01:28:38 -0700 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 6e9c061fad..d7af61a090 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -914,6 +914,7 @@ def patch_sft_trainer_tokenizer(): check_text = \ "\n"\ + "if 'tokenizer' not in locals(): tokenizer = processing_class\n"\ "test_text = dataset[0][dataset_text_field] if (formatting_func is not None and dataset_text_field is None) else formatting_func(dataset[0])[0]\n"\ "chat_template = getattr(tokenizer, 'chat_template', None)\n"\ "chat_template = '' if chat_template is None else chat_template\n"\