From b5e0255eabfabfd89651663c49acbcb5a4c4c5a4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 9 Nov 2024 17:34:47 -0800 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 6d0ee548c1..7967676d80 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -1001,7 +1001,7 @@ def patch_sft_trainer_tokenizer(): # Also DPO weirdly tokenizes non numeric columns? Delete them! check_text += \ "\n"\ - "if hasattr(self.train_dataset, 'column_names''):\n" + "if hasattr(self.train_dataset, 'column_names'):\n" " column_names = set(self.train_dataset.column_names)\n"\ " check = ['chosen', 'rejected', 'prompt', 'chosen_input_ids', 'chosen_attention_mask',\n"\ " 'chosen_labels', 'rejected_input_ids', 'rejected_attention_mask', 'rejected_labels',\n"\