ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2025-02-11 00:42:47 -08:00
commit 6ae8ef7fdf
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unsloth/tokenizer_utils.py
View file

@ -918,7 +918,8 @@ def patch_sft_trainer_tokenizer():
all_imports = dir(trl.trainer.sft_trainer)
for (function_name, replacer,) in (
("_prepare_non_packed_dataloader", "def tokenize(element):",),
# ("_prepare_non_packed_dataloader", "def tokenize(element):",),
("_prepare_non_packed_dataloader", None,),
("_prepare_dataset", None,),
# ("_prepare_packed_dataloader", "if dataset_text_field is not None",),
):