Update tokenizer_utils.py

This commit is contained in:
Daniel Han 2025-02-11 00:23:24 -08:00
commit 07f39e0b05

View file

@ -914,6 +914,7 @@ def patch_sft_trainer_tokenizer():
try:
sft_trainer = eval(f"trl.trainer.sft_trainer.SFTTrainer")
except:
return
all_imports = dir(trl.trainer.sft_trainer)
for function_name, replacer in (