ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2025-02-11 01:30:02 -08:00
commit 09e1fb7672
1 changed files with 1 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/tokenizer_utils.py
View file

@ -951,7 +951,7 @@ def patch_sft_trainer_tokenizer():
check_text = check_text.split("\n")
check_text = "\n".join(" "*where + x for x in check_text)
check_text = check_text.rstrip() + "\n"
if replacer is None:
# .*? matches first match. .+? matches final match.
replacer = re.findall(
@ -961,14 +961,10 @@ def patch_sft_trainer_tokenizer():
)
if len(replacer) == 0: continue
replacer = replacer[0]
print("====")
print(replacer)
print("====")
function = function.replace(replacer, replacer + check_text)
else:
function = function.replace(replacer, check_text + replacer)
pass
print(function)
x = [x for x in all_imports if x in function]
exec(f"from trl.trainer.sft_trainer import ({','.join(x)})", locals())