Update tokenizer_utils.py

This commit is contained in:
Daniel Han 2025-02-11 01:14:06 -08:00
commit 55967af47d

View file

@ -952,8 +952,9 @@ def patch_sft_trainer_tokenizer():
check_text = "\n".join(" "*where + x for x in check_text)
if replacer is None:
# .*? matches first match. .+? matches final match.
replacer = re.findall(
f"def {function_name}\(.+?\).+?\:\n",
f"def {function_name}\(.*?\).*?\:\n",
function,
flags = re.MULTILINE | re.DOTALL,
)