ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2024-10-22 01:22:13 -07:00
commit 8f0a165a53
1 changed files with 7 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
unsloth/tokenizer_utils.py
View file

@ -878,7 +878,13 @@ def patch_trl_tokenizer_processing_class(trainer_name):
parameters = eval(f"inspect.signature({trainer_name}).parameters")
if "tokenizer" in parameters: return None
args = {key : value.default for key, value in parameters.items()}
args = {
key : \
value.default \
if type(value.default) is not str else \
f"'{value.default}'" \
for key, value in parameters.items()
}
args["tokenizer"] = None
new_args = args.copy()
del new_args["tokenizer"]