ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update _utils.py (#520)

Browse source 
Fixed a typo in the tokenizer fixer.
This commit is contained in:
Z 2024-05-23 19:10:06 -06:00 • committed by GitHub
commit a2180942b6
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/_utils.py
View file

@ -176,7 +176,7 @@ def patch_tokenizer(model, tokenizer):
if bad_pad_token:
# Find a better pad token
aadded_tokens = [str(x) for x in tokenizer.added_tokens_decoder.values()]
added_tokens = [str(x) for x in tokenizer.added_tokens_decoder.values()]
possible_pad_token = None
n_possible_pad_tokens = 0
for added_token in added_tokens[::-1]: