ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2024-08-16 23:38:43 -07:00
commit 191f2e2261
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/tokenizer_utils.py
View file

@ -834,7 +834,7 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16):
# We instead check for repeated vectors
lm_head_where = torch.where(indicator_untrained1)[0]
lm_head_bad = lm_head_matrix[lm_head_where]
lm_head_bad = lm_head_bad.cpu().to(torch.float32).numpy().round(3)
lm_head_bad = lm_head_bad.cpu().float().numpy().round(3)
from collections import Counter
counter = Counter()
for row in lm_head_bad: counter[hash(row.data.tobytes())] += 1