ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2024-07-19 01:29:52 -07:00
commit 8ee997cac2
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
unsloth/tokenizer_utils.py
View file

@ -682,6 +682,11 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16):
embedding_matrix = model.get_input_embeddings ().weight
lm_head_matrix = model.get_output_embeddings().weight
# Ignore some model checks for now
if model.config._name_or_path in IGNORED_TOKENIZER_NAMES:
return
pass
# Get untrained tokens
indicator_untrained = torch.amax(embedding_matrix, axis = 1) <= eps
where_untrained = torch.where(indicator_untrained)[0]