ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Nightly (#784)

Browse source 
* Update __init__.py

* dynamic RoPE

* Update mistral.py

* Update llama.py

* Update tokenizer_utils.py

* Update mistral.py

* Update llama.py

* Update __init__.py

* Update flex_attention.py

* Update llama.py

* Update llama.py

* Mistral Nemo

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py
This commit is contained in:
Daniel Han 2024-07-19 01:39:08 -07:00 • committed by GitHub
commit ded20b2462
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
unsloth/tokenizer_utils.py
View file

@ -682,6 +682,11 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16):
embedding_matrix = model.get_input_embeddings ().weight
lm_head_matrix = model.get_output_embeddings().weight
# Ignore some model checks for now
if model.config._name_or_path in IGNORED_TOKENIZER_NAMES:
return
pass
# Get untrained tokens
indicator_untrained = torch.amax(embedding_matrix, axis = 1) <= eps
where_untrained = torch.where(indicator_untrained)[0]