ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tied weights

Browse source
This commit is contained in:
Daniel Han 2024-10-31 12:36:21 -07:00
commit 820cd4efef
1 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/models/llama.py
View file

@ -1838,7 +1838,8 @@ class FastLlamaModel:
except: old_output_embedding = torch.zeros(0)
# Check for tied weights as well
is_tied = old_input_embedding.data_ptr() == old_output_embedding.data_ptr()
# is_tied = old_input_embedding.data_ptr() == old_output_embedding.data_ptr()
is_tied = model.config.tie_word_embeddings
# Check pad token's id -> we need to expand the embedding
if len(tokenizer) > old_input_embedding.shape[0]:
@ -1887,6 +1888,9 @@ class FastLlamaModel:
else:
correct_dtype = old_input_embedding.dtype
pass
# Finally tie them if needed!
if is_tied: model.tie_weights()
# Also patch all dtypes - BnB seems to not allocate the correct type?
# BnB default dtype seems to be float16!