ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update gemma.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-02-25 02:34:18 +11:00
commit 5c31fd1e5b
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth/models/gemma.py
View file

@ -82,8 +82,8 @@ class FastGemmaRotaryEmbedding(torch.nn.Module):
freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2)
emb_new = torch.cat((freqs, freqs), dim=-1)
print(torch.dist(emd_old, emd_new))
return emb.cos().to(dtype=x.dtype), emb.sin().to(dtype=x.dtype)
print(torch.dist(emb_old, emb_new))
return emb_new.cos().to(dtype=x.dtype), emb_new.sin().to(dtype=x.dtype)
pass