ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update gemma.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-02-25 01:49:39 +11:00
commit b85b45e7a0
1 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
unsloth/models/gemma.py
View file

@ -92,12 +92,13 @@ def GemmaAttention_fast_forward(
K = K.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)
V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)
if True:#position_ids is None:
if position_ids is None:
cos = self.rotary_emb.cos_cached
sin = self.rotary_emb.sin_cached
Q, K = fast_rope_embedding(Q, K, cos, sin)
else:
cos, sin = self.rotary_emb(V, position_ids, seq_len=None)
print(cos.shape, sin.shape)
Q, K = inplace_rope_embedding(Q, K, cos, sin, position_ids)
pass
@ -518,8 +519,8 @@ class FastGemmaModel(FastLlamaModel):
# Inferene can now be CUDAGraphed, but we shall retain the old rotary embeddings.
# https://github.com/huggingface/transformers/pull/27931
# https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
import transformers.models.gemma.modeling_gemma
transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding = LlamaRotaryEmbedding
# import transformers.models.gemma.modeling_gemma
# transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding = LlamaRotaryEmbedding
return
pass