ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update gemma.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-02-25 01:58:33 +11:00
commit 5bf8cacebd
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/models/gemma.py
View file

@ -97,7 +97,7 @@ def GemmaAttention_fast_forward(
sin = self.rotary_emb.sin_cached
Q, K = fast_rope_embedding(Q, K, cos, sin)
else:
cos, sin = self.rotary_emb(V, position_ids, seq_len=None)
cos, sin = self.rotary_emb(V, position_ids, seq_len = q_len+1)
Q, K = inplace_rope_embedding(Q, K, cos, sin, position_ids)
pass
@ -518,8 +518,8 @@ class FastGemmaModel(FastLlamaModel):
# Inferene can now be CUDAGraphed, but we shall retain the old rotary embeddings.
# https://github.com/huggingface/transformers/pull/27931
# https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
# import transformers.models.gemma.modeling_gemma
# transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding = LlamaRotaryEmbedding
import transformers.models.gemma.modeling_gemma
transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding = LlamaRotaryEmbedding
return
pass