From f2fca2d6c1f474f9531a30af85397becc2b3af79 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sun, 25 Feb 2024 02:10:35 +1100 Subject: [PATCH] Update gemma.py --- unsloth/models/gemma.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index e73d0a6688..301f7fa923 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -92,9 +92,10 @@ def GemmaAttention_fast_forward( K = K.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2) V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2) - if position_ids is None: - cos = self.rotary_emb.cos_cached - sin = self.rotary_emb.sin_cached + if True:#position_ids is None: + # cos = self.rotary_emb.cos_cached + # sin = self.rotary_emb.sin_cached + cos, sin = self.rotary_emb(V, position_ids, seq_len = q_len+1) Q, K = fast_rope_embedding(Q, K, cos, sin) else: cos, sin = self.rotary_emb(V, position_ids, seq_len = q_len+1)