From 7ddeea39cceffa55e608a1693d8baa87b8f728d2 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 1 Feb 2025 19:02:36 -0800 Subject: [PATCH] Update gemma.py --- unsloth/models/gemma.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 408c55440c..53d0bb51a1 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -223,6 +223,7 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module): self.base = base # Dynamic RoPE we first set it to a max of 4 * 8192 tokens then we iteratively grow this self.current_rope_size = min(4 * 8192, self.max_position_embeddings) + print(dim, max_position_embeddings, base) # Build here to make `torch.jit.trace` work. self._set_cos_sin_cache(seq_len=self.current_rope_size, device=device, dtype=torch.get_default_dtype())