From 0dbff7bade35f021a1afb85766d0e6ec9fb12f25 Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Tue, 11 Nov 2025 20:11:31 -0600 Subject: [PATCH] Resize rope embeddings for long sequence training (#3586) --- unsloth/models/llama.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 3af9cba669..306f95b41b 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1051,6 +1051,10 @@ def LlamaModel_fast_forward( # Also, transformers 4.45.0 supports granite but with the attention refactor (it always had the refactor) # unsloth's check for granite too has "version >= 4.45.0 (rightly so)". # so let granite always use the attention refactor implementation. + + self.rotary_emb.extend_rope_embedding( + hidden_states, self.config.max_position_embeddings + ) position_embeddings = self.rotary_emb.get_cached( self.config.max_position_embeddings, hidden_states.device.index )