From 639ac6f9876e8ed9191b38bbc12e2e883d95b445 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Tue, 7 Jan 2025 03:39:08 -0800
Subject: [PATCH] Update llama.py

---
 unsloth/models/llama.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 7d803bbe9d..edd3ddf94f 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -792,7 +792,12 @@ def LlamaModel_fast_forward(
         pass
     pass
 
-    position_embeddings = None
+    if IS_ATTENTION_REFACTOR and not hasattr(self.layers[0].self_attn, "rotary_emb"):
+        # Transformers main has made it mandatory to pass position_embeddings
+        # https://github.com/huggingface/transformers/pull/34858
+        position_embeddings = self.rotary_emb(hidden_states, position_ids, self.config.max_position_embeddings)
+    else:
+        position_embeddings = None
 
     # Go through every layer!
     for idx, decoder_layer in enumerate(self.layers):