diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 42f5eea045..b7fb43adb1 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -761,10 +761,6 @@ def _patch_fp8_moe_experts():
     if experts_interface is None:
         return
 
-    # Pre-quantized FP8 MoE checkpoints replace `.experts` modules with
-    # transformers.integrations.finegrained_fp8.FP8Experts. Route those
-    # implementations to Unsloth's MoE backend so we avoid the optional
-    # Hugging Face `kernels` package at training time.
     experts_interface["grouped_mm"] = forward_moe_backend
     experts_interface["batched_mm"] = forward_native_moe_loop
     if hasattr(finegrained_fp8, "FP8Experts"):
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 43068ad481..b93dddb186 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -300,17 +300,10 @@ def Qwen3Attention_fast_forward_inference(
 
     # Need to do it prior 2 steps before hitting full on short KV cache
     # or else error
-    if position_ids.dim() == 1:
-        position_ids = position_ids[:, None]
-    position_ids = position_ids.to(Qn.device)
-    if position_ids.shape[-1] != Qn.shape[-2]:
-        position_ids = position_ids[:, -Qn.shape[-2]:]
-
-    rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
-    self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1)  # +1 slack
-    cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index or 0)
-    cos = cos[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
-    sin = sin[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
+    self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
+    cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+    cos = cos[position_ids].unsqueeze(1)
+    sin = sin[position_ids].unsqueeze(1)
     h = self.half_head_dim
 
     RH_Q = self.RH_Q