ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Undo qwen3 rope changes

Browse source
This commit is contained in:
Datta Nimmaturi 2026-03-16 07:10:10 +00:00
commit 5a3b1ed73b
2 changed files with 4 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth/kernels/fp8.py
View file

@ -761,10 +761,6 @@ def _patch_fp8_moe_experts():
if experts_interface is None:
return
# Pre-quantized FP8 MoE checkpoints replace `.experts` modules with
# transformers.integrations.finegrained_fp8.FP8Experts. Route those
# implementations to Unsloth's MoE backend so we avoid the optional
# Hugging Face `kernels` package at training time.
experts_interface["grouped_mm"] = forward_moe_backend
experts_interface["batched_mm"] = forward_native_moe_loop
if hasattr(finegrained_fp8, "FP8Experts"):

15
unsloth/models/qwen3.py
View file

@ -300,17 +300,10 @@ def Qwen3Attention_fast_forward_inference(
# Need to do it prior 2 steps before hitting full on short KV cache
# or else error
if position_ids.dim() == 1:
position_ids = position_ids[:, None]
position_ids = position_ids.to(Qn.device)
if position_ids.shape[-1] != Qn.shape[-2]:
position_ids = position_ids[:, -Qn.shape[-2]:]
rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1) # +1 slack
cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index or 0)
cos = cos[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
sin = sin[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
cos = cos[position_ids].unsqueeze(1)
sin = sin[position_ids].unsqueeze(1)
h = self.half_head_dim
RH_Q = self.RH_Q