ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update qwen3_moe.py

Browse source
This commit is contained in:
Daniel Han 2025-12-01 07:19:07 -08:00
commit 89fc908d85
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth/models/qwen3_moe.py
View file

@ -130,7 +130,7 @@ def Qwen3MoeDecoderLayer_fast_forward(
position_embeddings = position_embeddings,
_flag_for_generation = self._flag_for_generation,
)
hidden_states += hidden_states
hidden_states += residual
# MoE Router MLP
residual = hidden_states
@ -140,7 +140,7 @@ def Qwen3MoeDecoderLayer_fast_forward(
hidden_states, router_logits = Qwen3MoeSparseMoeBlock_fast_forward(
self.mlp, hidden_states
)
hidden_states += hidden_states
hidden_states += residual
else:
residual = hidden_states
hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)