From 89fc908d85c20d0c56a9841ec2e9fc2b511e8596 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 1 Dec 2025 07:19:07 -0800 Subject: [PATCH] Update qwen3_moe.py --- unsloth/models/qwen3_moe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.py index 4202608536..bec3fa7b0d 100644 --- a/unsloth/models/qwen3_moe.py +++ b/unsloth/models/qwen3_moe.py @@ -130,7 +130,7 @@ def Qwen3MoeDecoderLayer_fast_forward( position_embeddings = position_embeddings, _flag_for_generation = self._flag_for_generation, ) - hidden_states += hidden_states + hidden_states += residual # MoE Router MLP residual = hidden_states @@ -140,7 +140,7 @@ def Qwen3MoeDecoderLayer_fast_forward( hidden_states, router_logits = Qwen3MoeSparseMoeBlock_fast_forward( self.mlp, hidden_states ) - hidden_states += hidden_states + hidden_states += residual else: residual = hidden_states hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)