From da4f4d6a9f8500d7315f3fde577f01dd5a1f2a54 Mon Sep 17 00:00:00 2001
From: datta0 <venkatadattasainimmaturi@gmail.com>
Date: Fri, 28 Mar 2025 05:58:01 +0000
Subject: [PATCH] Enable qwen3 and qwen3moe

---
 unsloth/models/__init__.py | 18 ++++++++++--------
 unsloth/models/_utils.py   |  2 +-
 unsloth/models/loader.py   | 13 +++++++++++++
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py
index 317525c793..99db55c086 100644
--- a/unsloth/models/__init__.py
+++ b/unsloth/models/__init__.py
@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .llama   import FastLlamaModel
-from .loader  import FastLanguageModel, FastVisionModel, FastTextModel, FastModel
-from .mistral import FastMistralModel
-from .qwen2   import FastQwen2Model
-from .granite import FastGraniteModel
-from .dpo     import PatchDPOTrainer, PatchKTOTrainer
-from ._utils  import is_bfloat16_supported, __version__
-from .rl      import PatchFastRL, vLLMSamplingParams
+from .llama     import FastLlamaModel
+from .loader    import FastLanguageModel, FastVisionModel, FastTextModel, FastModel
+from .mistral   import FastMistralModel
+from .qwen2     import FastQwen2Model
+from .qwen3     import FastQwen3Model
+from .qwen3_moe import FastQwen3MoeModel
+from .granite   import FastGraniteModel
+from .dpo       import PatchDPOTrainer, PatchKTOTrainer
+from ._utils    import is_bfloat16_supported, __version__
+from .rl        import PatchFastRL, vLLMSamplingParams
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 840c15c003..e60881fa9c 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -243,7 +243,7 @@ pass
 
 from transformers import __version__ as transformers_version
 from transformers import PretrainedConfig
-model_architectures = ["llama", "mistral", "gemma", "gemma2", "qwen2", "granite"]
+model_architectures = ["llama", "mistral", "gemma", "gemma2", "qwen2", "granite", "qwen3", "qwen3_moe"]
 
 for model_name in model_architectures:
     config_filepath = f"transformers.models.{model_name}.configuration_{model_name}"
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index cac5acd838..caa49d0bfb 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -23,6 +23,8 @@ from .granite import FastGraniteModel
 from .llama   import FastLlamaModel, logger
 from .mistral import FastMistralModel
 from .qwen2   import FastQwen2Model
+from .qwen3   import FastQwen3Model
+from .qwen3_moe import FastQwen3MoeModel
 from .cohere  import FastCohereModel
 from transformers import AutoConfig
 from transformers import __version__ as transformers_version
@@ -51,6 +53,8 @@ SUPPORTS_GEMMA2  = transformers_version >= Version("4.42")
 SUPPORTS_LLAMA31 = transformers_version >= Version("4.43.2")
 SUPPORTS_LLAMA32 = transformers_version  > Version("4.45.0")
 SUPPORTS_GRANITE = transformers_version >= Version("4.46.0")
+SUPPORTS_QWEN3   = transformers_version >= Version("4.50.3")
+SUPPORTS_QWEN3_MOE = transformers_version >= Version("4.50.3")
 if SUPPORTS_GEMMA:
     from .gemma  import FastGemmaModel
 if SUPPORTS_GEMMA2:
@@ -298,6 +302,15 @@ class FastLanguageModel(FastLlamaModel):
             dispatch_model = FastGemma2Model
         elif model_type == "qwen2":
             dispatch_model = FastQwen2Model
+        elif model_type == "qwen3" or model_type == "qwen3_moe":
+            if not SUPPORTS_QWEN3 or not SUPPORTS_QWEN3_MOE:
+                raise ImportError(
+                    f"Unsloth: Your transformers version of {transformers_version} does not support Qwen3.\n"\
+                    f"The minimum required version is 4.50.3.\n"\
+                    f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\
+                    f"to obtain the latest transformers build, then restart this session."\
+                )
+            dispatch_model = FastQwen3Model if model_type == "qwen3" else FastQwen3MoeModel
         # Temporary disable optimized Cohere until errors match
         # elif model_type == "cohere":
         #     dispatch_model = FastCohereModel