From 096a49fa81cf448ba2d71232435714d01d5c136d Mon Sep 17 00:00:00 2001 From: kilavvy <140459108+kilavvy@users.noreply.github.com> Date: Mon, 23 Jun 2025 10:09:23 +0200 Subject: [PATCH 1/6] Docs: Fix typo and improve MoE docstrings (#2784) * Update qwen3_moe.py * Update interface.py --- unsloth/kernels/moe/grouped_gemm/interface.py | 4 ++-- .../kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py index 3cb186984f..99c58b36ec 100644 --- a/unsloth/kernels/moe/grouped_gemm/interface.py +++ b/unsloth/kernels/moe/grouped_gemm/interface.py @@ -114,7 +114,7 @@ def grouped_gemm_forward( - `permute_x`: fuse the permutation of hidden states from token order (original order) to grouped expert order, typically only needed for the first grouped GEMM in an MoE MLP. - When `permute_x` is True, `X` is expected to be of shape (num_tokens, K). - When `permute_x` is False, `X` is expected to be of shape (total_tokens, K) where `total_tokens = num_tokens * topk` AND already permuted to grouped expert order, i.e., hidden states are sorted such that tokens assigned to each expert are contiguous. - - `permute_y`: fused the permuation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. + - `permute_y`: fused the permutation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. - `fuse_mul_pre`: fuse the multiplication of the routed input with topk_weights, only done in the first grouped GEMM in an MoE MLP as for Llama4. Do not use, since results in performance regression as it interrupts the GEMM mainloop. - `fuse_mul_post`: fuse the multiplication of the routed output with topk_weights, used only when `permute_y` is True. NOTE: this should only be used when using this kernel for inference, not for training. @@ -881,7 +881,7 @@ def grouped_gemm( - `permute_x`: fuse the permutation of hidden states from token order (original order) to grouped expert order, typically only needed for the first grouped GEMM in an MoE MLP. - When `permute_x` is True, `X` is expected to be of shape (num_tokens, K). - When `permute_x` is False, `X` is expected to be of shape (total_tokens, K) where `total_tokens = num_tokens * topk` AND already permuted to grouped expert order, i.e., hidden states are sorted such that tokens assigned to each expert are contiguous. - - `permute_y`: fused the permuation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. + - `permute_y`: fused the permutation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. - `fuse_mul`: fuse the multiplication of the routed output with topk_weights, used only when `permute_y` is True. NOTE: this should only be used when using this kernel for inference, not for training. X: (M, K) hidden states where M is the num_tokens if `permute_x` is True, otherwise `total_tokens` where `total_tokens = num_tokens * topk`. diff --git a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py index 2bc9cc624d..37f001aefc 100644 --- a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py +++ b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py @@ -25,7 +25,7 @@ from grouped_gemm.reference.moe_ops import ( """ Reference implementation of HF Qwen3 MoE block using grouped gemm. -The Qwen3MoeGroupedGEMMBlock is a reference torch-native implemention. +The Qwen3MoeGroupedGEMMBlock is a reference torch-native implementation. Qwen3MoeFusedGroupedGEMMBlock is a version using the triton grouped gemm kernel. NOTE: This is NOT to be used for production as it contains many extra checks and saves all intermediate results for debugging. From 8846fbf1e8406890a57014595b29a3dd80fbfaf1 Mon Sep 17 00:00:00 2001 From: Lei Zhenyuan Date: Mon, 23 Jun 2025 19:47:34 +0800 Subject: [PATCH 2/6] [5/N] Enable intel GPU for unsloth (#2768) * add is_big_gpu support for xpu * make code unsloth's style --- unsloth/models/_utils.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c6156fa468..e40635222e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -534,13 +534,28 @@ UNSLOTH_COMPILE_MAXIMUM = os.environ.get("UNSLOTH_COMPILE_MAXIMUM", UNSLOTH_COMPILE_IGNORE_ERRORS = os.environ.get("UNSLOTH_COMPILE_IGNORE_ERRORS", "1") == "1" # Just remove max_autotune_gemm warning import functools +from torch._inductor.runtime.hints import DeviceProperties + +from unsloth import DEVICE_TYPE + @functools.lru_cache(None) -def is_big_gpu(index): - sms = torch.cuda.get_device_properties(index).multi_processor_count - if sms < 80: # V100 - # log.warning("not enough SMs to use max_autotune_gemm mode") +def is_big_gpu(index) -> bool: + + if DEVICE_TYPE == "xpu": + prop = torch.xpu.get_device_properties(index) + else: + prop = torch.cuda.get_device_properties(index) + + min_sms = 16 if device.type == "xpu" else 80 + avail_sms = prop.multi_processor_count + if avail_sms < min_sms: + log.warning( + "Not enough SMs to use max_autotune_gemm mode", + extra={"min_sms": min_sms, "avail_sms": avail_sms}, + ) return False return True + import torch._inductor.utils torch._inductor.utils.is_big_gpu = is_big_gpu patch_torch_compile( From b3a8df0beb803ed1b9535180cb7927d87e4c57ac Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 23 Jun 2025 05:26:28 -0700 Subject: [PATCH 3/6] Update issue templates --- .github/ISSUE_TEMPLATE/bug---issue.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug---issue.md b/.github/ISSUE_TEMPLATE/bug---issue.md index 28495385bb..88683132f9 100644 --- a/.github/ISSUE_TEMPLATE/bug---issue.md +++ b/.github/ISSUE_TEMPLATE/bug---issue.md @@ -10,9 +10,11 @@ assignees: '' 1. Did you update? `pip install --upgrade unsloth unsloth_zoo` 2. `Colab` or `Kaggle` or local / cloud 3. Number GPUs used, use `nvidia-smi` -4. Which notebook? -5. Paste `Unsloth` printout with :sloth: sloth emoji +4. Which notebook? Please link! +5. Which Unsloth version, TRL version, transformers version, PyTorch version? 6. Which trainer? `SFTTrainer`, `GRPOTrainer` etc -7. **Minimal code to reproduce error Remove Hugging Face token!** 🦥 You can also ask via our Reddit page: https://www.reddit.com/r/unsloth/ +```python +Put Minimal code to reproduce error here ###Remove Hugging Face token### +``` From 0da61b418e3b536ec86d4b1dede89f3d8f995ad6 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Mon, 23 Jun 2025 05:34:46 -0700 Subject: [PATCH 4/6] Update issue templates --- .github/ISSUE_TEMPLATE/bug---issue.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug---issue.md b/.github/ISSUE_TEMPLATE/bug---issue.md index 88683132f9..397d725f95 100644 --- a/.github/ISSUE_TEMPLATE/bug---issue.md +++ b/.github/ISSUE_TEMPLATE/bug---issue.md @@ -1,7 +1,7 @@ --- name: Bug / Issue about: Bug / Issue -title: "[Bug]" +title: "[Bug] Please fill in your issue title here." labels: bug assignees: '' @@ -14,7 +14,8 @@ assignees: '' 5. Which Unsloth version, TRL version, transformers version, PyTorch version? 6. Which trainer? `SFTTrainer`, `GRPOTrainer` etc -🦥 You can also ask via our Reddit page: https://www.reddit.com/r/unsloth/ ```python Put Minimal code to reproduce error here ###Remove Hugging Face token### ``` + +🦥 You can also ask via our Reddit page: https://www.reddit.com/r/unsloth/ From 1fc64a0a1ba3c043099fbc8fdc9cd96d9a9079f0 Mon Sep 17 00:00:00 2001 From: pluesclues <136766175+pluesclues@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:56:56 -0400 Subject: [PATCH 5/6] Fixed Sequence Classification errors, loaded model weirdly (#2793) --- unsloth/models/llama.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 125bc7e610..9db8abdd43 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -69,7 +69,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSequen from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING from transformers import set_seed as transformers_set_seed from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model -from peft import PeftModelForCausalLM +from peft import PeftModelForCausalLM, PeftModelForSequenceClassification from ..save import patch_saving_functions import re, os, inspect, math, sys import types @@ -762,8 +762,7 @@ def LlamaModel_fast_forward( # Ignore attention_mask if attention_mask is None: padding_mask = None - elif self.training: - # elif attention_mask is None: + elif self.training and os.environ.get("UNSLOTH_KEEP_PADDING", "0") != '1': attention_mask = None padding_mask = None else: @@ -2079,7 +2078,8 @@ class FastLlamaModel: model.for_inference = functools.partial(FastLlamaModel.for_inference, model) # Patch generate - if model.generate.__name__ != "unsloth_fast_generate": + is_classification = "Classification" in str(type(model)) + if not is_classification and model.generate.__name__ != "unsloth_fast_generate": model._old_generate = model.generate unsloth_fast_generate.__doc__ = model._old_generate.__doc__ model.generate = types.MethodType(unsloth_fast_generate, model) @@ -2159,7 +2159,7 @@ class FastLlamaModel: if r <= 0: raise TypeError(f"Unsloth: Rank of {str(r)} must be larger than 0.") - if isinstance(model, PeftModelForCausalLM): + if isinstance(model, PeftModelForCausalLM) or isinstance(model, PeftModelForSequenceClassification): # Check if exactly the same and then pass through! assert(hasattr(model, "peft_config")) @@ -2428,7 +2428,7 @@ class FastLlamaModel: is_classification = "Classification" in str(type(model)) # Get LoRA - # if not is_classification else TaskType.SEQ_CLS + # arguments = dict( r = r, @@ -2436,7 +2436,7 @@ class FastLlamaModel: target_modules = final_modules, lora_dropout = lora_dropout, bias = bias, - task_type = TaskType.CAUSAL_LM, + task_type = TaskType.CAUSAL_LM if not is_classification else TaskType.SEQ_CLS, layers_to_transform = layers_to_transform, init_lora_weights = init_lora_weights, loftq_config = loftq_config, @@ -2450,7 +2450,6 @@ class FastLlamaModel: _saved_temp_tokenizer = model._saved_temp_tokenizer lora_config = LoraConfig(**arguments) - # First offload lm_head and embed_tokens to disk input_embeddings_device = model.get_input_embeddings().weight.device if is_classification: @@ -2572,7 +2571,7 @@ class FastLlamaModel: use_gradient_checkpointing = use_gradient_checkpointing, ) pass - if not isinstance(model, PeftModelForCausalLM): + if not isinstance(model, PeftModelForCausalLM) and not isinstance(model, PeftModelForSequenceClassification): raise TypeError( "Unsloth: Your model needs to call `.get_peft_model` first!" ) From 1b4ca535ae634aafbe76ff37d4fed69a7b40ed95 Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Mon, 23 Jun 2025 20:57:55 -0500 Subject: [PATCH 6/6] move min_sms in is_big_gpu inside DEVICE_TYPE if else (#2792) log is not defined in torch inductor so remove Remove log.warning entirely --- unsloth/models/_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index e40635222e..84dd72e889 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -543,16 +543,13 @@ def is_big_gpu(index) -> bool: if DEVICE_TYPE == "xpu": prop = torch.xpu.get_device_properties(index) + min_sms = 16 else: prop = torch.cuda.get_device_properties(index) + min_sms = 80 - min_sms = 16 if device.type == "xpu" else 80 avail_sms = prop.multi_processor_count if avail_sms < min_sms: - log.warning( - "Not enough SMs to use max_autotune_gemm mode", - extra={"min_sms": min_sms, "avail_sms": avail_sms}, - ) return False return True