diff --git a/.github/ISSUE_TEMPLATE/bug---issue.md b/.github/ISSUE_TEMPLATE/bug---issue.md index 28495385bb..397d725f95 100644 --- a/.github/ISSUE_TEMPLATE/bug---issue.md +++ b/.github/ISSUE_TEMPLATE/bug---issue.md @@ -1,7 +1,7 @@ --- name: Bug / Issue about: Bug / Issue -title: "[Bug]" +title: "[Bug] Please fill in your issue title here." labels: bug assignees: '' @@ -10,9 +10,12 @@ assignees: '' 1. Did you update? `pip install --upgrade unsloth unsloth_zoo` 2. `Colab` or `Kaggle` or local / cloud 3. Number GPUs used, use `nvidia-smi` -4. Which notebook? -5. Paste `Unsloth` printout with :sloth: sloth emoji +4. Which notebook? Please link! +5. Which Unsloth version, TRL version, transformers version, PyTorch version? 6. Which trainer? `SFTTrainer`, `GRPOTrainer` etc -7. **Minimal code to reproduce error Remove Hugging Face token!** + +```python +Put Minimal code to reproduce error here ###Remove Hugging Face token### +``` 🦥 You can also ask via our Reddit page: https://www.reddit.com/r/unsloth/ diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py index 3cb186984f..99c58b36ec 100644 --- a/unsloth/kernels/moe/grouped_gemm/interface.py +++ b/unsloth/kernels/moe/grouped_gemm/interface.py @@ -114,7 +114,7 @@ def grouped_gemm_forward( - `permute_x`: fuse the permutation of hidden states from token order (original order) to grouped expert order, typically only needed for the first grouped GEMM in an MoE MLP. - When `permute_x` is True, `X` is expected to be of shape (num_tokens, K). - When `permute_x` is False, `X` is expected to be of shape (total_tokens, K) where `total_tokens = num_tokens * topk` AND already permuted to grouped expert order, i.e., hidden states are sorted such that tokens assigned to each expert are contiguous. - - `permute_y`: fused the permuation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. + - `permute_y`: fused the permutation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. - `fuse_mul_pre`: fuse the multiplication of the routed input with topk_weights, only done in the first grouped GEMM in an MoE MLP as for Llama4. Do not use, since results in performance regression as it interrupts the GEMM mainloop. - `fuse_mul_post`: fuse the multiplication of the routed output with topk_weights, used only when `permute_y` is True. NOTE: this should only be used when using this kernel for inference, not for training. @@ -881,7 +881,7 @@ def grouped_gemm( - `permute_x`: fuse the permutation of hidden states from token order (original order) to grouped expert order, typically only needed for the first grouped GEMM in an MoE MLP. - When `permute_x` is True, `X` is expected to be of shape (num_tokens, K). - When `permute_x` is False, `X` is expected to be of shape (total_tokens, K) where `total_tokens = num_tokens * topk` AND already permuted to grouped expert order, i.e., hidden states are sorted such that tokens assigned to each expert are contiguous. - - `permute_y`: fused the permuation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. + - `permute_y`: fused the permutation of the output from expert grouped order back to original token order, typically only needed for the second grouped GEMM in an MoE MLP. - `fuse_mul`: fuse the multiplication of the routed output with topk_weights, used only when `permute_y` is True. NOTE: this should only be used when using this kernel for inference, not for training. X: (M, K) hidden states where M is the num_tokens if `permute_x` is True, otherwise `total_tokens` where `total_tokens = num_tokens * topk`. diff --git a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py index 2bc9cc624d..37f001aefc 100644 --- a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py +++ b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py @@ -25,7 +25,7 @@ from grouped_gemm.reference.moe_ops import ( """ Reference implementation of HF Qwen3 MoE block using grouped gemm. -The Qwen3MoeGroupedGEMMBlock is a reference torch-native implemention. +The Qwen3MoeGroupedGEMMBlock is a reference torch-native implementation. Qwen3MoeFusedGroupedGEMMBlock is a version using the triton grouped gemm kernel. NOTE: This is NOT to be used for production as it contains many extra checks and saves all intermediate results for debugging. diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c6156fa468..84dd72e889 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -534,13 +534,25 @@ UNSLOTH_COMPILE_MAXIMUM = os.environ.get("UNSLOTH_COMPILE_MAXIMUM", UNSLOTH_COMPILE_IGNORE_ERRORS = os.environ.get("UNSLOTH_COMPILE_IGNORE_ERRORS", "1") == "1" # Just remove max_autotune_gemm warning import functools +from torch._inductor.runtime.hints import DeviceProperties + +from unsloth import DEVICE_TYPE + @functools.lru_cache(None) -def is_big_gpu(index): - sms = torch.cuda.get_device_properties(index).multi_processor_count - if sms < 80: # V100 - # log.warning("not enough SMs to use max_autotune_gemm mode") +def is_big_gpu(index) -> bool: + + if DEVICE_TYPE == "xpu": + prop = torch.xpu.get_device_properties(index) + min_sms = 16 + else: + prop = torch.cuda.get_device_properties(index) + min_sms = 80 + + avail_sms = prop.multi_processor_count + if avail_sms < min_sms: return False return True + import torch._inductor.utils torch._inductor.utils.is_big_gpu = is_big_gpu patch_torch_compile( diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 125bc7e610..9db8abdd43 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -69,7 +69,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSequen from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING from transformers import set_seed as transformers_set_seed from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model -from peft import PeftModelForCausalLM +from peft import PeftModelForCausalLM, PeftModelForSequenceClassification from ..save import patch_saving_functions import re, os, inspect, math, sys import types @@ -762,8 +762,7 @@ def LlamaModel_fast_forward( # Ignore attention_mask if attention_mask is None: padding_mask = None - elif self.training: - # elif attention_mask is None: + elif self.training and os.environ.get("UNSLOTH_KEEP_PADDING", "0") != '1': attention_mask = None padding_mask = None else: @@ -2079,7 +2078,8 @@ class FastLlamaModel: model.for_inference = functools.partial(FastLlamaModel.for_inference, model) # Patch generate - if model.generate.__name__ != "unsloth_fast_generate": + is_classification = "Classification" in str(type(model)) + if not is_classification and model.generate.__name__ != "unsloth_fast_generate": model._old_generate = model.generate unsloth_fast_generate.__doc__ = model._old_generate.__doc__ model.generate = types.MethodType(unsloth_fast_generate, model) @@ -2159,7 +2159,7 @@ class FastLlamaModel: if r <= 0: raise TypeError(f"Unsloth: Rank of {str(r)} must be larger than 0.") - if isinstance(model, PeftModelForCausalLM): + if isinstance(model, PeftModelForCausalLM) or isinstance(model, PeftModelForSequenceClassification): # Check if exactly the same and then pass through! assert(hasattr(model, "peft_config")) @@ -2428,7 +2428,7 @@ class FastLlamaModel: is_classification = "Classification" in str(type(model)) # Get LoRA - # if not is_classification else TaskType.SEQ_CLS + # arguments = dict( r = r, @@ -2436,7 +2436,7 @@ class FastLlamaModel: target_modules = final_modules, lora_dropout = lora_dropout, bias = bias, - task_type = TaskType.CAUSAL_LM, + task_type = TaskType.CAUSAL_LM if not is_classification else TaskType.SEQ_CLS, layers_to_transform = layers_to_transform, init_lora_weights = init_lora_weights, loftq_config = loftq_config, @@ -2450,7 +2450,6 @@ class FastLlamaModel: _saved_temp_tokenizer = model._saved_temp_tokenizer lora_config = LoraConfig(**arguments) - # First offload lm_head and embed_tokens to disk input_embeddings_device = model.get_input_embeddings().weight.device if is_classification: @@ -2572,7 +2571,7 @@ class FastLlamaModel: use_gradient_checkpointing = use_gradient_checkpointing, ) pass - if not isinstance(model, PeftModelForCausalLM): + if not isinstance(model, PeftModelForCausalLM) and not isinstance(model, PeftModelForSequenceClassification): raise TypeError( "Unsloth: Your model needs to call `.get_peft_model` first!" )