From c7047d014f4066aff67a8cf06698ed013374fd0b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 30 May 2025 01:38:53 -0700 Subject: [PATCH 1/5] DeepSeek R1 Qwen --- unsloth/models/_utils.py | 2 +- unsloth/models/mapper.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 4bfaf6a963..0ad258889e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.5.9" +__version__ = "2025.5.10" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 50f0f7d7fb..8bd31efd46 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -864,6 +864,11 @@ __INT_TO_FLOAT_MAPPER = \ "mistralai/Devstral-Small-2505", "unsloth/Devstral-Small-2505-bnb-4bit", ), + "unsloth/DeepSeek-R1-0528-Qwen3-8B-unsloth-bnb-4bit" : ( + "unsloth/DeepSeek-R1-0528-Qwen3-8B", + "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", + "unsloth/DeepSeek-R1-0528-Qwen3-8B-bnb-4bit", + ), } INT_TO_FLOAT_MAPPER = {} From fa98cee8f460b25627fdc797d13db5c071edf5a5 Mon Sep 17 00:00:00 2001 From: datta0 Date: Sat, 31 May 2025 18:52:46 +0000 Subject: [PATCH 2/5] Fix quant model param fetch regex --- unsloth/models/_utils.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 0ad258889e..f41f95a85f 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -201,6 +201,36 @@ except: # Patch get_model_param_count to record correct 4bit / 8bit from transformers.trainer_pt_utils import is_deepspeed_zero3_enabled + +def extract_approx_params_from_config(config): + """ + Extract approximate parameter count from model config's name_or_path + Returns int (param count) or None if not found. + """ + lowercase_b_families = ["gemma"] # gemma uses small 'b' : google/gemma-3-1b-it + model_name = getattr(config, "name_or_path", "") + import re + cleaned = re.sub(r"[-_]?bnb[-_]?4bit|[-_]?4bit|[-_]?8bit|[-_]?bnb", "", model_name, flags=re.IGNORECASE) # replace bnb and xbit + match_B = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*B", cleaned) # first prefer searching 'B' + if match_B: + # most model names would come in this flow + billions = float(match_B.group(1)) + return int(1_000_000_000 * billions) + else: + for fam in lowercase_b_families: + if fam in cleaned.lower(): + match_b = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*b", cleaned) + if match_b: + billions = float(match_b.group(1)) + return int(1_000_000_000 * billions) + else: + match_any = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*[bB]", cleaned) + if match_any: + billions = float(match_any.group(1)) + return int(1_000_000_000 * billions) + return None + + def get_model_param_count(model, trainable_only = False): """ Calculate model's total param count. If trainable_only is True then count only those requiring grads @@ -215,12 +245,9 @@ def get_model_param_count(model, trainable_only = False): if (not trainable_only) and \ hasattr(model, "config") and \ hasattr(model.config, "quantization_config"): - - billions = re.findall(r"([0-9]{1,})(?:b|B)", model.config.name_or_path) - if len(billions) != 0: - billions = int(billions[0]) - s = 1_000_000_000 * billions - pass + approx = extract_approx_params_from_config(model.config) + if approx is not None: + s = approx return s pass import transformers.trainer_pt_utils From 0ff19962733dca4abb62ab58fec863d35f974d4b Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sat, 31 May 2025 14:38:55 -0700 Subject: [PATCH 3/5] Update issue templates --- .github/ISSUE_TEMPLATE/bug---issue.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug---issue.md b/.github/ISSUE_TEMPLATE/bug---issue.md index a08a6cf2cf..ff508cfb91 100644 --- a/.github/ISSUE_TEMPLATE/bug---issue.md +++ b/.github/ISSUE_TEMPLATE/bug---issue.md @@ -15,5 +15,5 @@ assignees: '' 6. Which trainer? `SFTTrainer`, `GRPOTrainer` etc 7. **Minimal code to reproduce error Remove Hugging Face token!** -For quick replies, got to https://discord.com/invite/unsloth. -Have you tried https://docs.unsloth.ai/basics/errors-troubleshooting +You can also join our Discord: https://discord.com/invite/unsloth +Have you tried visiting our Docs? https://docs.unsloth.ai/basics/errors-troubleshooting From e6b1a3703d7fe19b581074277084c1fa610d4bcc Mon Sep 17 00:00:00 2001 From: datta0 Date: Sun, 1 Jun 2025 05:57:43 +0000 Subject: [PATCH 4/5] Make replacement logic conscise --- unsloth/models/_utils.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index f41f95a85f..0230f84565 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -217,12 +217,11 @@ def extract_approx_params_from_config(config): billions = float(match_B.group(1)) return int(1_000_000_000 * billions) else: - for fam in lowercase_b_families: - if fam in cleaned.lower(): - match_b = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*b", cleaned) - if match_b: - billions = float(match_b.group(1)) - return int(1_000_000_000 * billions) + if any(fam in cleaned.lower() for fam in lowercase_b_families): + match_b = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*b", cleaned) + if match_b: + billions = float(match_b.group(1)) + return int(1_000_000_000 * billions) else: match_any = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*[bB]", cleaned) if match_any: From d80e8a5cd845b79a693d51494f92675b73a65c08 Mon Sep 17 00:00:00 2001 From: RunFMe Date: Mon, 2 Jun 2025 13:59:10 +0300 Subject: [PATCH 5/5] Fix batched generation for prompts of different lengths (#2216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix ignoring of attention mask after prefill stage in decoding * update naming to avoid confusion --------- Co-authored-by: Неизвестный Пользователь722497 --- unsloth/models/llama.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 480d22a6c5..2587c5a501 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -99,7 +99,7 @@ torch_nn_functional_softmax = torch.nn.functional.softmax SDPA_HAS_GQA = "enable_gqa" in scaled_dot_product_attention.__doc__ # Fix new HF's inference code -def _fast_prepare_inputs_for_generation(self, input_ids, **kwargs,): +def _fast_prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs,): past_key_values = kwargs.get("past_key_values", None) if past_key_values is not None: # Check for uninitialized DynamicCache @@ -107,11 +107,38 @@ def _fast_prepare_inputs_for_generation(self, input_ids, **kwargs,): past_key_values = None kwargs["past_key_values"] = None else: + bs, cache_length = input_ids.shape input_ids = input_ids[:,[-1]] - kwargs["attention_mask"] = kwargs["attention_mask"][:,[-1]] + + # Get to the base model + base_model = self + if hasattr(base_model, 'base_model_prefix'): + base_model = getattr(base_model, base_model.base_model_prefix) + + if hasattr(base_model, "_prepare_4d_causal_attention_mask_with_cache_position"): + attention_mask = base_model._prepare_4d_causal_attention_mask_with_cache_position( + attention_mask, + sequence_length=1, + target_length=cache_length, + dtype=self.dtype, + device=input_ids.device, + cache_position=torch.arange(cache_length, cache_length+1, device=input_ids.device), + batch_size=bs, + config=self.config, + past_key_values=past_key_values, + ) + else: + attention_mask = attention_mask[:,[-1]] + logger.warning_once( + f"{self.__class__.__name__} has no `_prepare_4d_causal_attention_mask_with_cache_position` method " + "defined in its base modeling class. Compiled forward passes will be sub-optimal. If you're " + "writing code, see Llama for an example implementation. If you're a user, please report this " + "issue on GitHub." + ) + if "cache_position" in kwargs: kwargs["position_ids"] = kwargs["cache_position"] - return { "input_ids" : input_ids, **kwargs, } + return { "input_ids" : input_ids, "attention_mask": attention_mask, **kwargs, } pass