From 86884ab446b284d302bf558a868d1f56c0fc1e0c Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Mon, 28 Jul 2025 15:34:49 +0530 Subject: [PATCH 01/14] Fixup multi GPU workload. (#3049) * sync all instead * sync after move and rope init instead * sync after rope inside * Return new tensors and no sync * Sync only current stream * Fixup mask for xformers * sync for prefill only * clean up --- unsloth/models/llama.py | 2 ++ unsloth/models/mistral.py | 27 ++++++++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 3c0d5012ae..e7d9084ad8 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -499,6 +499,8 @@ def LlamaAttention_fast_forward( # else inplace_rope_embedding(Q, K, cos, sin, position_ids) # ) Q, K = fast_rope_embedding(Q, K, cos, sin) + # synchronize before cat to avoid race condition + torch.cuda.current_stream(Q.device).synchronize() if past_key_value is not None: K = torch.cat([past_key_value[0], K], dim = 2) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 68d4ba43fb..ef046d1f3c 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -190,7 +190,8 @@ def MistralForCausalLM_fast_forward( bsz, q_len = input_ids.shape sliding_window = getattr(self.config, "sliding_window", None) - if HAS_XFORMERS and attention_mask is None: + if HAS_XFORMERS: + # Always create causal mask for xformers if sliding_window is None or sliding_window == "null" or sliding_window <= 0: causal_mask = xformers.attn_bias.LowerTriangularMask() elif q_len <= sliding_window: @@ -200,12 +201,13 @@ def MistralForCausalLM_fast_forward( .from_seqlens([q_len]*bsz)\ .make_local_attention(window_size = sliding_window) - elif not HAS_XFORMERS and attention_mask is None: + # If attention_mask exists, it will be handled in the attention forward + + else: + # Not using xformers - need to create attention masks if sliding_window is None or sliding_window == "null" or sliding_window <= 0 or q_len <= sliding_window: # Fully causal mask - mask = torch.full((q_len, q_len), -torch.inf, device=input_ids.device) - mask = torch.triu(mask, diagonal=1) - attention_mask = mask.expand(bsz, 1, q_len, q_len) + causal_mask_values = torch.triu(torch.full((q_len, q_len), -torch.inf, device=input_ids.device), diagonal=1) else: # Sliding window attention q_indices = torch.arange(q_len, device=input_ids.device).view(-1, 1) @@ -214,8 +216,19 @@ def MistralForCausalLM_fast_forward( causal_bool_mask = k_indices <= q_indices window_bool_mask = (q_indices - k_indices) < sliding_window - mask = torch.where(causal_bool_mask & window_bool_mask, 0.0, -torch.inf) - attention_mask = mask[None, None, :, :].expand(bsz, 1, q_len, q_len) + causal_mask_values = torch.where(causal_bool_mask & window_bool_mask, 0.0, -torch.inf) + + # Combine with existing attention_mask if present + if attention_mask is None: + attention_mask = causal_mask_values[None, None, :, :].expand(bsz, 1, q_len, q_len) + else: + # attention_mask should be [bsz, 1, q_len, q_len] or broadcastable + # Add causal mask to existing attention mask + if attention_mask.dim() == 2: + # [bsz, seq_len] -> [bsz, 1, 1, seq_len] + attention_mask = attention_mask[:, None, None, :] + attention_mask = attention_mask.expand(bsz, 1, q_len, q_len) + attention_mask = attention_mask + causal_mask_values[None, None, :, :] attention_mask = attention_mask.to(dtype=_get_dtype(self.config.torch_dtype)) From 59c3461c6863f87faf925f541b44208bcb8c2cce Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 28 Jul 2025 08:28:18 -0700 Subject: [PATCH 02/14] Update _utils.py --- unsloth/models/_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 4f004ff93a..3851279af3 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -225,8 +225,8 @@ class _RaiseUninitialized(logging.Handler): raise Exception( f"Unsloth: Critical error since some weights are not initialized.\n"\ f"Please try updating Unsloth, transformers and timm via:\n"\ - f"`pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo transformers timm`\n"\ - f"".str(record)) + f"`pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo transformers timm`\n" + ) pass class RaiseUninitialized: def __init__(self): From 3275552160a954d8b0503559a804eef9a1ee45ff Mon Sep 17 00:00:00 2001 From: Etherll <61019402+Etherll@users.noreply.github.com> Date: Mon, 28 Jul 2025 20:53:38 +0300 Subject: [PATCH 03/14] Update loader.py --- unsloth/models/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 8b8ec1b1a0..e4d33e70f6 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -858,7 +858,7 @@ class FastModel(FastBaseModel): trust_remote_code = trust_remote_code, ) # Patch it as well! - model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing) + model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing,trust_remote_code = trust_remote_code) pass return model, tokenizer pass From 3e9d68bb805cb9ef4980d2382d36e66c59a1f02b Mon Sep 17 00:00:00 2001 From: Etherll <61019402+Etherll@users.noreply.github.com> Date: Mon, 28 Jul 2025 20:55:07 +0300 Subject: [PATCH 04/14] Update vision.py --- unsloth/models/vision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 2436db4ff4..246e5d5671 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -608,7 +608,7 @@ class FastBaseModel: finetune_mlp_modules = finetune_mlp_modules, ) else: - assert(type(target_modules) in (list, tuple,)) + assert(type(target_modules) in (list, tuple, str,)) pass # Clear deleted GPU items From e88ea0ad7fbb3160728d55988b6c0baa779c8cbc Mon Sep 17 00:00:00 2001 From: Etherll <61019402+Etherll@users.noreply.github.com> Date: Mon, 28 Jul 2025 21:48:36 +0300 Subject: [PATCH 05/14] Update _utils.py --- unsloth/models/_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 4f004ff93a..0e4df9e2c8 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -221,12 +221,13 @@ class _RaiseUninitialized(logging.Handler): def __init__(self): super().__init__() def emit(self, record): - if "some weights of" in str(record).lower(): + record_lower = str(record).lower() + if "some weights of" in record_lower and "score.weight" not in record_lower: raise Exception( f"Unsloth: Critical error since some weights are not initialized.\n"\ f"Please try updating Unsloth, transformers and timm via:\n"\ f"`pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo transformers timm`\n"\ - f"".str(record)) + f"{str(record)}") pass class RaiseUninitialized: def __init__(self): From 94be60007acaecbca83c6a730dc70009738bbab3 Mon Sep 17 00:00:00 2001 From: Etherll <61019402+Etherll@users.noreply.github.com> Date: Mon, 28 Jul 2025 22:15:23 +0300 Subject: [PATCH 06/14] Update loader.py --- unsloth/models/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index e4d33e70f6..a5b7d18164 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -858,7 +858,7 @@ class FastModel(FastBaseModel): trust_remote_code = trust_remote_code, ) # Patch it as well! - model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing,trust_remote_code = trust_remote_code) + model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing, trust_remote_code = trust_remote_code) pass return model, tokenizer pass From e6bb5b83b9de42b9678a1ffc04598bc0f93ef000 Mon Sep 17 00:00:00 2001 From: Sekinal Date: Mon, 28 Jul 2025 22:47:58 -0600 Subject: [PATCH 07/14] Fix: Added specific check for Gemma so models like BERT properly initialize --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 4f004ff93a..9c899e96e9 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -221,7 +221,7 @@ class _RaiseUninitialized(logging.Handler): def __init__(self): super().__init__() def emit(self, record): - if "some weights of" in str(record).lower(): + if "some weights of" in str(record).lower() and "gemma" in str(record).lower(): raise Exception( f"Unsloth: Critical error since some weights are not initialized.\n"\ f"Please try updating Unsloth, transformers and timm via:\n"\ From 76ceebd9c221e8df656436afee9a0ef74d342fd8 Mon Sep 17 00:00:00 2001 From: Sekinal Date: Mon, 28 Jul 2025 23:16:05 -0600 Subject: [PATCH 08/14] Fixed wrong syntax in f-string for exception --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 9c899e96e9..c7602ed193 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -226,7 +226,7 @@ class _RaiseUninitialized(logging.Handler): f"Unsloth: Critical error since some weights are not initialized.\n"\ f"Please try updating Unsloth, transformers and timm via:\n"\ f"`pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo transformers timm`\n"\ - f"".str(record)) + f"{record}") pass class RaiseUninitialized: def __init__(self): From 46d1dc4e94df1dd03d3e7c23521bb39308acd9bf Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 29 Jul 2025 00:36:25 -0700 Subject: [PATCH 09/14] Fix TRL 0.20.0 --- pyproject.toml | 4 ++-- unsloth/models/_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1529f9311b..15989915f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.7.10", + "unsloth_zoo>=2025.7.11", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -381,7 +381,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.7.10", + "unsloth_zoo>=2025.7.11", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 3851279af3..897f9a4475 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.7.9" +__version__ = "2025.7.10" __all__ = [ "SUPPORTS_BFLOAT16", From b2d61330b6f15a0738d087d3e62d6cf6c0514adf Mon Sep 17 00:00:00 2001 From: Etherll <61019402+Etherll@users.noreply.github.com> Date: Tue, 29 Jul 2025 10:39:38 +0300 Subject: [PATCH 10/14] Add gemma-3n chat template to chat_templates.py (#3051) * Update chat_templates.py * Update chat_templates.py --- unsloth/chat_templates.py | 72 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index e7849092cb..66d4b1c9aa 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -1190,6 +1190,78 @@ CHAT_TEMPLATES["qwen3"] = (qwen3_template, qwen3_template_eos_token, False, qwen DEFAULT_SYSTEM_MESSAGE["qwen3"] = None # No default system message for Qwen-3 pass +# =========================================== Gemma-3n +# Obtained via +# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n")) +gemma3n_template = \ +"""{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + '\n' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'audio' -%} + {{ '' }} + {%- elif item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ '\n' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model\n'}} +{%- endif -%} +""" + +# Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802 +gemma3n_ollama = \ +''' +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if or (eq .Role "user") (eq .Role "system") }}user +{{ .Content }} +{{ if $last }}model +{{ end }} +{{- else if eq .Role "assistant" }}model +{{ .Content }}{{ if not $last }} +{{ end }} +{{- end }} +{{- end }} +''' + +gemma3n_template_eos_token = "" +CHAT_TEMPLATES["gemma-3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,) +DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None # No system message in Gemma-3n + +CHAT_TEMPLATES["gemma3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,) +DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None # No system message in Gemma-3n +pass + def _change_system_message(template: str, type_chat_template: str, system_message: str = None): system_message_pattern = r"\{system_message\}" From e41e3c7a53c013c85c1e590c8d11c39a45a60f0a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 29 Jul 2025 01:03:19 -0700 Subject: [PATCH 11/14] Update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 15989915f0..a596ffe3cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ huggingface = [ "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", "peft>=0.7.1,!=0.11.0", "protobuf", - "huggingface_hub", + "huggingface_hub>=0.34.0", "hf_transfer", "unsloth[triton]", ] @@ -392,7 +392,7 @@ colab-new = [ "wheel>=0.42.0", "numpy", "protobuf", - "huggingface_hub", + "huggingface_hub>=0.34.0", "hf_transfer", "bitsandbytes>=0.45.5", "unsloth[triton]", From 02237cd1461b475687567050bef3eadf3c05fca2 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 29 Jul 2025 01:32:39 -0700 Subject: [PATCH 12/14] Update rl.py --- unsloth/models/rl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 664fe10c4f..deb779588c 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -702,13 +702,16 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import ) pass - # Remove peft_config init = init.replace("elif peft_config is None:", "elif False:") init = init.replace("elif peft_config is not None:", "elif False:") init = init.replace("if peft_config is None:", "if False:") init = init.replace("if peft_config is not None:", "if False:") init = init.replace("get_peft_model(model, peft_config)", "model") + # New TRL 0.20.0 + init = init.replace("if peft_config is not None or (is_peft_available() and isinstance(model, PeftModel)):", "if False:") + # New TRL 0.20.0 + init = init.replace("model = self._prepare_peft_model(model, peft_config, args)\n", "pass\n") # Set use_vllm if not set if "args.use_vllm" in init and "model" in init and "args" in init: From 2b5e2b93c6e0bf478a60e50730b1e7fd2bb72b98 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 29 Jul 2025 01:59:08 -0700 Subject: [PATCH 13/14] Update rl_replacements.py --- unsloth/models/rl_replacements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index a88385bf03..a126a3dde1 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -350,7 +350,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function): if function_name != "_get_per_token_logps_and_entropies": return function # Just copy over from _get_per_token_logps replacement function above. For now this returns None anyway - def _get_per_token_logps_and_entropies(self, model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False): + def _get_per_token_logps_and_entropies(self, model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False, *args, **kwargs): if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0': return {"logps": None, "entropies": None} # Unsloth efficient GRPO # Otherwise, calculate normally: From 64cf1934f9a20a74d7dd4006d1dbfa2cca07e9a4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 29 Jul 2025 02:19:43 -0700 Subject: [PATCH 14/14] Update _utils.py --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 83be0dddd6..4fad9cfa05 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.7.10" +__version__ = "2025.7.11" __all__ = [ "SUPPORTS_BFLOAT16",