diff --git a/pyproject.toml b/pyproject.toml index 1529f9311b..a596ffe3cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.7.10", + "unsloth_zoo>=2025.7.11", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -51,7 +51,7 @@ huggingface = [ "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", "peft>=0.7.1,!=0.11.0", "protobuf", - "huggingface_hub", + "huggingface_hub>=0.34.0", "hf_transfer", "unsloth[triton]", ] @@ -381,7 +381,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.7.10", + "unsloth_zoo>=2025.7.11", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -392,7 +392,7 @@ colab-new = [ "wheel>=0.42.0", "numpy", "protobuf", - "huggingface_hub", + "huggingface_hub>=0.34.0", "hf_transfer", "bitsandbytes>=0.45.5", "unsloth[triton]", diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index e7849092cb..66d4b1c9aa 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -1190,6 +1190,78 @@ CHAT_TEMPLATES["qwen3"] = (qwen3_template, qwen3_template_eos_token, False, qwen DEFAULT_SYSTEM_MESSAGE["qwen3"] = None # No default system message for Qwen-3 pass +# =========================================== Gemma-3n +# Obtained via +# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n")) +gemma3n_template = \ +"""{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + '\n' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'audio' -%} + {{ '' }} + {%- elif item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ '\n' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model\n'}} +{%- endif -%} +""" + +# Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802 +gemma3n_ollama = \ +''' +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if or (eq .Role "user") (eq .Role "system") }}user +{{ .Content }} +{{ if $last }}model +{{ end }} +{{- else if eq .Role "assistant" }}model +{{ .Content }}{{ if not $last }} +{{ end }} +{{- end }} +{{- end }} +''' + +gemma3n_template_eos_token = "" +CHAT_TEMPLATES["gemma-3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,) +DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None # No system message in Gemma-3n + +CHAT_TEMPLATES["gemma3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,) +DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None # No system message in Gemma-3n +pass + def _change_system_message(template: str, type_chat_template: str, system_message: str = None): system_message_pattern = r"\{system_message\}" diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 4f004ff93a..4fad9cfa05 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.7.9" +__version__ = "2025.7.11" __all__ = [ "SUPPORTS_BFLOAT16", @@ -221,12 +221,16 @@ class _RaiseUninitialized(logging.Handler): def __init__(self): super().__init__() def emit(self, record): - if "some weights of" in str(record).lower(): + record_lower = str(record).lower() + if ("some weights of" in record_lower) and \ + ("score.weight" not in record_lower) and \ + ("classifier.weight" not in record_lower): raise Exception( f"Unsloth: Critical error since some weights are not initialized.\n"\ f"Please try updating Unsloth, transformers and timm via:\n"\ f"`pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo transformers timm`\n"\ - f"".str(record)) + f"{str(record)}" + ) pass class RaiseUninitialized: def __init__(self): diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 3c0d5012ae..e7d9084ad8 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -499,6 +499,8 @@ def LlamaAttention_fast_forward( # else inplace_rope_embedding(Q, K, cos, sin, position_ids) # ) Q, K = fast_rope_embedding(Q, K, cos, sin) + # synchronize before cat to avoid race condition + torch.cuda.current_stream(Q.device).synchronize() if past_key_value is not None: K = torch.cat([past_key_value[0], K], dim = 2) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 8b8ec1b1a0..a5b7d18164 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -858,7 +858,7 @@ class FastModel(FastBaseModel): trust_remote_code = trust_remote_code, ) # Patch it as well! - model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing) + model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing, trust_remote_code = trust_remote_code) pass return model, tokenizer pass diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 68d4ba43fb..ef046d1f3c 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -190,7 +190,8 @@ def MistralForCausalLM_fast_forward( bsz, q_len = input_ids.shape sliding_window = getattr(self.config, "sliding_window", None) - if HAS_XFORMERS and attention_mask is None: + if HAS_XFORMERS: + # Always create causal mask for xformers if sliding_window is None or sliding_window == "null" or sliding_window <= 0: causal_mask = xformers.attn_bias.LowerTriangularMask() elif q_len <= sliding_window: @@ -200,12 +201,13 @@ def MistralForCausalLM_fast_forward( .from_seqlens([q_len]*bsz)\ .make_local_attention(window_size = sliding_window) - elif not HAS_XFORMERS and attention_mask is None: + # If attention_mask exists, it will be handled in the attention forward + + else: + # Not using xformers - need to create attention masks if sliding_window is None or sliding_window == "null" or sliding_window <= 0 or q_len <= sliding_window: # Fully causal mask - mask = torch.full((q_len, q_len), -torch.inf, device=input_ids.device) - mask = torch.triu(mask, diagonal=1) - attention_mask = mask.expand(bsz, 1, q_len, q_len) + causal_mask_values = torch.triu(torch.full((q_len, q_len), -torch.inf, device=input_ids.device), diagonal=1) else: # Sliding window attention q_indices = torch.arange(q_len, device=input_ids.device).view(-1, 1) @@ -214,8 +216,19 @@ def MistralForCausalLM_fast_forward( causal_bool_mask = k_indices <= q_indices window_bool_mask = (q_indices - k_indices) < sliding_window - mask = torch.where(causal_bool_mask & window_bool_mask, 0.0, -torch.inf) - attention_mask = mask[None, None, :, :].expand(bsz, 1, q_len, q_len) + causal_mask_values = torch.where(causal_bool_mask & window_bool_mask, 0.0, -torch.inf) + + # Combine with existing attention_mask if present + if attention_mask is None: + attention_mask = causal_mask_values[None, None, :, :].expand(bsz, 1, q_len, q_len) + else: + # attention_mask should be [bsz, 1, q_len, q_len] or broadcastable + # Add causal mask to existing attention mask + if attention_mask.dim() == 2: + # [bsz, seq_len] -> [bsz, 1, 1, seq_len] + attention_mask = attention_mask[:, None, None, :] + attention_mask = attention_mask.expand(bsz, 1, q_len, q_len) + attention_mask = attention_mask + causal_mask_values[None, None, :, :] attention_mask = attention_mask.to(dtype=_get_dtype(self.config.torch_dtype)) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 664fe10c4f..deb779588c 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -702,13 +702,16 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import ) pass - # Remove peft_config init = init.replace("elif peft_config is None:", "elif False:") init = init.replace("elif peft_config is not None:", "elif False:") init = init.replace("if peft_config is None:", "if False:") init = init.replace("if peft_config is not None:", "if False:") init = init.replace("get_peft_model(model, peft_config)", "model") + # New TRL 0.20.0 + init = init.replace("if peft_config is not None or (is_peft_available() and isinstance(model, PeftModel)):", "if False:") + # New TRL 0.20.0 + init = init.replace("model = self._prepare_peft_model(model, peft_config, args)\n", "pass\n") # Set use_vllm if not set if "args.use_vllm" in init and "model" in init and "args" in init: diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index a88385bf03..a126a3dde1 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -350,7 +350,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function): if function_name != "_get_per_token_logps_and_entropies": return function # Just copy over from _get_per_token_logps replacement function above. For now this returns None anyway - def _get_per_token_logps_and_entropies(self, model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False): + def _get_per_token_logps_and_entropies(self, model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False, *args, **kwargs): if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0': return {"logps": None, "entropies": None} # Unsloth efficient GRPO # Otherwise, calculate normally: diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 2436db4ff4..246e5d5671 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -608,7 +608,7 @@ class FastBaseModel: finetune_mlp_modules = finetune_mlp_modules, ) else: - assert(type(target_modules) in (list, tuple,)) + assert(type(target_modules) in (list, tuple, str,)) pass # Clear deleted GPU items