ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into nightly

Browse source
This commit is contained in:
Daniel Han 2025-07-29 03:07:00 -07:00
commit 856dabf556
9 changed files with 112 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

8
pyproject.toml
View file

@ -37,7 +37,7 @@ triton = [
]
huggingface = [
"unsloth_zoo>=2025.7.10",
"unsloth_zoo>=2025.7.11",
"packaging",
"tyro",
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0",
@ -51,7 +51,7 @@ huggingface = [
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0",
"peft>=0.7.1,!=0.11.0",
"protobuf",
"huggingface_hub",
"huggingface_hub>=0.34.0",
"hf_transfer",
"unsloth[triton]",
]
@ -381,7 +381,7 @@ colab-ampere-torch220 = [
"flash-attn>=2.6.3",
]
colab-new = [
"unsloth_zoo>=2025.7.10",
"unsloth_zoo>=2025.7.11",
"packaging",
"tyro",
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0",
@ -392,7 +392,7 @@ colab-new = [
"wheel>=0.42.0",
"numpy",
"protobuf",
"huggingface_hub",
"huggingface_hub>=0.34.0",
"hf_transfer",
"bitsandbytes>=0.45.5",
"unsloth[triton]",

72
unsloth/chat_templates.py
View file

@ -1190,6 +1190,78 @@ CHAT_TEMPLATES["qwen3"] = (qwen3_template, qwen3_template_eos_token, False, qwen
DEFAULT_SYSTEM_MESSAGE["qwen3"] = None # No default system message for Qwen-3
pass
# =========================================== Gemma-3n
# Obtained via
# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
gemma3n_template = \
"""{{ bos_token }}
{%- if messages[0]['role'] == 'system' -%}
{%- if messages[0]['content'] is string -%}
{%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}
{%- else -%}
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}
{%- endif -%}
{%- set loop_messages = messages[1:] -%}
{%- else -%}
{%- set first_user_prefix = "" -%}
{%- set loop_messages = messages -%}
{%- endif -%}
{%- for message in loop_messages -%}
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
{%- endif -%}
{%- if (message['role'] == 'assistant') -%}
{%- set role = "model" -%}
{%- else -%}
{%- set role = message['role'] -%}
{%- endif -%}
{{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else "") }}
{%- if message['content'] is string -%}
{{ message['content'] | trim }}
{%- elif message['content'] is iterable -%}
{%- for item in message['content'] -%}
{%- if item['type'] == 'audio' -%}
{{ '<audio_soft_token>' }}
{%- elif item['type'] == 'image' -%}
{{ '<image_soft_token>' }}
{%- elif item['type'] == 'text' -%}
{{ item['text'] | trim }}
{%- endif -%}
{%- endfor -%}
{%- else -%}
{{ raise_exception("Invalid content type") }}
{%- endif -%}
{{ '<end_of_turn>\n' }}
{%- endfor -%}
{%- if add_generation_prompt -%}
{{'<start_of_turn>model\n'}}
{%- endif -%}
"""
# Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802
gemma3n_ollama = \
'''
{{- range $i, $_ := .Messages }}
{{- $last := eq (len (slice $.Messages $i)) 1 }}
{{- if or (eq .Role "user") (eq .Role "system") }}<start_of_turn>user
{{ .Content }}<end_of_turn>
{{ if $last }}<start_of_turn>model
{{ end }}
{{- else if eq .Role "assistant" }}<start_of_turn>model
{{ .Content }}{{ if not $last }}<end_of_turn>
{{ end }}
{{- end }}
{{- end }}
'''
gemma3n_template_eos_token = "<end_of_turn>"
CHAT_TEMPLATES["gemma-3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,)
DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None # No system message in Gemma-3n
CHAT_TEMPLATES["gemma3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,)
DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None # No system message in Gemma-3n
pass
def _change_system_message(template: str, type_chat_template: str, system_message: str = None):
system_message_pattern = r"\{system_message\}"

10
unsloth/models/_utils.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "2025.7.9"
__version__ = "2025.7.11"
__all__ = [
"SUPPORTS_BFLOAT16",
@ -221,12 +221,16 @@ class _RaiseUninitialized(logging.Handler):
def __init__(self):
super().__init__()
def emit(self, record):
if "some weights of" in str(record).lower():
record_lower = str(record).lower()
if ("some weights of" in record_lower) and \
("score.weight" not in record_lower) and \
("classifier.weight" not in record_lower):
raise Exception(
f"Unsloth: Critical error since some weights are not initialized.\n"\
f"Please try updating Unsloth, transformers and timm via:\n"\
f"`pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo transformers timm`\n"\
f"".str(record))
f"{str(record)}"
)
pass
class RaiseUninitialized:
def __init__(self):

2
unsloth/models/llama.py
View file

@ -499,6 +499,8 @@ def LlamaAttention_fast_forward(
# else inplace_rope_embedding(Q, K, cos, sin, position_ids)
# )
Q, K = fast_rope_embedding(Q, K, cos, sin)
# synchronize before cat to avoid race condition
torch.cuda.current_stream(Q.device).synchronize()
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)

2
unsloth/models/loader.py
View file

@ -858,7 +858,7 @@ class FastModel(FastBaseModel):
trust_remote_code = trust_remote_code,
)
# Patch it as well!
model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing)
model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing, trust_remote_code = trust_remote_code)
pass
return model, tokenizer
pass

27
unsloth/models/mistral.py
View file

@ -190,7 +190,8 @@ def MistralForCausalLM_fast_forward(
bsz, q_len = input_ids.shape
sliding_window = getattr(self.config, "sliding_window", None)
if HAS_XFORMERS and attention_mask is None:
if HAS_XFORMERS:
# Always create causal mask for xformers
if sliding_window is None or sliding_window == "null" or sliding_window <= 0:
causal_mask = xformers.attn_bias.LowerTriangularMask()
elif q_len <= sliding_window:
@ -200,12 +201,13 @@ def MistralForCausalLM_fast_forward(
.from_seqlens([q_len]*bsz)\
.make_local_attention(window_size = sliding_window)
elif not HAS_XFORMERS and attention_mask is None:
# If attention_mask exists, it will be handled in the attention forward
else:
# Not using xformers - need to create attention masks
if sliding_window is None or sliding_window == "null" or sliding_window <= 0 or q_len <= sliding_window:
# Fully causal mask
mask = torch.full((q_len, q_len), -torch.inf, device=input_ids.device)
mask = torch.triu(mask, diagonal=1)
attention_mask = mask.expand(bsz, 1, q_len, q_len)
causal_mask_values = torch.triu(torch.full((q_len, q_len), -torch.inf, device=input_ids.device), diagonal=1)
else:
# Sliding window attention
q_indices = torch.arange(q_len, device=input_ids.device).view(-1, 1)
@ -214,8 +216,19 @@ def MistralForCausalLM_fast_forward(
causal_bool_mask = k_indices <= q_indices
window_bool_mask = (q_indices - k_indices) < sliding_window
mask = torch.where(causal_bool_mask & window_bool_mask, 0.0, -torch.inf)
attention_mask = mask[None, None, :, :].expand(bsz, 1, q_len, q_len)
causal_mask_values = torch.where(causal_bool_mask & window_bool_mask, 0.0, -torch.inf)
# Combine with existing attention_mask if present
if attention_mask is None:
attention_mask = causal_mask_values[None, None, :, :].expand(bsz, 1, q_len, q_len)
else:
# attention_mask should be [bsz, 1, q_len, q_len] or broadcastable
# Add causal mask to existing attention mask
if attention_mask.dim() == 2:
# [bsz, seq_len] -> [bsz, 1, 1, seq_len]
attention_mask = attention_mask[:, None, None, :]
attention_mask = attention_mask.expand(bsz, 1, q_len, q_len)
attention_mask = attention_mask + causal_mask_values[None, None, :, :]
attention_mask = attention_mask.to(dtype=_get_dtype(self.config.torch_dtype))

5
unsloth/models/rl.py
View file

@ -702,13 +702,16 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
)
pass
# Remove peft_config
init = init.replace("elif peft_config is None:", "elif False:")
init = init.replace("elif peft_config is not None:", "elif False:")
init = init.replace("if peft_config is None:", "if False:")
init = init.replace("if peft_config is not None:", "if False:")
init = init.replace("get_peft_model(model, peft_config)", "model")
# New TRL 0.20.0
init = init.replace("if peft_config is not None or (is_peft_available() and isinstance(model, PeftModel)):", "if False:")
# New TRL 0.20.0
init = init.replace("model = self._prepare_peft_model(model, peft_config, args)\n", "pass\n")
# Set use_vllm if not set
if "args.use_vllm" in init and "model" in init and "args" in init:

2
unsloth/models/rl_replacements.py
View file

@ -350,7 +350,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
if function_name != "_get_per_token_logps_and_entropies": return function
# Just copy over from _get_per_token_logps replacement function above. For now this returns None anyway
def _get_per_token_logps_and_entropies(self, model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False):
def _get_per_token_logps_and_entropies(self, model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False, *args, **kwargs):
if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0':
return {"logps": None, "entropies": None} # Unsloth efficient GRPO
# Otherwise, calculate normally:

2
unsloth/models/vision.py
View file

@ -608,7 +608,7 @@ class FastBaseModel:
finetune_mlp_modules = finetune_mlp_modules,
)
else:
assert(type(target_modules) in (list, tuple,))
assert(type(target_modules) in (list, tuple, str,))
pass
# Clear deleted GPU items