From 5c1b0ae9ddcd20077d32de4e697aaa6797fac162 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 4 Sep 2025 05:17:59 -0700 Subject: [PATCH] Bug fixes (#3266) * Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py * Update rl_replacements.py * Update rl.py * Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Torch 2.8 * Update rl_replacements.py * Update loader.py * UNSLOTH_ENABLE_CCE * Fix * Update loader.py * Update loader.py * Update __init__.py * Update __init__.py * Update __init__.py * Update __init__.py * Import fixes * Update loader.py * Fix aimv2 issue * Update loader.py * Update import_fixes.py * Update import_fixes.py * Update loader.py * Update loader.py * Update loader.py * Upgrade * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * custom_datatype * recheck * Float16 * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Bug fix * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * torch_dtype * Update rl.py * Fix CE Loss * Versioning --------- Co-authored-by: Datta Nimmaturi --- pyproject.toml | 4 ++-- unsloth/__init__.py | 2 +- unsloth/models/llama.py | 4 ++-- unsloth/models/mistral.py | 29 +++++++++++++++++++++-------- unsloth/models/rl.py | 2 +- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8c60cb5866..160182c2a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.8.9", + "unsloth_zoo>=2025.9.1", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1", @@ -453,7 +453,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.8.9", + "unsloth_zoo>=2025.9.1", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 1b2a9310ff..25a54165b7 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -214,7 +214,7 @@ elif DEVICE_TYPE == "xpu": # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2025.8.8"): + if Version(unsloth_zoo_version) < Version("2025.9.1"): print( "Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n"\ "Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`" diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index cf2ca75f75..f978060c9c 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1236,7 +1236,7 @@ def CausalLM_fast_forward(fast_forward_inference): # < 1024 Normal Unsloth uses less VRAM! if bsz*q_len <= 1024: RETURN_LOGITS = True - if not RETURN_LOGITS and HAS_CUT_CROSS_ENTROPY and labels is not None: + if not RETURN_LOGITS and labels is not None: n_items = kwargs.get("num_items_in_batch", None) or kwargs.get("n_items", None) @@ -1259,7 +1259,7 @@ def CausalLM_fast_forward(fast_forward_inference): mask = None, n_items = n_items, scaling = getattr(self, "accelerator_scaler", None), - target_gb = 1, + target_gb = None, torch_compile = True, logit_softcapping = logit_softcapping, ) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 6274f2e5df..faab2d30b1 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -300,17 +300,30 @@ def MistralForCausalLM_fast_forward( # < 1024 Normal Unsloth uses less VRAM! if bsz * q_len <= 1024: RETURN_LOGITS = True - if not RETURN_LOGITS and HAS_CUT_CROSS_ENTROPY and os.environ.get("UNSLOTH_ENABLE_CCE", "1") != "0" and labels is not None: + if not RETURN_LOGITS and labels is not None: n_items = kwargs.get("num_items_in_batch", None) or kwargs.get("n_items", None) logit_softcapping = getattr(self.config, "final_logit_softcapping", 0) - loss = fused_linear_cross_entropy( - hidden_states = hidden_states, - lm_weight = lm_head, - labels = labels, - num_items_in_batch = n_items, - logit_softcapping = logit_softcapping, - ) + # loss = fused_linear_cross_entropy( + # hidden_states = hidden_states, + # lm_weight = lm_head, + # labels = labels, + # num_items_in_batch = n_items, + # logit_softcapping = logit_softcapping, + # ) + loss = unsloth_fused_ce_loss( + trainer = None, + hidden_states = hidden_states, + lm_head_weight = lm_head, + lm_head_bias = None, + labels = labels, + mask = None, + n_items = n_items, + scaling = getattr(self, "accelerator_scaler", None), + target_gb = None, + torch_compile = True, + logit_softcapping = logit_softcapping, + ) if not return_dict: output = (logits,) + outputs[1:] return (loss,) + output if loss is not None else output diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index fe9951ddf0..f342a4d86b 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -513,7 +513,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "fp16" : False, "include_tokens_per_second" : False, "include_num_input_tokens_seen" : False, - "auto_find_batch_size" : True, # Auto /2 batch size + "auto_find_batch_size" : False, # Auto /2 batch size - too many people complained so removing "dataloader_pin_memory" : True, # Might fail so disable for now # "dataloader_persistent_workers" : True, # Keeps dataloader in RAM