From 609fe97cf22976221cbb3d072b60946b61e994e9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 18 Aug 2025 06:12:49 -0700 Subject: [PATCH] Bug fixes (#3180) * Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py --------- Co-authored-by: Datta Nimmaturi --- pyproject.toml | 4 ++-- unsloth/models/loader.py | 37 +++++++++++++++++++++++++----------- unsloth/models/rl.py | 2 ++ unsloth/models/vision.py | 6 ++++++ unsloth/save.py | 41 ++++++++++++++++++++++++++++++++++++---- 5 files changed, 73 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6f6f225bde..f8558a83b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.8.5", + "unsloth_zoo>=2025.8.6", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -384,7 +384,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.8.5", + "unsloth_zoo>=2025.8.6", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 1c6a726b14..fae6ae0770 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -571,8 +571,11 @@ class FastModel(FastBaseModel): elif "qwen2.5" in lowered_model_name and transformers_version < Version("4.49.0"): raise RuntimeError("Unsloth: Qwen 2.5 only works on transformers >= 4.49.0." + LATEST) # Gemma 3 - elif "gemma-3" in lowered_model_name and transformers_version < Version("4.50.0.dev0"): - raise RuntimeError("Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY) + elif "gemma-3" in lowered_model_name: + if transformers_version < Version("4.50.0.dev0"): + raise RuntimeError("Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY) + # Set norms to float32 since anyways they get upcasted to float32 + os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1" # Cohere elif "c4ai-command-a-03-2025" in lowered_model_name and transformers_version < Version("4.50.0.dev0"): raise RuntimeError("Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY) @@ -582,31 +585,36 @@ class FastModel(FastBaseModel): os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "all;torch.float32;torch.float16;"\ - "if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16);" + "if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)"\ + ";" # Granite 4 elif 'granite-4' in lowered_model_name: - # granite-4 rms norms are stored as 16 bit, but we upcast - os.environ["UNSLOTH_UPCAST_LAYERNORM"] = "1" + # Granite-4 rms norms are stored as 16 bit, but we upcast + os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1" os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Olmo 2 elif "olmo-2" in lowered_model_name and transformers_version < Version("4.50.0.dev0"): raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY) # Gemma 3N elif "gemma-3n" in lowered_model_name: + if transformers_version < Version("4.53.0"): + raise RuntimeError("Unsloth: Gemma 3N only works on transformers >= 4.53.0" + LATEST) os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "float16;torch.float16;torch.float16;"\ - "if name.endswith(('.conv')): module;"\ + "if name.endswith('norm'): "\ + "module._pre_set_compute_dtype = torch.float32\n"\ + ";"\ "from unsloth_zoo.temporary_patches.gemma3n import patch_Gemma3nConvNormAct_forward; patch_Gemma3nConvNormAct_forward()" - - if transformers_version < Version("4.53.0"): - raise RuntimeError("Unsloth: Gemma 3N only works on transformers >= 4.53.0" + LATEST) + # Set norms to float32 since anyways they get upcasted to float32 + os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1" elif "falcon-h1" in lowered_model_name: # Falcon must use float32 Triton ie TRITON_F32_DEFAULT = 'ieee' # since Mamba kernels error out on using lower precision os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "float16;torch.float32;torch.float16;"\ - "if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16);"\ + "if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"\ + ";"\ "os.environ['TRITON_F32_DEFAULT'] = 'ieee'" elif "gpt-oss" in lowered_model_name: os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" @@ -615,23 +623,30 @@ class FastModel(FastBaseModel): os.environ["UNSLOTH_ENABLE_CCE"] = "0" if not load_in_4bit: # Only upcast MoE biases for MXFP4, not BnB + # Set norms to float32 since anyways they get upcasted to float32 os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "all;None;None;"\ "x = 'gate_up_proj_bias'\n"\ "if hasattr(module, x): "\ "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\ + ""\ "x = 'down_proj_bias'\n"\ "if hasattr(module, x): "\ "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\ + ""\ ";" else: # Set down projection compute dtype to be float32 for float16 machines + # Set norms to float32 since anyways they get upcasted to float32 os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "all;None;None;"\ - "if 'down_projs' in name and hasattr(module, 'compute_dtype') and "\ + "if 'down_projs' in name and hasattr(module, 'weight') and "\ "torch.amax(dequantize_module_weight(module)) >= 1024:"\ "module._pre_set_compute_dtype = torch.float32\n"\ + ""\ ";" + # Set norms to float32 since anyways they get upcasted to float32 + os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1" else: for check_model_name in DISABLE_COMPILE_MODEL_NAMES: if check_model_name in lowered_model_name: diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e751ef5e30..52b1e83694 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -487,6 +487,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "logging_steps" : 1, "max_seq_length" : None, "num_generations" : 8, + # "steps_per_generation" : 1, # Otherwise defaults to ga_steps which is wrong + # "generation_batch_size" : None, # Useless. If steps_per_generation set, generation_batch_size clashes "top_k" : None, "vllm_mode" : "colocate", "generation_kwargs" : {}, diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index a5de457cef..6790c5cd12 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -455,6 +455,12 @@ class FastBaseModel: # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer + # Check float32 norm weights + if os.environ.get("UNSLOTH_HIGH_PRECISION_LAYERNORM", "0") == "1": + for jj, (name, module) in enumerate(model.named_modules()): + if name.endswith("norm") and hasattr(module, "weight"): + module._pre_set_compute_dtype = torch.float32 + pass # Edit data-types if custom_datatype is not None: with torch.no_grad(): diff --git a/unsloth/save.py b/unsloth/save.py index bfd45a3645..9539b66701 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1195,6 +1195,41 @@ def save_to_gguf( f"--outfile {final_location} --vocab-type {vocab_type} "\ f"--outtype {first_conversion} --concurrency {n_cpus} --pad-vocab" else: + # Fix up conversion script is possible + with open(convert_location, "rb") as f: converter_latest = f.read() + # Fix metadata + converter_latest = re.sub( + rb"(self\.metadata \= .+?\(.+?\)"\ + rb"[\n]{1,}([\s]{4,}))", + rb"\1"\ + rb"if hasattr(self.metadata, 'quantized_by'): self.metadata.quantized_by = 'Unsloth'\n"\ + rb"\2if hasattr(self.metadata, 'repo_url'): self.metadata.repo_url = 'https://huggingface.co/unsloth'\n"\ + rb"\2if hasattr(self.metadata, 'tags'): self.metadata.tags = ['unsloth', 'llama.cpp']\n"\ + rb"\2", + converter_latest, + ) + + # Make mistral_common optional for now + # from x import y + converter_latest = re.sub( + rb"(from mistral_common[^\n\(]{1,})[\s]{0,}\n", + rb"try:\n \1\nexcept:\n pass\n", + converter_latest, + ) + # from x import (y, z,) + converter_latest = re.sub( + rb"(from mistral_common[^\n\(]{1,}[\s]{0,}\(.+?\))", + rb"try:\n \1\nexcept:\n pass\n", + converter_latest, + flags = re.MULTILINE | re.DOTALL, + ) + + try: + # Write file + with open(convert_location, "wb") as file: + file.write(converter_latest) + except: + pass command = f"python {convert_location} {model_directory} "\ f"--outfile {final_location} "\ f"--outtype {first_conversion}" @@ -1694,7 +1729,7 @@ def push_to_ollama_hub(username: str, model_name: str, tag: str): print(f"\nMODEL PUBLISHED FAILED WITH RETURN CODE {return_code}") else: print("\nMODEL PUBLISHED SUCCESSFULLY") - +pass def push_to_ollama( tokenizer, @@ -1726,9 +1761,7 @@ def push_to_ollama( ) print("Successfully pushed to ollama") - - - +pass def unsloth_save_pretrained_gguf(