Bug fixes (#3180)
* Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py --------- Co-authored-by: Datta Nimmaturi <venkatadattasainimmaturi@gmail.com>
This commit is contained in:
Daniel Han
GitHub
parent
a14cf9a732
commit
609fe97cf2
5 changed files with 73 additions and 17 deletions
|
|
@ -37,7 +37,7 @@ triton = [
|
|||
]
|
||||
|
||||
huggingface = [
|
||||
"unsloth_zoo>=2025.8.5",
|
||||
"unsloth_zoo>=2025.8.6",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0",
|
||||
|
|
@ -384,7 +384,7 @@ colab-ampere-torch220 = [
|
|||
"flash-attn>=2.6.3",
|
||||
]
|
||||
colab-new = [
|
||||
"unsloth_zoo>=2025.8.5",
|
||||
"unsloth_zoo>=2025.8.6",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0",
|
||||
|
|
|
|||
|
|
@ -571,8 +571,11 @@ class FastModel(FastBaseModel):
|
|||
elif "qwen2.5" in lowered_model_name and transformers_version < Version("4.49.0"):
|
||||
raise RuntimeError("Unsloth: Qwen 2.5 only works on transformers >= 4.49.0." + LATEST)
|
||||
# Gemma 3
|
||||
elif "gemma-3" in lowered_model_name and transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "gemma-3" in lowered_model_name:
|
||||
if transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
|
||||
# Cohere
|
||||
elif "c4ai-command-a-03-2025" in lowered_model_name and transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
|
|
@ -582,31 +585,36 @@ class FastModel(FastBaseModel):
|
|||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \
|
||||
"all;torch.float32;torch.float16;"\
|
||||
"if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16);"
|
||||
"if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)"\
|
||||
";"
|
||||
# Granite 4
|
||||
elif 'granite-4' in lowered_model_name:
|
||||
# granite-4 rms norms are stored as 16 bit, but we upcast
|
||||
os.environ["UNSLOTH_UPCAST_LAYERNORM"] = "1"
|
||||
# Granite-4 rms norms are stored as 16 bit, but we upcast
|
||||
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
|
||||
# Olmo 2
|
||||
elif "olmo-2" in lowered_model_name and transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
# Gemma 3N
|
||||
elif "gemma-3n" in lowered_model_name:
|
||||
if transformers_version < Version("4.53.0"):
|
||||
raise RuntimeError("Unsloth: Gemma 3N only works on transformers >= 4.53.0" + LATEST)
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \
|
||||
"float16;torch.float16;torch.float16;"\
|
||||
"if name.endswith(('.conv')): module;"\
|
||||
"if name.endswith('norm'): "\
|
||||
"module._pre_set_compute_dtype = torch.float32\n"\
|
||||
";"\
|
||||
"from unsloth_zoo.temporary_patches.gemma3n import patch_Gemma3nConvNormAct_forward; patch_Gemma3nConvNormAct_forward()"
|
||||
|
||||
if transformers_version < Version("4.53.0"):
|
||||
raise RuntimeError("Unsloth: Gemma 3N only works on transformers >= 4.53.0" + LATEST)
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
|
||||
elif "falcon-h1" in lowered_model_name:
|
||||
# Falcon must use float32 Triton ie TRITON_F32_DEFAULT = 'ieee'
|
||||
# since Mamba kernels error out on using lower precision
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \
|
||||
"float16;torch.float32;torch.float16;"\
|
||||
"if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16);"\
|
||||
"if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"\
|
||||
";"\
|
||||
"os.environ['TRITON_F32_DEFAULT'] = 'ieee'"
|
||||
elif "gpt-oss" in lowered_model_name:
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
|
||||
|
|
@ -615,23 +623,30 @@ class FastModel(FastBaseModel):
|
|||
os.environ["UNSLOTH_ENABLE_CCE"] = "0"
|
||||
if not load_in_4bit:
|
||||
# Only upcast MoE biases for MXFP4, not BnB
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \
|
||||
"all;None;None;"\
|
||||
"x = 'gate_up_proj_bias'\n"\
|
||||
"if hasattr(module, x): "\
|
||||
"setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\
|
||||
""\
|
||||
"x = 'down_proj_bias'\n"\
|
||||
"if hasattr(module, x): "\
|
||||
"setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\
|
||||
""\
|
||||
";"
|
||||
else:
|
||||
# Set down projection compute dtype to be float32 for float16 machines
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \
|
||||
"all;None;None;"\
|
||||
"if 'down_projs' in name and hasattr(module, 'compute_dtype') and "\
|
||||
"if 'down_projs' in name and hasattr(module, 'weight') and "\
|
||||
"torch.amax(dequantize_module_weight(module)) >= 1024:"\
|
||||
"module._pre_set_compute_dtype = torch.float32\n"\
|
||||
""\
|
||||
";"
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
|
||||
else:
|
||||
for check_model_name in DISABLE_COMPILE_MODEL_NAMES:
|
||||
if check_model_name in lowered_model_name:
|
||||
|
|
|
|||
|
|
@ -487,6 +487,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
"logging_steps" : 1,
|
||||
"max_seq_length" : None,
|
||||
"num_generations" : 8,
|
||||
# "steps_per_generation" : 1, # Otherwise defaults to ga_steps which is wrong
|
||||
# "generation_batch_size" : None, # Useless. If steps_per_generation set, generation_batch_size clashes
|
||||
"top_k" : None,
|
||||
"vllm_mode" : "colocate",
|
||||
"generation_kwargs" : {},
|
||||
|
|
|
|||
|
|
@ -455,6 +455,12 @@ class FastBaseModel:
|
|||
# Return old flag
|
||||
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer
|
||||
|
||||
# Check float32 norm weights
|
||||
if os.environ.get("UNSLOTH_HIGH_PRECISION_LAYERNORM", "0") == "1":
|
||||
for jj, (name, module) in enumerate(model.named_modules()):
|
||||
if name.endswith("norm") and hasattr(module, "weight"):
|
||||
module._pre_set_compute_dtype = torch.float32
|
||||
pass
|
||||
# Edit data-types
|
||||
if custom_datatype is not None:
|
||||
with torch.no_grad():
|
||||
|
|
|
|||
|
|
@ -1195,6 +1195,41 @@ def save_to_gguf(
|
|||
f"--outfile {final_location} --vocab-type {vocab_type} "\
|
||||
f"--outtype {first_conversion} --concurrency {n_cpus} --pad-vocab"
|
||||
else:
|
||||
# Fix up conversion script is possible
|
||||
with open(convert_location, "rb") as f: converter_latest = f.read()
|
||||
# Fix metadata
|
||||
converter_latest = re.sub(
|
||||
rb"(self\.metadata \= .+?\(.+?\)"\
|
||||
rb"[\n]{1,}([\s]{4,}))",
|
||||
rb"\1"\
|
||||
rb"if hasattr(self.metadata, 'quantized_by'): self.metadata.quantized_by = 'Unsloth'\n"\
|
||||
rb"\2if hasattr(self.metadata, 'repo_url'): self.metadata.repo_url = 'https://huggingface.co/unsloth'\n"\
|
||||
rb"\2if hasattr(self.metadata, 'tags'): self.metadata.tags = ['unsloth', 'llama.cpp']\n"\
|
||||
rb"\2",
|
||||
converter_latest,
|
||||
)
|
||||
|
||||
# Make mistral_common optional for now
|
||||
# from x import y
|
||||
converter_latest = re.sub(
|
||||
rb"(from mistral_common[^\n\(]{1,})[\s]{0,}\n",
|
||||
rb"try:\n \1\nexcept:\n pass\n",
|
||||
converter_latest,
|
||||
)
|
||||
# from x import (y, z,)
|
||||
converter_latest = re.sub(
|
||||
rb"(from mistral_common[^\n\(]{1,}[\s]{0,}\(.+?\))",
|
||||
rb"try:\n \1\nexcept:\n pass\n",
|
||||
converter_latest,
|
||||
flags = re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
|
||||
try:
|
||||
# Write file
|
||||
with open(convert_location, "wb") as file:
|
||||
file.write(converter_latest)
|
||||
except:
|
||||
pass
|
||||
command = f"python {convert_location} {model_directory} "\
|
||||
f"--outfile {final_location} "\
|
||||
f"--outtype {first_conversion}"
|
||||
|
|
@ -1694,7 +1729,7 @@ def push_to_ollama_hub(username: str, model_name: str, tag: str):
|
|||
print(f"\nMODEL PUBLISHED FAILED WITH RETURN CODE {return_code}")
|
||||
else:
|
||||
print("\nMODEL PUBLISHED SUCCESSFULLY")
|
||||
|
||||
pass
|
||||
|
||||
def push_to_ollama(
|
||||
tokenizer,
|
||||
|
|
@ -1726,9 +1761,7 @@ def push_to_ollama(
|
|||
)
|
||||
|
||||
print("Successfully pushed to ollama")
|
||||
|
||||
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def unsloth_save_pretrained_gguf(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue