From 80e7af5b9f23e9e4aebcd1d0c358741cb17c84e3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 21 Jul 2025 05:30:14 -0700 Subject: [PATCH] Bug fixes (#3017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Revert "Revert "Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model me…" (#2990) This reverts commit 4021da634ac2f4fa71441c209b2c0b8aa6c56583. * skip_guard_eval_unsafe fix * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update llama.py * Update llama.py * Fix `quantization_method` * versioning --- pyproject.toml | 4 +- unsloth/dataprep/synthetic.py | 4 ++ unsloth/models/_utils.py | 2 +- unsloth/save.py | 77 +++++++++++++++++++++++++---------- 4 files changed, 63 insertions(+), 24 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3d381fa669..b3ee2d7aea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.7.7", + "unsloth_zoo>=2025.7.8", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -381,7 +381,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.7.7", + "unsloth_zoo>=2025.7.8", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index f3fbf77222..23268807b1 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -80,6 +80,10 @@ class SyntheticDataKit: ) if "dtype" in engine_args: dtype_val = engine_args["dtype"] + if dtype_val == torch.float16: dtype_val = "float16" + elif dtype_val == torch.bfloat16: dtype_val = "bfloat16" + elif dtype_val == torch.float32: dtype_val = "float32" + engine_args["dtype"] = dtype_val # Convert torch.bfloat16, torch.float16, etc. to valid CLI string if hasattr(dtype_val, "name"): engine_args["dtype"] = dtype_val.name diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 1774ed373a..21ebbfd7f4 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.7.5" +__version__ = "2025.7.6" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/save.py b/unsloth/save.py index e61026318a..38c73c9956 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2240,6 +2240,7 @@ from unsloth_zoo.llama_cpp import ( def save_to_gguf_generic( model, save_directory, + quantization_method = None, quantization_type = "Q8_0", repo_id = None, token = None, @@ -2252,29 +2253,63 @@ def save_to_gguf_generic( install_llama_cpp(just_clone_repo = True) pass - metadata = _convert_to_gguf( - save_directory, - print_output = True, - quantization_type = quantization_type, - ) - if repo_id is not None: - prepare_saving( - model, - repo_id, - push_to_hub = True, - max_shard_size = "50GB", - private = True, - token = token, - ) + # Use old style quantization_method + new_quantization_methods = [] + if quantization_method is not None: + # Convert quantization_method to list + if isinstance(quantization_method, list): pass + elif isinstance(quantization_method, str): quantization_method = [ quantization_method, ] + elif isinstance(quantization_method, tuple): quantization_method = list(quantization_method) + else: + raise TypeError("Unsloth: quantization_method can only be a string or a list of strings") + pass + for i, quant_method in enumerate(quantization_method): + quant_method = quant_method.lower() + if quant_method == "not_quantized": quant_method = "f16" + elif quant_method == "fast_quantized": quant_method = "q8_0" + elif quant_method == "quantized": quant_method = "q4_k_m" + elif quant_method is None: quant_method = "q8_0" + new_quantization_methods.append(quant_method.lower()) + pass + else: + new_quantization_methods.append(quantization_type.lower()) + # Check if wrong method + for quant_method in new_quantization_methods: + if quant_method not in ALLOWED_QUANTS.keys(): + error = f"Unsloth: Quant method = [{quant_method}] not supported. Choose from below:\n" + for key, value in ALLOWED_QUANTS.items(): + error += f"[{key}] => {value}\n" + raise RuntimeError(error) + pass + pass - from huggingface_hub import HfApi - api = HfApi(token = token) - api.upload_folder( - folder_path = save_directory, - repo_id = repo_id, - repo_type = "model", - allow_patterns = ["*.gguf"], + # Go through all types and save individually - somewhat inefficient + # since we save F16 / BF16 multiple times + for quantization_type in new_quantization_methods: + metadata = _convert_to_gguf( + save_directory, + print_output = True, + quantization_type = quantization_type, ) + if repo_id is not None: + prepare_saving( + model, + repo_id, + push_to_hub = True, + max_shard_size = "50GB", + private = True, + token = token, + ) + + from huggingface_hub import HfApi + api = HfApi(token = token) + api.upload_folder( + folder_path = save_directory, + repo_id = repo_id, + repo_type = "model", + allow_patterns = ["*.gguf"], + ) + pass pass return metadata pass