diff --git a/pyproject.toml b/pyproject.toml index 3d381fa669..b3ee2d7aea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.7.7", + "unsloth_zoo>=2025.7.8", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -381,7 +381,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.7.7", + "unsloth_zoo>=2025.7.8", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index f3fbf77222..23268807b1 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -80,6 +80,10 @@ class SyntheticDataKit: ) if "dtype" in engine_args: dtype_val = engine_args["dtype"] + if dtype_val == torch.float16: dtype_val = "float16" + elif dtype_val == torch.bfloat16: dtype_val = "bfloat16" + elif dtype_val == torch.float32: dtype_val = "float32" + engine_args["dtype"] = dtype_val # Convert torch.bfloat16, torch.float16, etc. to valid CLI string if hasattr(dtype_val, "name"): engine_args["dtype"] = dtype_val.name diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 1774ed373a..21ebbfd7f4 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.7.5" +__version__ = "2025.7.6" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/save.py b/unsloth/save.py index e61026318a..38c73c9956 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2240,6 +2240,7 @@ from unsloth_zoo.llama_cpp import ( def save_to_gguf_generic( model, save_directory, + quantization_method = None, quantization_type = "Q8_0", repo_id = None, token = None, @@ -2252,29 +2253,63 @@ def save_to_gguf_generic( install_llama_cpp(just_clone_repo = True) pass - metadata = _convert_to_gguf( - save_directory, - print_output = True, - quantization_type = quantization_type, - ) - if repo_id is not None: - prepare_saving( - model, - repo_id, - push_to_hub = True, - max_shard_size = "50GB", - private = True, - token = token, - ) + # Use old style quantization_method + new_quantization_methods = [] + if quantization_method is not None: + # Convert quantization_method to list + if isinstance(quantization_method, list): pass + elif isinstance(quantization_method, str): quantization_method = [ quantization_method, ] + elif isinstance(quantization_method, tuple): quantization_method = list(quantization_method) + else: + raise TypeError("Unsloth: quantization_method can only be a string or a list of strings") + pass + for i, quant_method in enumerate(quantization_method): + quant_method = quant_method.lower() + if quant_method == "not_quantized": quant_method = "f16" + elif quant_method == "fast_quantized": quant_method = "q8_0" + elif quant_method == "quantized": quant_method = "q4_k_m" + elif quant_method is None: quant_method = "q8_0" + new_quantization_methods.append(quant_method.lower()) + pass + else: + new_quantization_methods.append(quantization_type.lower()) + # Check if wrong method + for quant_method in new_quantization_methods: + if quant_method not in ALLOWED_QUANTS.keys(): + error = f"Unsloth: Quant method = [{quant_method}] not supported. Choose from below:\n" + for key, value in ALLOWED_QUANTS.items(): + error += f"[{key}] => {value}\n" + raise RuntimeError(error) + pass + pass - from huggingface_hub import HfApi - api = HfApi(token = token) - api.upload_folder( - folder_path = save_directory, - repo_id = repo_id, - repo_type = "model", - allow_patterns = ["*.gguf"], + # Go through all types and save individually - somewhat inefficient + # since we save F16 / BF16 multiple times + for quantization_type in new_quantization_methods: + metadata = _convert_to_gguf( + save_directory, + print_output = True, + quantization_type = quantization_type, ) + if repo_id is not None: + prepare_saving( + model, + repo_id, + push_to_hub = True, + max_shard_size = "50GB", + private = True, + token = token, + ) + + from huggingface_hub import HfApi + api = HfApi(token = token) + api.upload_folder( + folder_path = save_directory, + repo_id = repo_id, + repo_type = "model", + allow_patterns = ["*.gguf"], + ) + pass pass return metadata pass