From 10137f1ba3a252ad975c94d4a0d2202b5ce81cdd Mon Sep 17 00:00:00 2001 From: Dan Saunders Date: Fri, 19 Dec 2025 22:09:16 -0500 Subject: [PATCH 01/11] Fix VLM DDP checkpointing (#3751) --- unsloth/models/vision.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index ed19f587cf..138b1b633a 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -32,6 +32,13 @@ from ..kernels import ( from ._utils import __version__, importlib_version, _prepare_model_for_qat from ._utils import * from ..save import patch_saving_functions +from ..models.loader_utils import is_distributed +from unsloth_zoo.gradient_checkpointing import ( + unpatch_unsloth_gradient_checkpointing, + unpatch_unsloth_smart_gradient_checkpointing, +) +import torch.utils.checkpoint as torch_checkpoint +import transformers.modeling_utils as hf_modeling_utils from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model from peft import PeftModelForCausalLM from transformers import set_seed as transformers_set_seed @@ -1086,10 +1093,27 @@ class FastBaseModel: # Use bfloat16 precision for full finetuning float32_mixed_precision = False + # VLMs can hit DDP "marked ready twice" with re-entrant checkpointing. + # See: https://github.com/unslothai/unsloth/issues/3713. + use_reentrant = not is_distributed() + if not use_reentrant: + # Under DDP, avoid the offloaded/re-entrant checkpoint patch. + unpatch_unsloth_gradient_checkpointing() + unpatch_unsloth_smart_gradient_checkpointing() + # Force native checkpoint to default to non-reentrant for downstream calls. + _orig_checkpoint = torch_checkpoint.checkpoint + + def _nonre_checkpoint(function, *args, **kwargs): + kwargs["use_reentrant"] = False + return _orig_checkpoint(function, *args, **kwargs) + + torch_checkpoint.checkpoint = _nonre_checkpoint + hf_modeling_utils.checkpoint = _nonre_checkpoint + model = prepare_model_for_training( model, use_gradient_checkpointing = use_gradient_checkpointing, - use_reentrant = True, + use_reentrant = use_reentrant, full_finetuning = full_finetuning, train_layernorms = full_finetuning, train_embedding = full_finetuning, From 191a9511f4a1920a2ce94bdb970415a1883eb419 Mon Sep 17 00:00:00 2001 From: Strahinja Stamenkovic Date: Sat, 20 Dec 2025 04:22:56 +0100 Subject: [PATCH 02/11] Enable 4-bit quantization on AMD Radeon GPUs (#3748) * Enable 4-bit quant on Radeon * Fix table centering * Update comments for clarity * Handle failure to import Bitsandbytes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update device_type.py * Apply suggestion from @danielhanchen * Update device_type.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- unsloth/device_type.py | 52 ++++++++++++++++++++++++++++++---------- unsloth/models/loader.py | 8 +++---- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/unsloth/device_type.py b/unsloth/device_type.py index adc09b05df..68038de679 100644 --- a/unsloth/device_type.py +++ b/unsloth/device_type.py @@ -25,7 +25,6 @@ __all__ = [ import torch import functools from unsloth_zoo.utils import Version -import inspect @functools.cache @@ -78,21 +77,50 @@ def get_device_count(): DEVICE_COUNT: int = get_device_count() -# Check blocksize for 4bit -> 64 for CUDA, 128 for AMD -# If AMD, we cannot load pre-quantized models for now :( +# 4-bit quantization requires a block size of 64 +# this is not supported on AMD Instinct GPUs currently +# | Device Type | Warp Size | Block Size | +# |-----------------|-----------|------------| +# | CUDA | 32 | 64 | +# | Radeon (Navi) | 32 | 64 | +# | Instinct (MI) | 64 | 128 | +# +# Since bitsandbytes 0.49.0, pre-quantized models with 64 blockwise now works +# on Radeon GPUs, but not Instinct MI300x for eg [WIP] +# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1748 + ALLOW_PREQUANTIZED_MODELS: bool = True # HSA_STATUS_ERROR_EXCEPTION checks - sometimes AMD fails for BnB ALLOW_BITSANDBYTES: bool = True if DEVICE_TYPE == "hip": try: - from bitsandbytes.nn.modules import Params4bit - - if "blocksize = 64 if not HIP_ENVIRONMENT else 128" in inspect.getsource( - Params4bit - ): - ALLOW_PREQUANTIZED_MODELS = False import bitsandbytes - - ALLOW_BITSANDBYTES = Version(bitsandbytes.__version__) > Version("0.48.2.dev0") except: - pass + print( + "Unsloth: `bitsandbytes` is not installed - 4bit QLoRA unallowed, but 16bit and full finetuning works." + ) + ALLOW_PREQUANTIZED_MODELS = False + ALLOW_BITSANDBYTES = False + if ALLOW_BITSANDBYTES: + ALLOW_BITSANDBYTES = Version(bitsandbytes.__version__) > Version("0.48.2.dev0") + if Version(bitsandbytes.__version__) > Version("0.49.0"): + try: + # Pre-quantized bitsandbytes models use blocksize 64, so we need to check the GPU + from bitsandbytes.cextension import ROCM_WARP_SIZE_64 + + ALLOW_PREQUANTIZED_MODELS = not ROCM_WARP_SIZE_64 + except Exception as e: + print( + "Unsloth: Checking `from bitsandbytes.cextension import ROCM_WARP_SIZE_64` had error = \n" + f"{str(e)}\n" + "4bit QLoRA disabled for now, but 16bit and full finetuning works." + ) + ALLOW_PREQUANTIZED_MODELS = False + ALLOW_BITSANDBYTES = False + elif ALLOW_BITSANDBYTES: + from bitsandbytes.nn.modules import Params4bit + + if "blocksize = 64 if not HIP_ENVIRONMENT else 128" in inspect.getsource( + Params4bit + ): + ALLOW_PREQUANTIZED_MODELS = False diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 6148b1783e..a1e5756060 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -249,7 +249,7 @@ class FastLanguageModel(FastLlamaModel): model_name = new_model_name # Check if pre-quantized models are allowed - # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 + # For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64 if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith( ("-unsloth-bnb-4bit", "-bnb-4bit") ): @@ -383,7 +383,7 @@ class FastLanguageModel(FastLlamaModel): if not use_exact_model_name: model_name = get_model_name(model_name, load_in_4bit) # Check if pre-quantized models are allowed - # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 + # For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64 if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith( ("-unsloth-bnb-4bit", "-bnb-4bit") ): @@ -790,7 +790,7 @@ class FastModel(FastBaseModel): model_name = new_model_name # Check if pre-quantized models are allowed - # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 + # For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64 if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith( ("-unsloth-bnb-4bit", "-bnb-4bit") ): @@ -1056,7 +1056,7 @@ class FastModel(FastBaseModel): if not use_exact_model_name: model_name = get_model_name(model_name, load_in_4bit) # Check if pre-quantized models are allowed - # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 + # For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64 if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith( ("-unsloth-bnb-4bit", "-bnb-4bit") ): From eceba83dab707114897d029a566c6c8624270167 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 19 Dec 2025 19:24:49 -0800 Subject: [PATCH 03/11] Update _utils.py --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 17d0ab9631..d769f815bd 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.7" +__version__ = "2025.12.8" __all__ = [ "SUPPORTS_BFLOAT16", From 20adb4465ab9f698e2f0910433d292a04fccf2d5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 19 Dec 2025 19:35:41 -0800 Subject: [PATCH 04/11] Nightly (#3753) * Update _utils.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [FIX] [Transformers] VLM input embeds fix for gradients (#3715) * Fix get_input_embeds call for VLMs * patch input_require_grads instead * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup old patch * cleanup old patch * cleanup * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestion from @danielhanchen * use logger instead of prints * Move unsloth present set * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han * Update rope_embedding.py * Fixes * Update _utils.py * Update import_fixes.py * Update rl_replacements.py * fix_openenv_no_vllm * Fix * Update __init__.py * Update __init__.py * Update __init__.py * Update import_fixes.py * Update import_fixes.py * Update import_fixes.py * logger * Update __init__.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update __init__.py * Update import_fixes.py * Update __init__.py * Update import_fixes.py * Update import_fixes.py * Update import_fixes.py * Update import_fixes.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update import_fixes.py * Update unsloth/import_fixes.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Update save.py * [fbgemm] Silence tma fbgemm (#3735) * Silence fbgemm TMA print Also safer .push_to_hub * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update loader.py * Update save.py * Update save.py * Update _utils.py * Update _utils.py * Diffusers warnings * Update pyproject.toml * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [hf_hub] Token login (#3739) * login on token * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup old code * safer imports * cleanup * Return token after login * correct return types * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestion from @danielhanchen * add back imports * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * finish return token --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han * Do not overwrite slots (#3752) * Do not overwrite slots * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Datta Nimmaturi Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- unsloth/import_fixes.py | 2 -- unsloth/models/_utils.py | 21 +++++++++++++++++++++ unsloth/models/llama.py | 3 +-- unsloth/models/loader.py | 23 +++-------------------- unsloth/models/vision.py | 3 +-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index f3aae7f523..efc7a7f4cd 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -72,8 +72,6 @@ class HideLoggingMessage(logging.Filter): class HidePrintMessage: - __slots__ = ("_original_stream", "_hidden_texts") - def __init__(self, original_stream): self._original_stream = original_stream self._hidden_texts = [] diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index d769f815bd..9929277719 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -72,6 +72,7 @@ __all__ = [ "patch_hf_quantizer", "verify_fp8_support_if_applicable", "_get_inference_mode_context_manager", + "hf_login", ] import torch @@ -2344,3 +2345,23 @@ def _get_inference_mode_context_manager(model: torch.nn.Module): return torch.no_grad() else: return torch.inference_mode() + + +def hf_login(token: Optional[str] = None) -> Optional[str]: + if token is None: + try: + from huggingface_hub import get_token + + token = get_token() + if token is None: + return None + except: + return None + try: + from huggingface_hub import login + + login(token = token) + return token + except Exception as e: + logger.info(f"Failed to login to huggingface using token with error: {e}") + return token diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 4c9337ccf9..1d7695b9aa 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2130,8 +2130,7 @@ class FastLlamaModel: "Unsloth: `unsloth_vllm_standby` is True, but environment variable `UNSLOTH_VLLM_STANDBY` is not set to 1!" ) - if token is None: - token = get_token() + token = hf_login(token) if model_patcher is None: model_patcher = FastLlamaModel SUPPORTS_BFLOAT16 = is_bfloat16_supported() diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index a1e5756060..fdfc313732 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -20,6 +20,7 @@ from ._utils import ( HAS_FLASH_ATTENTION_SOFTCAPPING, USE_MODELSCOPE, get_transformers_model_type, + hf_login, ) from .granite import FastGraniteModel from .llama import FastLlamaModel, logger @@ -151,15 +152,7 @@ class FastLanguageModel(FastLlamaModel): **kwargs, ): # Login to allow private models - if token is None: - token = get_token() - if token is not None: - try: - from huggingface_hub import login - - login(token = token) - except: - pass + token = hf_login(token) if load_in_8bit or full_finetuning or qat_scheme is not None: return FastModel.from_pretrained( model_name = model_name, @@ -195,8 +188,6 @@ class FastLanguageModel(FastLlamaModel): **kwargs, ) - if token is None: - token = get_token() if isinstance(dtype, str) and dtype in ["float16", "bfloat16"]: dtype = getattr(torch, dtype) assert ( @@ -687,16 +678,8 @@ class FastModel(FastBaseModel): *args, **kwargs, ): - if token is None: - token = get_token() # Login to allow private models - if token is not None: - try: - from huggingface_hub import login - - login(token = token) - except: - pass + token = hf_login(token) if whisper_language is not None: assert type(whisper_language) is str if whisper_task is not None: diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 138b1b633a..b78b190bcb 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -397,8 +397,7 @@ class FastBaseModel: "Unsloth: WARNING `trust_remote_code` is True.\n" "Are you certain you want to do remote code execution?" ) - if token is None: - token = get_token() + token = hf_login(token) SUPPORTS_BFLOAT16 = is_bfloat16_supported() if DEVICE_TYPE == "cuda": From 2eb6b0d5f363a60ed3792ea1f04250537ac66939 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 19 Dec 2025 19:37:49 -0800 Subject: [PATCH 05/11] Update loader.py --- unsloth/models/loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index fdfc313732..91016a13ba 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -184,6 +184,7 @@ class FastLanguageModel(FastLlamaModel): disable_log_stats = disable_log_stats, qat_scheme = qat_scheme, load_in_fp8 = load_in_fp8, + unsloth_tiled_mlp = unsloth_tiled_mlp, *args, **kwargs, ) From 2fe8825fe89ce46d2f740371b3477c007a8d3e46 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 23:12:00 -0800 Subject: [PATCH 06/11] [pre-commit.ci] pre-commit autoupdate (#3760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.9 → v0.14.10](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.9...v0.14.10) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d70d22426..545c7899aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.9 + rev: v0.14.10 hooks: - id: ruff args: From 886ec49d7f03d25f130d46dbcf70b02b17c7572c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Dec 2025 00:55:06 -0800 Subject: [PATCH 07/11] Update save.py --- unsloth/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/save.py b/unsloth/save.py index 24303aba52..f9c677f5f7 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -69,8 +69,8 @@ __all__ = [ # llama.cpp specific targets - all takes 90s. Below takes 60s LLAMA_CPP_TARGETS = [ "llama-quantize", - "llama-export-lora", "llama-cli", + "llama-server", ] # Check environments From 65f3b4b38ed24ee3a234385aee88c45ebebd3675 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Dec 2025 04:52:29 -0800 Subject: [PATCH 08/11] Nightly (#3767) * Update _utils.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [FIX] [Transformers] VLM input embeds fix for gradients (#3715) * Fix get_input_embeds call for VLMs * patch input_require_grads instead * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup old patch * cleanup old patch * cleanup * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestion from @danielhanchen * use logger instead of prints * Move unsloth present set * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han * Update rope_embedding.py * Fixes * Update _utils.py * Update import_fixes.py * Update rl_replacements.py * fix_openenv_no_vllm * Fix * Update __init__.py * Update __init__.py * Update __init__.py * Update import_fixes.py * Update import_fixes.py * Update import_fixes.py * logger * Update __init__.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update __init__.py * Update import_fixes.py * Update __init__.py * Update import_fixes.py * Update import_fixes.py * Update import_fixes.py * Update import_fixes.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update import_fixes.py * Update unsloth/import_fixes.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Update save.py * [fbgemm] Silence tma fbgemm (#3735) * Silence fbgemm TMA print Also safer .push_to_hub * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update loader.py * Update save.py * Update save.py * Update _utils.py * Update _utils.py * Diffusers warnings * Update pyproject.toml * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [hf_hub] Token login (#3739) * login on token * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup old code * safer imports * cleanup * Return token after login * correct return types * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestion from @danielhanchen * add back imports * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * finish return token --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han * Do not overwrite slots (#3752) * Do not overwrite slots * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update save.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Datta Nimmaturi Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- unsloth/save.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index f9c677f5f7..3a275cf0c3 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1429,7 +1429,7 @@ language: - **License:** apache-2.0 - **Finetuned from model :** {base_model} -This {model_type} model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library. +This {model_type} model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) [](https://github.com/unslothai/unsloth) """ @@ -2234,13 +2234,13 @@ tags: {"- vision-language-model" if is_vlm else ""} --- -# {repo_id.split("/")[-1]} - GGUF +# {repo_id.split("/")[-1]} : GGUF This model was finetuned and converted to GGUF format using [Unsloth](https://github.com/unslothai/unsloth). **Example usage**: -- For text only LLMs: **llama-cli** **--hf** repo_id/model_name **-p** "why is the sky blue?" -- For multimodal models: **llama-mtmd-cli** **-m** model_name.gguf **--mmproj** mmproj_file.gguf +- For text only LLMs: `./llama.cpp/llama-cli -hf {repo_id} --jinja` +- For multimodal models: `./llama.cpp/llama-mtmd-cli -hf {repo_id} --jinja` ## Available Model files: """ @@ -2281,6 +2281,11 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi "The model's BOS token behavior was adjusted for GGUF compatibility.\n" ) + readme_content += ( + "This was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth)\n" + '[](https://github.com/unslothai/unsloth)\n' + ) + readme_path = os.path.join(actual_save_directory, "README.md") with open(readme_path, "w") as f: f.write(readme_content) From 3407c788041665ea1d95e8de731e234afd477975 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Dec 2025 05:35:06 -0800 Subject: [PATCH 09/11] Update rl.py --- unsloth/models/rl.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 31316e45b7..4ffd790d1e 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -812,8 +812,13 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): if "dataset_num_proc" in call_args: num_proc_check = ( "if dataset_num_proc is None:\n" - " from multiprocessing import cpu_count\n" - " dataset_num_proc = min(max(cpu_count()+4, 2), 64)\n" + " import psutil\n" + " dataset_num_proc = min(max(psutil.cpu_count()+4, 2), 64)\n" + " memory_gb_left = psutil.virtual_memory().available / (1024**3)\n" + " if memory_gb_left <= 4: dataset_num_proc = 1 # Too risky, so set to 1\n" + " elif memory_gb_left <= 6: dataset_num_proc = min(2, dataset_num_proc)\n" + " elif memory_gb_left <= 8: dataset_num_proc = min(4, dataset_num_proc)\n" + " elif memory_gb_left <= 12: dataset_num_proc = min(6, dataset_num_proc)\n" ) extra_args += num_proc_check From 0e355c2d3718a69989102ab10dc4bee127e7a23a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Dec 2025 05:42:58 -0800 Subject: [PATCH 10/11] Update rl.py --- unsloth/models/rl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 4ffd790d1e..4ea36519d9 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -817,8 +817,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): " memory_gb_left = psutil.virtual_memory().available / (1024**3)\n" " if memory_gb_left <= 4: dataset_num_proc = 1 # Too risky, so set to 1\n" " elif memory_gb_left <= 6: dataset_num_proc = min(2, dataset_num_proc)\n" - " elif memory_gb_left <= 8: dataset_num_proc = min(4, dataset_num_proc)\n" - " elif memory_gb_left <= 12: dataset_num_proc = min(6, dataset_num_proc)\n" + " elif memory_gb_left <= 10: dataset_num_proc = min(4, dataset_num_proc)\n" + " elif memory_gb_left <= 14: dataset_num_proc = min(6, dataset_num_proc)\n" ) extra_args += num_proc_check From 06daf28c8b79782375bb7e17a830b11266407bc9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Dec 2025 05:50:26 -0800 Subject: [PATCH 11/11] llama.cpp fixes --- pyproject.toml | 4 ++-- unsloth/models/_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 84d6d86d93..decc0e9f5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.6", + "unsloth_zoo>=2025.12.7", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.6", + "unsloth_zoo>=2025.12.7", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 9929277719..abc8380562 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.8" +__version__ = "2025.12.9" __all__ = [ "SUPPORTS_BFLOAT16",