ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'unslothai:main' into main

Browse source
This commit is contained in:
electron271 2025-12-23 22:46:13 -06:00 • committed by GitHub
commit dfddcf8f98
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 117 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.pre-commit-config.yaml
View file

@ -1,6 +1,6 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.14.9
rev: v0.14.10
hooks:
- id: ruff
args:

4
pyproject.toml
View file

@ -60,7 +60,7 @@ huggingfacenotorch = [
]
huggingface = [
"unsloth[huggingfacenotorch]",
"unsloth_zoo>=2025.12.6",
"unsloth_zoo>=2025.12.7",
"torchvision",
"unsloth[triton]",
]
@ -523,7 +523,7 @@ colab-ampere-torch220 = [
"flash-attn>=2.6.3 ; ('linux' in sys_platform)",
]
colab-new = [
"unsloth_zoo>=2025.12.6",
"unsloth_zoo>=2025.12.7",
"packaging",
"tyro",
"transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3",

52
unsloth/device_type.py
View file

@ -25,7 +25,6 @@ __all__ = [
import torch
import functools
from unsloth_zoo.utils import Version
import inspect
@functools.cache
@ -78,21 +77,50 @@ def get_device_count():
DEVICE_COUNT: int = get_device_count()
# Check blocksize for 4bit -> 64 for CUDA, 128 for AMD
# If AMD, we cannot load pre-quantized models for now :(
# 4-bit quantization requires a block size of 64
# this is not supported on AMD Instinct GPUs currently
# | Device Type | Warp Size | Block Size |
# |-----------------|-----------|------------|
# | CUDA | 32 | 64 |
# | Radeon (Navi) | 32 | 64 |
# | Instinct (MI) | 64 | 128 |
#
# Since bitsandbytes 0.49.0, pre-quantized models with 64 blockwise now works
# on Radeon GPUs, but not Instinct MI300x for eg [WIP]
# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1748
ALLOW_PREQUANTIZED_MODELS: bool = True
# HSA_STATUS_ERROR_EXCEPTION checks - sometimes AMD fails for BnB
ALLOW_BITSANDBYTES: bool = True
if DEVICE_TYPE == "hip":
try:
from bitsandbytes.nn.modules import Params4bit
if "blocksize = 64 if not HIP_ENVIRONMENT else 128" in inspect.getsource(
Params4bit
):
ALLOW_PREQUANTIZED_MODELS = False
import bitsandbytes
ALLOW_BITSANDBYTES = Version(bitsandbytes.__version__) > Version("0.48.2.dev0")
except:
pass
print(
"Unsloth: `bitsandbytes` is not installed - 4bit QLoRA unallowed, but 16bit and full finetuning works."
)
ALLOW_PREQUANTIZED_MODELS = False
ALLOW_BITSANDBYTES = False
if ALLOW_BITSANDBYTES:
ALLOW_BITSANDBYTES = Version(bitsandbytes.__version__) > Version("0.48.2.dev0")
if Version(bitsandbytes.__version__) > Version("0.49.0"):
try:
# Pre-quantized bitsandbytes models use blocksize 64, so we need to check the GPU
from bitsandbytes.cextension import ROCM_WARP_SIZE_64
ALLOW_PREQUANTIZED_MODELS = not ROCM_WARP_SIZE_64
except Exception as e:
print(
"Unsloth: Checking `from bitsandbytes.cextension import ROCM_WARP_SIZE_64` had error = \n"
f"{str(e)}\n"
"4bit QLoRA disabled for now, but 16bit and full finetuning works."
)
ALLOW_PREQUANTIZED_MODELS = False
ALLOW_BITSANDBYTES = False
elif ALLOW_BITSANDBYTES:
from bitsandbytes.nn.modules import Params4bit
if "blocksize = 64 if not HIP_ENVIRONMENT else 128" in inspect.getsource(
Params4bit
):
ALLOW_PREQUANTIZED_MODELS = False

2
unsloth/import_fixes.py
View file

@ -72,8 +72,6 @@ class HideLoggingMessage(logging.Filter):
class HidePrintMessage:
__slots__ = ("_original_stream", "_hidden_texts")
def __init__(self, original_stream):
self._original_stream = original_stream
self._hidden_texts = []

23
unsloth/models/_utils.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "2025.12.7"
__version__ = "2025.12.9"
__all__ = [
"SUPPORTS_BFLOAT16",
@ -72,6 +72,7 @@ __all__ = [
"patch_hf_quantizer",
"verify_fp8_support_if_applicable",
"_get_inference_mode_context_manager",
"hf_login",
]
import torch
@ -2344,3 +2345,23 @@ def _get_inference_mode_context_manager(model: torch.nn.Module):
return torch.no_grad()
else:
return torch.inference_mode()
def hf_login(token: Optional[str] = None) -> Optional[str]:
if token is None:
try:
from huggingface_hub import get_token
token = get_token()
if token is None:
return None
except:
return None
try:
from huggingface_hub import login
login(token = token)
return token
except Exception as e:
logger.info(f"Failed to login to huggingface using token with error: {e}")
return token

3
unsloth/models/llama.py
View file

@ -2130,8 +2130,7 @@ class FastLlamaModel:
"Unsloth: `unsloth_vllm_standby` is True, but environment variable `UNSLOTH_VLLM_STANDBY` is not set to 1!"
)
if token is None:
token = get_token()
token = hf_login(token)
if model_patcher is None:
model_patcher = FastLlamaModel
SUPPORTS_BFLOAT16 = is_bfloat16_supported()

32
unsloth/models/loader.py
View file

@ -20,6 +20,7 @@ from ._utils import (
HAS_FLASH_ATTENTION_SOFTCAPPING,
USE_MODELSCOPE,
get_transformers_model_type,
hf_login,
)
from .granite import FastGraniteModel
from .llama import FastLlamaModel, logger
@ -151,15 +152,7 @@ class FastLanguageModel(FastLlamaModel):
**kwargs,
):
# Login to allow private models
if token is None:
token = get_token()
if token is not None:
try:
from huggingface_hub import login
login(token = token)
except:
pass
token = hf_login(token)
if load_in_8bit or full_finetuning or qat_scheme is not None:
return FastModel.from_pretrained(
model_name = model_name,
@ -191,12 +184,11 @@ class FastLanguageModel(FastLlamaModel):
disable_log_stats = disable_log_stats,
qat_scheme = qat_scheme,
load_in_fp8 = load_in_fp8,
unsloth_tiled_mlp = unsloth_tiled_mlp,
*args,
**kwargs,
)
if token is None:
token = get_token()
if isinstance(dtype, str) and dtype in ["float16", "bfloat16"]:
dtype = getattr(torch, dtype)
assert (
@ -249,7 +241,7 @@ class FastLanguageModel(FastLlamaModel):
model_name = new_model_name
# Check if pre-quantized models are allowed
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
@ -383,7 +375,7 @@ class FastLanguageModel(FastLlamaModel):
if not use_exact_model_name:
model_name = get_model_name(model_name, load_in_4bit)
# Check if pre-quantized models are allowed
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
@ -687,16 +679,8 @@ class FastModel(FastBaseModel):
*args,
**kwargs,
):
if token is None:
token = get_token()
# Login to allow private models
if token is not None:
try:
from huggingface_hub import login
login(token = token)
except:
pass
token = hf_login(token)
if whisper_language is not None:
assert type(whisper_language) is str
if whisper_task is not None:
@ -790,7 +774,7 @@ class FastModel(FastBaseModel):
model_name = new_model_name
# Check if pre-quantized models are allowed
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
@ -1056,7 +1040,7 @@ class FastModel(FastBaseModel):
if not use_exact_model_name:
model_name = get_model_name(model_name, load_in_4bit)
# Check if pre-quantized models are allowed
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):

9
unsloth/models/rl.py
View file

@ -812,8 +812,13 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
if "dataset_num_proc" in call_args:
num_proc_check = (
"if dataset_num_proc is None:\n"
" from multiprocessing import cpu_count\n"
" dataset_num_proc = min(max(cpu_count()+4, 2), 64)\n"
" import psutil\n"
" dataset_num_proc = min(max(psutil.cpu_count()+4, 2), 64)\n"
" memory_gb_left = psutil.virtual_memory().available / (1024**3)\n"
" if memory_gb_left <= 4: dataset_num_proc = 1 # Too risky, so set to 1\n"
" elif memory_gb_left <= 6: dataset_num_proc = min(2, dataset_num_proc)\n"
" elif memory_gb_left <= 10: dataset_num_proc = min(4, dataset_num_proc)\n"
" elif memory_gb_left <= 14: dataset_num_proc = min(6, dataset_num_proc)\n"
)
extra_args += num_proc_check

29
unsloth/models/vision.py
View file

@ -32,6 +32,13 @@ from ..kernels import (
from ._utils import __version__, importlib_version, _prepare_model_for_qat
from ._utils import *
from ..save import patch_saving_functions
from ..models.loader_utils import is_distributed
from unsloth_zoo.gradient_checkpointing import (
unpatch_unsloth_gradient_checkpointing,
unpatch_unsloth_smart_gradient_checkpointing,
)
import torch.utils.checkpoint as torch_checkpoint
import transformers.modeling_utils as hf_modeling_utils
from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model
from peft import PeftModelForCausalLM
from transformers import set_seed as transformers_set_seed
@ -390,8 +397,7 @@ class FastBaseModel:
"Unsloth: WARNING `trust_remote_code` is True.\n"
"Are you certain you want to do remote code execution?"
)
if token is None:
token = get_token()
token = hf_login(token)
SUPPORTS_BFLOAT16 = is_bfloat16_supported()
if DEVICE_TYPE == "cuda":
@ -1086,10 +1092,27 @@ class FastBaseModel:
# Use bfloat16 precision for full finetuning
float32_mixed_precision = False
# VLMs can hit DDP "marked ready twice" with re-entrant checkpointing.
# See: https://github.com/unslothai/unsloth/issues/3713.
use_reentrant = not is_distributed()
if not use_reentrant:
# Under DDP, avoid the offloaded/re-entrant checkpoint patch.
unpatch_unsloth_gradient_checkpointing()
unpatch_unsloth_smart_gradient_checkpointing()
# Force native checkpoint to default to non-reentrant for downstream calls.
_orig_checkpoint = torch_checkpoint.checkpoint
def _nonre_checkpoint(function, *args, **kwargs):
kwargs["use_reentrant"] = False
return _orig_checkpoint(function, *args, **kwargs)
torch_checkpoint.checkpoint = _nonre_checkpoint
hf_modeling_utils.checkpoint = _nonre_checkpoint
model = prepare_model_for_training(
model,
use_gradient_checkpointing = use_gradient_checkpointing,
use_reentrant = True,
use_reentrant = use_reentrant,
full_finetuning = full_finetuning,
train_layernorms = full_finetuning,
train_embedding = full_finetuning,

15
unsloth/save.py
View file

@ -69,8 +69,8 @@ __all__ = [
# llama.cpp specific targets - all takes 90s. Below takes 60s
LLAMA_CPP_TARGETS = [
"llama-quantize",
"llama-export-lora",
"llama-cli",
"llama-server",
]
# Check environments
@ -1429,7 +1429,7 @@ language:
- **License:** apache-2.0
- **Finetuned from model :** {base_model}
This {model_type} model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library.
This {model_type} model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth)
[<img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20made%20with%20love.png" width="200"/>](https://github.com/unslothai/unsloth)
"""
@ -2234,13 +2234,13 @@ tags:
{"- vision-language-model" if is_vlm else ""}
---
# {repo_id.split("/")[-1]} - GGUF
# {repo_id.split("/")[-1]} : GGUF
This model was finetuned and converted to GGUF format using [Unsloth](https://github.com/unslothai/unsloth).
**Example usage**:
- For text only LLMs: **llama-cli** **--hf** repo_id/model_name **-p** "why is the sky blue?"
- For multimodal models: **llama-mtmd-cli** **-m** model_name.gguf **--mmproj** mmproj_file.gguf
- For text only LLMs: `./llama.cpp/llama-cli -hf {repo_id} --jinja`
- For multimodal models: `./llama.cpp/llama-mtmd-cli -hf {repo_id} --jinja`
## Available Model files:
"""
@ -2281,6 +2281,11 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
"The model's BOS token behavior was adjusted for GGUF compatibility.\n"
)
readme_content += (
"This was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth)\n"
'[<img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20made%20with%20love.png" width="200"/>](https://github.com/unslothai/unsloth)\n'
)
readme_path = os.path.join(actual_save_directory, "README.md")
with open(readme_path, "w") as f:
f.write(readme_content)