ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2026-02-05 14:12:16 +00:00
commit 62a1ac9d33
57 changed files with 1450 additions and 1192 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/saving/language_models/test_merge_4bit_validation.py
View file

@ -1,6 +1,6 @@
from unsloth import FastLanguageModel
from unsloth.chat_templates import get_chat_template
from trl import SFTTrainer, SFTConfig
from trl import SFTTrainer
from transformers import DataCollatorForSeq2Seq, TrainingArguments
from datasets import load_dataset
import torch

1
tests/saving/language_models/test_save_merged_grpo_model.py
View file

@ -11,7 +11,6 @@ import sys
from pathlib import Path
import multiprocessing as mp
import gc
from multiprocessing import Queue
REPO_ROOT = Path(__file__).parents[3]
sys.path.insert(0, str(REPO_ROOT))

2
tests/saving/non_peft/test_mistral_non_peft.py
View file

@ -1,6 +1,4 @@
from unsloth import FastLanguageModel
from transformers import AutoModelForCausalLM
from peft import PeftModel
from pathlib import Path
import sys
import warnings

5
tests/saving/non_peft/test_whisper_non_peft.py
View file

@ -1,6 +1,5 @@
from unsloth import FastLanguageModel, FastModel
from transformers import AutoModelForCausalLM, WhisperForConditionalGeneration
from peft import PeftModel
from unsloth import FastModel
from transformers import WhisperForConditionalGeneration
from pathlib import Path
import sys
import warnings

18
tests/saving/vision_models/test_index_file_sharded_model.py
View file

@ -28,7 +28,7 @@ train_dataset = dataset.select(range(2000))
# To select the next 200 examples for evaluation
eval_dataset = dataset.select(range(2000, 2200))
print(f"✅ Dataset loaded successfully!")
print("✅ Dataset loaded successfully!")
print(f" 📈 Training samples: {len(train_dataset)}")
print(f" 📊 Evaluation samples: {len(eval_dataset)}")
@ -110,10 +110,10 @@ try:
loftq_config = None, # And LoftQ
)
print("✅ LoRA configuration applied successfully!")
print(f" 🎯 LoRA rank (r): 16")
print(f" 📊 LoRA alpha: 32")
print(f" 🔍 Vision layers: Enabled")
print(f" 💬 Language layers: Enabled")
print(" 🎯 LoRA rank (r): 16")
print(" 📊 LoRA alpha: 32")
print(" 🔍 Vision layers: Enabled")
print(" 💬 Language layers: Enabled")
except Exception as e:
print(f"❌ Failed to apply LoRA configuration: {e}")
raise
@ -165,10 +165,10 @@ try:
),
)
print("✅ Trainer setup completed!")
print(f" 📦 Batch size: 2")
print(f" 🔄 Gradient accumulation steps: 4")
print(f" 📈 Max training steps: 10")
print(f" 🎯 Learning rate: 2e-4")
print(" 📦 Batch size: 2")
print(" 🔄 Gradient accumulation steps: 4")
print(" 📈 Max training steps: 10")
print(" 🎯 Learning rate: 2e-4")
print(f" 💾 Precision: {'BF16' if is_bf16_supported() else 'FP16'}")
except Exception as e:
print(f"❌ Failed to setup trainer: {e}")

18
tests/saving/vision_models/test_push_to_hub_merged.py
View file

@ -29,7 +29,7 @@ train_dataset = dataset.select(range(2000))
# To select the next 200 examples for evaluation
eval_dataset = dataset.select(range(2000, 2200))
print(f"✅ Dataset loaded successfully!")
print("✅ Dataset loaded successfully!")
print(f" 📈 Training samples: {len(train_dataset)}")
print(f" 📊 Evaluation samples: {len(eval_dataset)}")
@ -111,10 +111,10 @@ try:
loftq_config = None, # And LoftQ
)
print("✅ LoRA configuration applied successfully!")
print(f" 🎯 LoRA rank (r): 16")
print(f" 📊 LoRA alpha: 32")
print(f" 🔍 Vision layers: Enabled")
print(f" 💬 Language layers: Enabled")
print(" 🎯 LoRA rank (r): 16")
print(" 📊 LoRA alpha: 32")
print(" 🔍 Vision layers: Enabled")
print(" 💬 Language layers: Enabled")
except Exception as e:
print(f"❌ Failed to apply LoRA configuration: {e}")
raise
@ -166,10 +166,10 @@ try:
),
)
print("✅ Trainer setup completed!")
print(f" 📦 Batch size: 2")
print(f" 🔄 Gradient accumulation steps: 4")
print(f" 📈 Max training steps: 10")
print(f" 🎯 Learning rate: 2e-4")
print(" 📦 Batch size: 2")
print(" 🔄 Gradient accumulation steps: 4")
print(" 📈 Max training steps: 10")
print(" 🎯 Learning rate: 2e-4")
print(f" 💾 Precision: {'BF16' if is_bf16_supported() else 'FP16'}")
except Exception as e:
print(f"❌ Failed to setup trainer: {e}")

10
tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py
View file

@ -2,9 +2,6 @@
from unsloth import FastVisionModel
import torch
from qwen_vl_utils import process_vision_info
import os
from datasets import load_dataset
from trl import SFTTrainer, SFTConfig
@ -20,7 +17,6 @@ from tests.utils.ocr_eval import OCRModelEvaluator
## Dataset Preparation
from datasets import load_dataset
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train")
# To select the first 2000 examples
@ -66,12 +62,6 @@ train_dataset = [format_data(sample) for sample in train_dataset]
eval_dataset = [format_data(sample) for sample in eval_dataset]
## Setup OCR main evaluation function and helpers
import os
import torch
from tqdm import tqdm
import pandas as pd
from jiwer import wer, cer
from qwen_vl_utils import process_vision_info
#
ocr_evaluator = OCRModelEvaluator()

10
tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py
View file

@ -2,9 +2,6 @@
from unsloth import FastVisionModel
import torch
from qwen_vl_utils import process_vision_info
import os
from datasets import load_dataset
from trl import SFTTrainer, SFTConfig
@ -20,7 +17,6 @@ from tests.utils.ocr_eval import OCRModelEvaluator
## Dataset Preparation
from datasets import load_dataset
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train")
# To select the first 2000 examples
@ -66,12 +62,6 @@ train_dataset = [format_data(sample) for sample in train_dataset]
eval_dataset = [format_data(sample) for sample in eval_dataset]
## Setup OCR main evaluation function and helpers
import os
import torch
from tqdm import tqdm
import pandas as pd
from jiwer import wer, cer
from qwen_vl_utils import process_vision_info
#
ocr_evaluator = OCRModelEvaluator()

1
tests/test_raw_text.py
View file

@ -7,7 +7,6 @@ Tests basic functionality without heavy dependencies.
import sys
import os
import tempfile
from pathlib import Path
import importlib.util

12
tests/utils/aime_eval.py
View file

@ -207,7 +207,7 @@ def evaluate_model_aime(
print(f"\n{'='*70}")
print(f"🧮 AIME EVALUATION - {model_type.upper()} MODEL")
print(f"Combined Dataset: test2024 + test2025-I + test2025-II")
print("Combined Dataset: test2024 + test2025-I + test2025-II")
print(f"{'='*70}")
# Load combined AIME dataset
@ -244,7 +244,7 @@ def evaluate_model_aime(
seed = seed,
)
print(f"\n🔧 Configuration:")
print("\n🔧 Configuration:")
print(f" Temperature: {temperature}")
print(f" Samples per question: {n_sampling}")
print(f" Max tokens: {max_tokens}")
@ -421,28 +421,28 @@ def evaluate_model_aime(
print(f"📊 AIME EVALUATION RESULTS - {model_type.upper()}")
print(f"{'='*70}")
print(f"\n🎯 Overall Performance:")
print("\n🎯 Overall Performance:")
print(f" Total problems: {total_problems:>6}")
print(
f" Correct answers: {correct_answers:>6}/{total_problems} ({accuracy:>5.1f}%)"
)
print(f" Pass@{n_sampling}: {pass_at_k:>10.1f}%")
print(f"\n📈 Performance by Dataset:")
print("\n📈 Performance by Dataset:")
for source, stats in source_stats.items():
source_acc = source_accuracies[source]
print(
f" {source:>12}: {stats['correct']:>3}/{stats['total']:>3} ({source_acc:>5.1f}%)"
)
print(f"\n🔧 Configuration:")
print("\n🔧 Configuration:")
print(f" Temperature: {temperature}")
print(f" Samples per problem: {n_sampling}")
print(f" Max tokens: {max_tokens}")
print(f" Top-p: {top_p}")
print(f" Seed: {seed}")
print(f"\n📝 Token Statistics:")
print("\n📝 Token Statistics:")
print(f" Avg input tokens: {results['avg_input_tokens']:>10.1f}")
print(f" Avg output tokens: {results['avg_output_tokens']:>10.1f}")
print(f" Max input tokens: {results['max_input_tokens']:>10}")

5
tests/utils/hf_utils.py
View file

@ -12,9 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from contextlib import contextmanager, nullcontext
from typing import Callable, Optional
from contextlib import nullcontext
from typing import Callable
import bitsandbytes as bnb
import torch

2
tests/utils/ocr_eval.py
View file

@ -348,7 +348,7 @@ class OCRModelEvaluator:
plt.savefig("ocr_model_comparison.png")
plt.show()
print(f"\nVisualization saved to ocr_model_comparison.png")
print("\nVisualization saved to ocr_model_comparison.png")
def get_comparison_results(self) -> Dict[str, Dict[str, float]]:
"""Get the current comparison results."""

8
tests/utils/os_utils.py
View file

@ -98,10 +98,10 @@ def require_package(package_name, executable_name = None):
for pm_name, cmd in install_commands.items():
print(f" {pm_name}: {cmd}")
print(f"\nAlternatively, install with conda:")
print("\nAlternatively, install with conda:")
print(f" conda install -c conda-forge {package_name}")
print(f"\nPlease install the required package and run the script again.")
print("\nPlease install the required package and run the script again.")
sys.exit(1)
@ -120,9 +120,9 @@ def require_python_package(package_name, import_name = None, pip_name = None):
print(f"❌ Error: Python package '{package_name}' is not installed")
print(f"\nPlease install {package_name} using pip:")
print(f" pip install {pip_name}")
print(f" # or with conda:")
print(" # or with conda:")
print(f" conda install {pip_name}")
print(f"\nAfter installation, run this script again.")
print("\nAfter installation, run this script again.")
sys.exit(1)
else:
print(f"✓ Python package '{package_name}' is installed")

2
unsloth-cli.py
View file

@ -101,8 +101,6 @@ def run(args):
return {"text": texts}
def load_dataset_smart(args):
from transformers.utils import strtobool
if args.raw_text_file:
# Use raw text loader
loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride)

12
unsloth/__init__.py
View file

@ -12,9 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import warnings, importlib, sys
import warnings
import importlib
import sys
from packaging.version import Version
import os, re, subprocess, inspect, functools
import os
import re
import subprocess
import inspect
import functools
import numpy as np
# Log Unsloth is being used
@ -95,7 +101,7 @@ try:
import unsloth_zoo
except PackageNotFoundError:
raise ImportError(
f"Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo` then retry!"
"Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo` then retry!"
)
except:
raise

74
unsloth/_auto_install.py
View file

@ -12,32 +12,60 @@
# See the License for the specific language governing permissions and
# limitations under the License.
try: import torch
except: raise ImportError('Install torch via `pip install torch`')
try:
import torch
except:
raise ImportError("Install torch via `pip install torch`")
from packaging.version import Version as V
import re
v = V(re.match(r"[0-9\.]{3,}", torch.__version__).group(0))
cuda = str(torch.version.cuda)
is_ampere = torch.cuda.get_device_capability()[0] >= 8
USE_ABI = torch._C._GLIBCXX_USE_CXX11_ABI
if cuda not in ("11.8", "12.1", "12.4", "12.6", "12.8", "13.0"): raise RuntimeError(f"CUDA = {cuda} not supported!")
if v <= V('2.1.0'): raise RuntimeError(f"Torch = {v} too old!")
elif v <= V('2.1.1'): x = 'cu{}{}-torch211'
elif v <= V('2.1.2'): x = 'cu{}{}-torch212'
elif v < V('2.3.0'): x = 'cu{}{}-torch220'
elif v < V('2.4.0'): x = 'cu{}{}-torch230'
elif v < V('2.5.0'): x = 'cu{}{}-torch240'
elif v < V('2.5.1'): x = 'cu{}{}-torch250'
elif v <= V('2.5.1'): x = 'cu{}{}-torch251'
elif v < V('2.7.0'): x = 'cu{}{}-torch260'
elif v < V('2.7.9'): x = 'cu{}{}-torch270'
elif v < V('2.8.0'): x = 'cu{}{}-torch271'
elif v < V('2.8.9'): x = 'cu{}{}-torch280'
elif v < V('2.9.1'): x = 'cu{}{}-torch290'
elif v < V('2.9.2'): x = 'cu{}{}-torch291'
elif v < V('2.10.1'): x = 'cu{}{}-torch2100'
else: raise RuntimeError(f"Torch = {v} too new!")
if v > V('2.6.9') and cuda not in ("11.8", "12.6", "12.8", "13.0"): raise RuntimeError(f"CUDA = {cuda} not supported!")
if v >= V('2.10.0') and cuda not in ("12.6", "12.8", "13.0"): raise RuntimeError(f"Torch 2.10 requires CUDA 12.6, 12.8, or 13.0! Got CUDA = {cuda}")
x = x.format(cuda.replace(".", ""), "-ampere" if False else "") # is_ampere is broken due to flash-attn
print(f'pip install --upgrade pip && pip install --no-deps git+https://github.com/unslothai/unsloth-zoo.git && pip install "unsloth[{x}] @ git+https://github.com/unslothai/unsloth.git" --no-build-isolation')
if cuda not in ("11.8", "12.1", "12.4", "12.6", "12.8", "13.0"):
raise RuntimeError(f"CUDA = {cuda} not supported!")
if v <= V("2.1.0"):
raise RuntimeError(f"Torch = {v} too old!")
elif v <= V("2.1.1"):
x = "cu{}{}-torch211"
elif v <= V("2.1.2"):
x = "cu{}{}-torch212"
elif v < V("2.3.0"):
x = "cu{}{}-torch220"
elif v < V("2.4.0"):
x = "cu{}{}-torch230"
elif v < V("2.5.0"):
x = "cu{}{}-torch240"
elif v < V("2.5.1"):
x = "cu{}{}-torch250"
elif v <= V("2.5.1"):
x = "cu{}{}-torch251"
elif v < V("2.7.0"):
x = "cu{}{}-torch260"
elif v < V("2.7.9"):
x = "cu{}{}-torch270"
elif v < V("2.8.0"):
x = "cu{}{}-torch271"
elif v < V("2.8.9"):
x = "cu{}{}-torch280"
elif v < V("2.9.1"):
x = "cu{}{}-torch290"
elif v < V("2.9.2"):
x = "cu{}{}-torch291"
elif v < V("2.10.1"):
x = "cu{}{}-torch2100"
else:
raise RuntimeError(f"Torch = {v} too new!")
if v > V("2.6.9") and cuda not in ("11.8", "12.6", "12.8", "13.0"):
raise RuntimeError(f"CUDA = {cuda} not supported!")
if v >= V("2.10.0") and cuda not in ("12.6", "12.8", "13.0"):
raise RuntimeError(
f"Torch 2.10 requires CUDA 12.6, 12.8, or 13.0! Got CUDA = {cuda}"
)
x = x.format(
cuda.replace(".", ""), "-ampere" if False else ""
) # is_ampere is broken due to flash-attn
print(
f'pip install --upgrade pip && pip install --no-deps git+https://github.com/unslothai/unsloth-zoo.git && pip install "unsloth[{x}] @ git+https://github.com/unslothai/unsloth.git" --no-build-isolation'
)

1597
unsloth/chat_templates.py
View file

File diff suppressed because it is too large Load diff

2
unsloth/dataprep/raw_text.py
View file

@ -12,11 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import json
import csv
from typing import List, Dict, Any, Union, Optional
from datasets import Dataset
from pathlib import Path

1
unsloth/dataprep/synthetic.py
View file

@ -25,7 +25,6 @@ os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
import requests
import torch
import gc
import time
import re
from unsloth_zoo.vllm_utils import (
load_vllm,

2
unsloth/import_fixes.py
View file

@ -69,7 +69,7 @@ class HideLoggingMessage(logging.Filter):
self.text = text
def filter(self, x):
return not (self.text in x.getMessage())
return self.text not in x.getMessage()
class HidePrintMessage:

2
unsloth/kernels/cross_entropy_loss.py
View file

@ -23,12 +23,10 @@ from .utils import (
torch_gpu_device,
is_cdna,
)
from transformers.models.llama.modeling_llama import logger
from unsloth_zoo.utils import Version
from unsloth_zoo.loss_utils import (
patch_loss_functions as _patch_loss_functions,
post_patch_loss_function,
)

1
unsloth/kernels/fast_lora.py
View file

@ -18,7 +18,6 @@ from .utils import (
fast_dequantize,
QUANT_STATE,
get_lora_parameters,
get_lora_parameters_bias,
matmul_lora,
torch_amp_custom_fwd,
torch_amp_custom_bwd,

5
unsloth/kernels/flex_attention.py
View file

@ -13,8 +13,6 @@
# limitations under the License.
import torch
from functools import lru_cache
from transformers.models.llama.modeling_llama import logger
import os
torch_compile_options = {
@ -80,7 +78,8 @@ else:
# See https://github.com/pytorch-labs/attention-gym/blob/main/examples/flex_attn.ipynb
# for more examples
# BSD 3-Clause License Copyright (c) 2023, Driss Guessous, Horace He et al
import functools, math
import functools
import math
def generate_tanh_softcap(t):
def tanh_softcap(x, b, h, q_idx, kv_idx):

5
unsloth/kernels/fp8.py
View file

@ -13,11 +13,8 @@
# limitations under the License.
import os
import torch
import torch.nn as nn
import triton
import triton.language as tl
from torch.nn import functional as F
import math
from unsloth_zoo.utils import Version
from unsloth_zoo.log import logger
from unsloth_zoo.temporary_patches.common import torch_compile
@ -582,7 +579,7 @@ try:
# This check is a must for consumer grade GPUs which fail
if test_has_fbgemm():
os.environ["UNSLOTH_HAS_FBGEMM"] = "1"
logger.info(f"Using fbgemm_gpu block quantized FP8 matmul")
logger.info("Using fbgemm_gpu block quantized FP8 matmul")
fp8_block_quant_linear = fp8_fbgemm_block_linear
else:
os.environ["UNSLOTH_HAS_FBGEMM"] = "0"

1
unsloth/kernels/geglu.py
View file

@ -16,7 +16,6 @@ import triton
import triton.language as tl
import torch
from .utils import (
calculate_settings,
triton_tanh,
torch_gpu_device,
)

3
unsloth/kernels/layernorm.py
View file

@ -17,9 +17,6 @@ import triton
import triton.language as tl
import torch
from .utils import calculate_settings, torch_gpu_device
from unsloth_zoo.patching_utils import (
patch_layernorm,
)
@triton.jit

3
unsloth/kernels/moe/autotune_cache.py
View file

@ -23,9 +23,8 @@ import json
import logging
import os
import time
from typing import Dict, List, Optional, Tuple, Any
from typing import Dict, Optional, Tuple, Any
import torch
import triton
logger = logging.getLogger(__name__)

2
unsloth/kernels/moe/benchmark/utils.py
View file

@ -178,7 +178,7 @@ def save_autotune_results(autotune_cache, mode, ref_time, fused_time, results_di
for key, config in autotune_cache.items():
key = [
str(k) if not "torch" in str(k) else str(k.split("torch.")[-1]) for k in key
str(k) if "torch" not in str(k) else str(k.split("torch.")[-1]) for k in key
]
filename = "_".join(key)
save_path = f"{save_dir}/{filename}.json"

1
unsloth/kernels/moe/grouped_gemm/interface.py
View file

@ -3,7 +3,6 @@
import logging
import warnings
from dataclasses import asdict
import torch
import triton

1
unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py
View file

@ -336,7 +336,6 @@ def exceeds_smem_capacity(
def common_prune_criteria(config: triton.Config, kwargs: dict, dtype):
from ..interface import supports_tma
from .tuning import get_device_properties
smem_size = get_device_properties().SIZE_SMEM

1
unsloth/kernels/moe/grouped_gemm/kernels/backward.py
View file

@ -1,7 +1,6 @@
# SPDX-License-Identifier: GNU Affero General Public License v3.0
# Copyright 2023-present the Unsloth team. All rights reserved.
import torch
import triton
import triton.language as tl

1
unsloth/kernels/moe/grouped_gemm/kernels/forward.py
View file

@ -1,7 +1,6 @@
# SPDX-License-Identifier: GNU Affero General Public License v3.0
# Copyright 2023-present the Unsloth team. All rights reserved.
import torch
import triton
import triton.language as tl

2
unsloth/kernels/moe/tests/test_grouped_gemm.py
View file

@ -7,7 +7,6 @@ import pytest
import torch
from grouped_gemm.interface import (
grouped_gemm,
grouped_gemm_dW,
grouped_gemm_dX,
grouped_gemm_forward,
@ -582,7 +581,6 @@ def _test_grouped_gemm_backward_dX(
kernel_config_bwd_dW = KernelConfigBackward_dW()
else:
from grouped_gemm.kernels.backward import (
_autotuned_grouped_gemm_dW_kernel,
_autotuned_grouped_gemm_dX_kernel,
)
from grouped_gemm.kernels.forward import (

1
unsloth/kernels/moe/tests/test_llama4_moe.py
View file

@ -2,7 +2,6 @@
# Copyright 2023-present the Unsloth team. All rights reserved.
import argparse
import sys
from contextlib import contextmanager
from functools import partial

2
unsloth/kernels/swiglu.py
View file

@ -15,7 +15,7 @@
import triton
import triton.language as tl
import torch
from .utils import calculate_settings, torch_gpu_device
from .utils import torch_gpu_device
# signed int32 max is 2**31-1 so num_elements cannot exceed 2**31
NUM_INT32_ELEMENTS = 2**31

5
unsloth/kernels/utils.py
View file

@ -19,18 +19,13 @@ import ctypes
MAX_FUSED_SIZE: int = 65536
next_power_of_2 = triton.next_power_of_2
import functools
from typing import Optional
from ..device_type import (
is_hip,
get_device_type,
DEVICE_TYPE,
DEVICE_TYPE_TORCH,
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
)
from .fp8 import weight_dequant, fp8_linear
import functools
# torch.cuda.amp.custom_fwd is deprecated >= 2.4
import torch

49
unsloth/models/_utils.py
View file

@ -85,34 +85,25 @@ from typing import Union, Optional, List, Any, Callable, Tuple, Iterator
from platform import system as platform_system
platform_system = platform_system()
import numpy as np
import contextlib
import re
from dataclasses import dataclass, field
import functools
import textwrap
import logging
import warnings, subprocess, inspect, psutil, os, math
import warnings
import inspect
import psutil
import os
from unsloth_zoo.utils import Version, get_quant_type
from importlib.metadata import version as importlib_version
from ..device_type import (
is_hip,
get_device_type,
DEVICE_TYPE,
DEVICE_TYPE_TORCH,
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
)
from unsloth_zoo.log import logger
from unsloth_zoo.tokenizer_utils import (
patch_tokenizer as _patch_tokenizer,
)
from unsloth_zoo.rl_environments import (
check_python_modules,
create_locked_down_function,
execute_with_time_limit,
Benchmarker,
)
from unsloth_zoo.patching_utils import (
patch_compiling_bitsandbytes,
patch_layernorm,
@ -125,8 +116,6 @@ from unsloth_zoo.gradient_checkpointing import (
unsloth_offloaded_gradient_checkpoint,
patch_unsloth_gradient_checkpointing,
unpatch_unsloth_gradient_checkpointing,
Unsloth_Gradient_Checkpointer,
unsloth_gradient_checkpoint,
patch_gradient_checkpointing,
unpatch_gradient_checkpointing,
patch_unsloth_smart_gradient_checkpointing,
@ -252,7 +241,7 @@ class HideLoggingMessage(logging.Filter):
self.text = text
def filter(self, x):
return not (self.text in x.getMessage())
return self.text not in x.getMessage()
# Stop vLLM messages
@ -598,21 +587,21 @@ def patch_mistral_nemo_config(config):
try:
# Some Config files use layer_type_validation
# for eg Gemma-2, so we must import it to stop errors.
from transformers.configuration_utils import layer_type_validation
pass
except:
pass
try:
# Transformers 5.0+ uses RotaryEmbeddingConfigMixin as a base class for configs
from transformers.modeling_rope_utils import RotaryEmbeddingConfigMixin
pass
except:
pass
from transformers import __version__ as transformers_version
try:
from transformers import PreTrainedConfig
pass
except:
from transformers import PretrainedConfig
pass
model_architectures = [
"llama",
@ -711,7 +700,7 @@ from transformers.utils import is_openai_available
if is_openai_available():
try:
from openai import OpenAI
pass
except:
print("Unsloth: OpenAI failed to import - ignoring for now.")
import transformers.utils
@ -723,9 +712,7 @@ if is_openai_available():
# =============================================
# Get Flash Attention v2 if Ampere (RTX 30xx, A100)
import bitsandbytes as bnb
from transformers import AutoTokenizer
from transformers.utils.import_utils import _is_package_available
SUPPORTS_BFLOAT16 = False
@ -743,9 +730,9 @@ if DEVICE_TYPE == "cuda":
try:
try:
# See https://github.com/unslothai/unsloth/issues/1437
from flash_attn.flash_attn_interface import flash_attn_gpu
pass
except:
from flash_attn.flash_attn_interface import flash_attn_cuda
pass
HAS_FLASH_ATTENTION = True
# Also check for softcapping
@ -795,9 +782,9 @@ elif DEVICE_TYPE == "hip":
try:
try:
# See https://github.com/unslothai/unsloth/issues/1437
from flash_attn.flash_attn_interface import flash_attn_gpu
pass
except:
from flash_attn.flash_attn_interface import flash_attn_cuda
pass
HAS_FLASH_ATTENTION = True
# Also check for softcapping
@ -1148,7 +1135,7 @@ USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1"
if USE_MODELSCOPE:
if importlib.util.find_spec("modelscope") is None:
raise ImportError(
f"You are using the modelscope hub, please install modelscope by `pip install modelscope -U`"
"You are using the modelscope hub, please install modelscope by `pip install modelscope -U`"
)
import socket
@ -1166,13 +1153,10 @@ def has_internet(host = "8.8.8.8", port = 53, timeout = 3):
return True
finally:
sock.close()
except socket.error as ex:
except socket.error:
return False
import psutil
def _get_statistics(statistics = None, force_download = True):
# We log some basic stats about which environment is being used.
# We simply download a README.md file from HF - all data is made public.
@ -1339,7 +1323,6 @@ def get_statistics(local_files_only = False):
# Fixes Bitsandbytes to remove missing warnings
from transformers.utils.quantization_config import (
BitsAndBytesConfig,
QuantizationMethod,
)
BitsAndBytesConfig__init__ = inspect.getsource(BitsAndBytesConfig.__init__)

4
unsloth/models/cohere.py
View file

@ -13,7 +13,6 @@
# limitations under the License.
from .llama import *
from ._utils import __version__
from unsloth_zoo.hf_utils import dtype_from_config
from unsloth_zoo.utils import _get_dtype, Version
from ..utils.packing import get_packed_info_from_kwargs
@ -30,9 +29,6 @@ try:
CohereDecoderLayer,
CohereModel,
CohereForCausalLM,
CohereRotaryEmbedding,
apply_rotary_pos_emb,
repeat_kv,
)
except:
transformers_version = Version(transformers_version)

4
unsloth/models/falcon_h1.py
View file

@ -13,8 +13,6 @@
# limitations under the License.
from .llama import *
import os
from ._utils import __version__
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from ..utils.packing import get_packed_info_from_kwargs
@ -28,7 +26,6 @@ from ..utils.attention_dispatch import (
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
_LlamaModel_fast_forward_inference,
)
try:
@ -37,7 +34,6 @@ try:
FalconH1DecoderLayer,
FalconH1Model,
FalconH1ForCausalLM,
FalconHybridMambaAttentionDynamicCache,
)
except:
from transformers import __version__ as transformers_version

9
unsloth/models/gemma.py
View file

@ -13,14 +13,8 @@
# limitations under the License.
from .llama import *
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype, Version
from unsloth_zoo.hf_utils import dtype_from_config
from ..utils.packing import (
build_sdpa_packed_attention_mask,
build_xformers_block_causal_mask,
get_packed_info_from_kwargs,
)
import math
try:
@ -29,9 +23,6 @@ try:
GemmaDecoderLayer,
GemmaModel,
GemmaForCausalLM,
GemmaRotaryEmbedding,
apply_rotary_pos_emb,
repeat_kv,
)
except:
transformers_version = Version(transformers_version)

7
unsloth/models/gemma2.py
View file

@ -13,7 +13,6 @@
# limitations under the License.
from .llama import *
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype, Version
from unsloth_zoo.hf_utils import dtype_from_config
from ..utils.packing import get_packed_info_from_kwargs
@ -22,7 +21,6 @@ from ..utils.attention_dispatch import (
AttentionContext,
run_attention,
select_attention_backend,
SDPA,
)
from .gemma import (
GemmaFixedRotaryEmbedding,
@ -36,9 +34,6 @@ try:
Gemma2DecoderLayer,
Gemma2Model,
Gemma2ForCausalLM,
Gemma2RotaryEmbedding,
apply_rotary_pos_emb,
repeat_kv,
)
except:
transformers_version = Version(transformers_version)
@ -65,7 +60,7 @@ except:
Gemma2FlashAttention2 = Gemma2Attention
if HAS_FLASH_ATTENTION_SOFTCAPPING:
from flash_attn import flash_attn_func
pass
# Logit softcapping

9
unsloth/models/glm4_moe.py
View file

@ -25,20 +25,11 @@ Key architecture differences from Qwen3 MoE:
from .llama import *
import os
from ._utils import __version__
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
fix_prepare_inputs_for_generation,
fast_rms_layernorm_inference,
fast_swiglu_inference,
LlamaModel_fast_forward,
LlamaModel_fast_forward_inference,
CausalLM_fast_forward,
PeftModel_fast_forward,
)
import torch
import torch.nn.functional as F
from typing import Optional, Tuple
from ..kernels import fast_rms_layernorm

5
unsloth/models/granite.py
View file

@ -13,7 +13,6 @@
# limitations under the License.
from .llama import *
import os
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype, Version
from unsloth_zoo.hf_utils import dtype_from_config
@ -265,8 +264,6 @@ def GraniteDecoderLayer_fast_forward(
return outputs
from math import sqrt as math_sqrt
KV_CACHE_INCREMENT = 256 # KV Cache update size
torch_nn_functional_softmax = torch.nn.functional.softmax
torch_matmul = torch.matmul
@ -285,7 +282,7 @@ def GraniteAttention_fast_forward_inference(
):
assert (
position_embeddings is not None
), f"Granite model requires position embeddings to be specified"
), "Granite model requires position embeddings to be specified"
Xn = hidden_states
bsz, _, hd = hidden_states.size()

14
unsloth/models/llama.py
View file

@ -46,12 +46,9 @@ from unsloth_zoo.hf_utils import (
)
from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
from ..device_type import (
is_hip,
get_device_type,
DEVICE_TYPE,
DEVICE_TYPE_TORCH,
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
)
transformers_version = Version(transformers_version)
@ -93,7 +90,6 @@ except:
LlamaFlashAttention2 = LlamaAttention
from transformers import (
AutoTokenizer,
AutoModelForCausalLM,
AutoModelForSequenceClassification,
BitsAndBytesConfig,
@ -104,14 +100,16 @@ from transformers import set_seed as transformers_set_seed
from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model
from peft import PeftModelForCausalLM, PeftModelForSequenceClassification
from ..save import patch_saving_functions
import re, os, inspect, math, sys
import re
import os
import inspect
import types
try:
from huggingface_hub.utils import get_token
pass
except:
# Old HF Hub versions <= 0.0.25
from huggingface_hub.utils._token import get_token
pass
from triton import __version__ as triton_version
HAS_XFORMERS = xformers is not None
@ -2903,7 +2901,7 @@ class FastLlamaModel:
try:
assert module in accepted_modules
final_modules.append(module)
except AssertionError as e:
except AssertionError:
final_modules.append(module)
print(
"Unsloth: You added custom modules, but Unsloth hasn't optimized for this.\n"

21
unsloth/models/loader.py
View file

@ -15,20 +15,17 @@
from ._utils import (
_prepare_model_for_qat,
is_bfloat16_supported,
is_vLLM_available,
HAS_FLASH_ATTENTION,
HAS_FLASH_ATTENTION_SOFTCAPPING,
USE_MODELSCOPE,
get_transformers_model_type,
hf_login,
)
from .granite import FastGraniteModel
from .llama import FastLlamaModel, logger
from .mistral import FastMistralModel
from .qwen2 import FastQwen2Model
from .qwen3 import FastQwen3Model
from .qwen3_moe import FastQwen3MoeModel
from .cohere import FastCohereModel
from transformers import AutoConfig
from transformers import __version__ as transformers_version
from peft import PeftConfig, PeftModel
@ -38,22 +35,20 @@ from .loader_utils import (
_tag_model_with_fp8_torchao_config,
get_model_name,
)
import os, contextlib, sys
import os
import contextlib
try:
from huggingface_hub import get_token
pass
except:
try:
from huggingface_hub.utils import get_token
pass
except:
# For older versions of huggingface_hub
from huggingface_hub.utils._token import get_token
pass
from huggingface_hub import HfFileSystem
import importlib.util
from ..device_type import (
is_hip,
get_device_type,
DEVICE_TYPE,
DEVICE_TYPE_TORCH,
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
@ -82,15 +77,12 @@ if SUPPORTS_GEMMA:
if SUPPORTS_GEMMA2:
from .gemma2 import FastGemma2Model
if SUPPORTS_FALCON_H1:
from .falcon_h1 import FastFalconH1Model
pass
import torch
from ._utils import (
patch_compiling_bitsandbytes,
patch_model_and_tokenizer,
prepare_model_for_kbit_training,
apply_unsloth_gradient_checkpointing,
patch_compiled_autograd,
process_vision_info,
unsloth_compile_transformers,
fast_inference_setup,
)
@ -679,7 +671,6 @@ class FastLanguageModel(FastLlamaModel):
from ..kernels import (
patch_loss_functions,
post_patch_loss_function,
)
from .vision import FastBaseModel
from transformers import (

1
unsloth/models/loader_utils.py
View file

@ -16,7 +16,6 @@ from ..device_type import DEVICE_TYPE_TORCH
import importlib
import os
import torch
import re
import tempfile
from typing import Union
from .mapper import (

601
unsloth/models/mapper.py
View file

File diff suppressed because it is too large Load diff

2
unsloth/models/mistral.py
View file

@ -14,7 +14,6 @@
from .llama import *
import os
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from ..utils.packing import (
@ -47,7 +46,6 @@ try:
except:
MistralSdpaAttention = MistralAttention
MistralFlashAttention2 = MistralAttention
from unsloth_zoo.utils import Version, _get_dtype
def MistralAttention_fast_forward(

7
unsloth/models/qwen3.py
View file

@ -13,9 +13,7 @@
# limitations under the License.
from .llama import *
import os
from ._utils import __version__
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.utils import Version
from ..utils.packing import get_packed_info_from_kwargs
from ..utils.attention_dispatch import (
AttentionConfig,
@ -47,9 +45,6 @@ except:
f'Try `pip install --upgrade "transformers>=4.50.3"`\n'
f"to obtain the latest transformers build, then restart this session."
)
from transformers.modeling_attn_mask_utils import (
_prepare_4d_causal_attention_mask_for_sdpa,
)
# For Pytorch 2.1.1
try:

3
unsloth/models/qwen3_moe.py
View file

@ -13,8 +13,6 @@
# limitations under the License.
from .llama import *
import os
from ._utils import __version__
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
@ -43,7 +41,6 @@ from transformers.models.qwen3_moe.modeling_qwen3_moe import (
# Qwen3SdpaAttention = Qwen3Attention
# Qwen3FlashAttention2 = Qwen3Attention
# pass
from unsloth_zoo.utils import Version, _get_dtype
torch_nn_functional_softmax = torch.nn.functional.softmax

3
unsloth/models/rl.py
View file

@ -18,7 +18,6 @@ __all__ = [
]
import torch
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
import inspect
import os
import re
@ -414,8 +413,6 @@ def _wrap_grpo_generate_and_score(trainer_cls):
def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
# Patch for vLLM and Unsloth PEFT
import trl
import trl.trainer
try:
trainer = eval(f"trl.trainer.{trainer_file}")

5
unsloth/models/rl_replacements.py
View file

@ -33,12 +33,7 @@ from unsloth_zoo.log import logger
from unsloth_zoo.device_type import device_synchronize
import importlib.util
from ..device_type import (
is_hip,
get_device_type,
DEVICE_TYPE,
DEVICE_TYPE_TORCH,
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
)
import textwrap
from ._utils import _get_inference_mode_context_manager

4
unsloth/models/sentence_transformer.py
View file

@ -346,9 +346,9 @@ class FastSentenceTransformer(FastModel):
print(f"Pooling mode detected as {mode}, updating...")
return mode
except Exception as e:
except Exception:
print(
f"Failed to detect pooling mode, not a sentence-transformers model. Using default pooling mode 'mean', this may or may not work."
"Failed to detect pooling mode, not a sentence-transformers model. Using default pooling mode 'mean', this may or may not work."
)
return "mean"

30
unsloth/models/vision.py
View file

@ -17,7 +17,6 @@ from transformers import (
BitsAndBytesConfig,
AutoProcessor,
AutoTokenizer,
AutoModelForCausalLM,
)
try:
@ -61,29 +60,22 @@ from unsloth_zoo.patching_utils import patch_model_and_tokenizer
from unsloth_zoo.training_utils import prepare_model_for_training
from unsloth_zoo.utils import Version
from transformers import __version__ as transformers_version
import types
import functools
import os
import gc
import math
from typing import Optional, Tuple, List, Union
import re, inspect, sys
import contextlib
import inspect
try:
from huggingface_hub.utils import get_token
pass
except:
# Old HF Hub versions <= 0.0.25
from huggingface_hub.utils._token import get_token
pass
from ..device_type import (
is_hip,
get_device_type,
DEVICE_TYPE,
DEVICE_TYPE_TORCH,
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
)
__all__ = [
@ -107,7 +99,7 @@ PRE_COMPILE_INFERENCE = [
"gpt_oss",
]
from transformers import GenerationConfig, CompileConfig, AutoConfig
from transformers import CompileConfig, AutoConfig
try:
from transformers import PreTrainedConfig
@ -125,10 +117,6 @@ _compile_config = CompileConfig(
)
_compile_config.disable = True # Must set manually
from unsloth_zoo.vllm_utils import (
convert_lora_modules,
return_lora_modules,
)
try:
torch_compiler_set_stance = torch.compiler.set_stance
@ -532,7 +520,7 @@ class FastBaseModel:
flex_attn_impl = prefer_flex_attn_if_supported(model_class, auto_config)
default_attn_impl = "flex_attention" if flex_attn_impl else "sdpa"
if not ("attn_implementation" in kwargs):
if "attn_implementation" not in kwargs:
kwargs["attn_implementation"] = default_attn_impl
if not supports_sdpa and kwargs.get("attn_implementation") == "sdpa":
if os.environ.get("UNSLOTH_ENABLE_FLEX_ATTENTION", "0") == "0":
@ -580,13 +568,13 @@ class FastBaseModel:
if dtype == torch.bfloat16:
if float32_mixed_precision != True:
print(
f"Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.\n"
f"To enable float32 training, use `float32_mixed_precision = True` during FastLanguageModel.from_pretrained"
"Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.\n"
"To enable float32 training, use `float32_mixed_precision = True` during FastLanguageModel.from_pretrained"
)
else:
print(
f"Unsloth: Using full float32 full finetuning. "
f"To enable bfloat16 training to reduce VRAM usage by 50% albeit with a slightly higher loss, do:\n"
"Unsloth: Using full float32 full finetuning. "
"To enable bfloat16 training to reduce VRAM usage by 50% albeit with a slightly higher loss, do:\n"
"use `float32_mixed_precision = False` during FastLanguageModel.from_pretrained"
)
os.environ["UNSLOTH_BFLOAT16_MIXED_PRECISION"] = "1"

17
unsloth/save.py
View file

@ -28,19 +28,16 @@ from peft.tuners.lora import Linear4bit as Peft_Linear4bit
from peft.tuners.lora import Linear as Peft_Linear
from typing import Optional, Callable, Union, List
import sys
import requests
import torch
import os
import shutil
import pickle
import gc
from transformers.models.llama.modeling_llama import logger
from .kernels import fast_dequantize, QUANT_STATE, get_lora_parameters_bias
from .kernels import fast_dequantize, get_lora_parameters_bias
import subprocess
import psutil
import re
from transformers.models.llama.modeling_llama import logger
from .tokenizer_utils import fix_sentencepiece_gguf
from .models.loader_utils import get_model_name
from .models._utils import _convert_torchao_model
from .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPER
@ -1224,7 +1221,7 @@ def save_to_gguf(
print(
f"Unsloth: [1] Converting model into {first_conversion_dtype} GGUF format."
)
print(f"This might take 3 minutes...")
print("This might take 3 minutes...")
initial_files, is_vlm_update = convert_to_gguf(
model_name = model_name,
@ -1289,7 +1286,7 @@ def save_to_gguf(
)
all_saved_locations.append(quantized_file)
quants_created = True
except Exception as e:
except Exception:
if IS_KAGGLE_ENVIRONMENT:
raise RuntimeError(
f"Unsloth: Quantization failed for {output_location}\n"
@ -1328,7 +1325,7 @@ def save_to_gguf(
else:
want_full_precision = first_conversion in frozenset(quantization_method)
print(f"Unsloth: All GGUF conversions completed successfully!")
print("Unsloth: All GGUF conversions completed successfully!")
print(f"Generated files: {all_saved_locations}")
return all_saved_locations, want_full_precision, is_vlm
@ -2118,7 +2115,7 @@ def unsloth_push_to_hub_gguf(
cleanup_temp = False
# Step 2: Call save_pretrained_gguf to do the conversion
print(f"Unsloth: Converting model to GGUF format...")
print("Unsloth: Converting model to GGUF format...")
try:
# Call save_pretrained_gguf - it returns all the info we need
@ -2512,13 +2509,11 @@ def unsloth_convert_lora_to_ggml_and_save_locally(
)
from .models.loader_utils import get_model_name
from unsloth_zoo.saving_utils import (
merge_and_overwrite_lora,
prepare_saving,
)
from unsloth_zoo.llama_cpp import (
install_llama_cpp,
convert_to_gguf as _convert_to_gguf,
)
@ -2829,7 +2824,6 @@ def _unsloth_save_torchao_with_given_config(
AutoModelForImageTextToText,
AutoProcessor,
)
from torchao import quantize_
if isinstance(torchao_config, TorchAoConfig):
quantization_config = torchao_config
@ -2967,7 +2961,6 @@ def not_implemented_save(*args, **kwargs):
def patch_saving_functions(model, vision = False):
import inspect
import types
from typing import Callable, Optional, Union, List
# And now re add our saving methods!
if model.push_to_hub.__name__ == "unsloth_push_to_hub":

16
unsloth/tokenizer_utils.py
View file

@ -18,22 +18,11 @@ from transformers import PreTrainedTokenizerFast
import re
import os
from transformers.models.llama.modeling_llama import logger
from peft import PeftModelForCausalLM
import torch
import itertools
import collections
import numpy as np
import gc
import subprocess
import psutil
from unsloth_zoo.tokenizer_utils import (
mean_of_trained_tokens,
add_new_tokens,
fix_untrained_tokens,
)
from unsloth_zoo.training_utils import (
fix_zero_training_loss,
)
__all__ = [
@ -356,7 +345,7 @@ def fix_sentencepiece_tokenizer(
from transformers.convert_slow_tokenizer import import_protobuf
sentencepiece_model_pb2 = import_protobuf()
except Exception as e:
except Exception:
try:
import google.protobuf
from unsloth_zoo.utils import Version
@ -896,7 +885,6 @@ def check_tokenizer(
return convert_to_fast_tokenizer(tokenizer)
import inspect
from inspect import getsource
import trl
import trl.trainer.sft_trainer
@ -941,7 +929,7 @@ def patch_sft_trainer_tokenizer():
Patches the trainer with changes
"""
try:
sft_trainer = eval(f"trl.trainer.sft_trainer.SFTTrainer")
sft_trainer = eval("trl.trainer.sft_trainer.SFTTrainer")
except:
return
all_imports = dir(trl.trainer.sft_trainer)

5
unsloth/trainer.py
View file

@ -14,16 +14,11 @@
import logging
import os
import psutil
import warnings
from dataclasses import dataclass, field
from typing import Optional
from functools import wraps
import trl
import inspect
from trl import SFTTrainer
from . import is_bfloat16_supported
from unsloth.utils import (
configure_padding_free,
configure_sample_packing,