ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Full finetuning and other fixes

Browse source
This commit is contained in:
Daniel Han 2025-03-09 22:52:34 -07:00
commit a585536a3c
7 changed files with 188 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

4
pyproject.toml
View file

@ -40,7 +40,7 @@ triton = [
]
huggingface = [
"unsloth_zoo>=2025.3.8",
"unsloth_zoo>=2025.3.9",
"packaging",
"tyro",
"transformers>=4.46.1,!=4.47.0",
@ -354,7 +354,7 @@ colab-ampere-torch220 = [
"flash-attn>=2.6.3",
]
colab-new = [
"unsloth_zoo>=2025.3.8",
"unsloth_zoo>=2025.3.9",
"packaging",
"tyro",
"transformers>=4.46.1,!=4.47.0",

17
unsloth/__init__.py
View file

@ -198,14 +198,19 @@ pass
# Check for unsloth_zoo
try:
unsloth_zoo_version = importlib_version("unsloth_zoo")
if Version(unsloth_zoo_version) < Version("2025.3.8"):
try:
os.system("pip install --upgrade --no-cache-dir --no-deps unsloth_zoo")
except:
if Version(unsloth_zoo_version) < Version("2025.3.9"):
print(
"Unsloth: Updating Unsloth-Zoo utilies to the latest version.\n"\
"To disable this, set os.environ['UNSLOTH_DISABLE_AUTO_UPDATES'] = '1'"
)
if os.environ.get("UNSLOTH_DISABLE_AUTO_UPDATES", "0") == "0":
try:
os.system("pip install --upgrade --no-cache-dir --no-deps --user unsloth_zoo")
os.system("pip install --upgrade --no-cache-dir --no-deps unsloth_zoo")
except:
raise ImportError("Unsloth: Please update unsloth_zoo via `pip install --upgrade --no-cache-dir --no-deps unsloth_zoo`")
try:
os.system("pip install --upgrade --no-cache-dir --no-deps --user unsloth_zoo")
except:
raise ImportError("Unsloth: Please update unsloth_zoo via `pip install --upgrade --no-cache-dir --no-deps unsloth_zoo`")
import unsloth_zoo
except:
raise ImportError("Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo`")

76
unsloth/models/_utils.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "2025.3.9"
__version__ = "2025.3.10"
__all__ = [
"SUPPORTS_BFLOAT16",
@ -109,6 +109,9 @@ from unsloth_zoo.compiler import (
get_transformers_model_type,
unsloth_compile_transformers as _unsloth_compile_transformers,
)
from unsloth_zoo.training_utils import (
prepare_model_for_training,
)
# =============================================
# Disable some warnings which can get annoying
@ -509,67 +512,16 @@ def prepare_model_for_kbit_training(
use_gradient_checkpointing : Optional = True,
use_reentrant : Optional[bool] = True,
) -> Any:
"""
Calculates where to place the gradient checkpoints given n_layers.
We also freeze all other layers's gradients
Args:
model: Any LlamaModel with layers.
use_gradient_checkpointing (`bool`, *optional*):
Default enabled. Provides memory savings by not saving all activations,
but only some.
use_reentrant (`bool`, *optional*):
https://github.com/pytorch/pytorch/blob/main/torch/utils/checkpoint.py#L354
Optimal gradient checkpointing algorithm which will be the default in
future Pytorch versions.
"""
# Freeze all parameters except LoRA
with torch.no_grad():
for name, param in model.named_parameters():
if ".lora_A." in name or ".lora_B." in name or ".lora_magnitude_vector" in name:
param.requires_grad_(True)
# Also must be in float32!
if param.dtype != torch.float32:
name = name.replace("base_model", "model", 1)
layer_number = re.search(r"\.[\d]{1,}\.", name).group(0)
name = name.replace(layer_number, f"[{layer_number[1:-1]}].")
name = name.replace(".weight", "", 1)
exec(f"{name}.to(torch.float32)")
pass
else:
param.requires_grad_(False)
pass
pass
# Gradient checkpointing!
if use_gradient_checkpointing == "unsloth":
# Saves VRAM!
original_model = model
while hasattr(original_model, "model"):
original_model._offloaded_gradient_checkpointing = True
original_model = original_model.model
pass
original_model._offloaded_gradient_checkpointing = True
model.gradient_checkpointing_enable()
elif use_gradient_checkpointing == True:
model.gradient_checkpointing_enable()
pass
# If use_reentrant = True which is the Pytorch default, we just make the input requires_grad.
if use_reentrant:
if hasattr(model, "enable_input_require_grads"):
model.enable_input_require_grads()
else:
def make_inputs_require_grad(module, input, output):
output.requires_grad_(True)
model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
pass
return model
return prepare_model_for_training(
model = model,
use_gradient_checkpointing = use_gradient_checkpointing,
use_reentrant = use_reentrant,
full_finetuning = False,
train_layernorms = False,
train_embedding = False,
train_lm_head = False,
float32_mixed_precision = True,
)
pass
# =============================================

85
unsloth/models/loader.py
View file

@ -73,6 +73,8 @@ class FastLanguageModel(FastLlamaModel):
max_seq_length = None,
dtype = None,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
token = None,
device_map = "sequential",
rope_scaling = None,
@ -91,6 +93,28 @@ class FastLanguageModel(FastLlamaModel):
disable_log_stats = True,
*args, **kwargs,
):
if load_in_8bit or full_finetuning:
return FastModel.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length, # [TODO] No effect
dtype = dtype,
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
token = token,
device_map = device_map,
rope_scaling = rope_scaling, # [TODO] No effect
fix_tokenizer = fix_tokenizer, # [TODO] No effect
trust_remote_code = trust_remote_code,
use_gradient_checkpointing = use_gradient_checkpointing,
resize_model_vocab = resize_model_vocab, # [TODO] No effect
revision = revision,
return_logits = return_logits, # Return logits
fullgraph = fullgraph, # No graph breaks
use_exact_model_name = use_exact_model_name,
*args, **kwargs,
)
pass
if token is None: token = get_token()
assert (dtype is None or dtype == torch.float16 or dtype == torch.bfloat16)
@ -150,7 +174,7 @@ class FastLanguageModel(FastLlamaModel):
# Old transformers versions check
both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32
# New transformers need to check manually.
if SUPPORTS_LLAMA32:
# Check if folder exists locally
@ -261,15 +285,31 @@ class FastLanguageModel(FastLlamaModel):
dispatch_model = FastGemma2Model
elif model_type == "qwen2":
dispatch_model = FastQwen2Model
elif model_type == "cohere":
dispatch_model = FastCohereModel
elif model_type == "granite":
dispatch_model = FastGraniteModel
# Temporary disable optimized Cohere until errors match
# elif model_type == "cohere":
# dispatch_model = FastCohereModel
# Temporary disable optimized Granite until errors match
# elif model_type == "granite":
# dispatch_model = FastGraniteModel
else:
raise NotImplementedError(
f"Unsloth: {model_name} not supported yet!\n"\
"Maybe you're doing vision finetuning? Please use FastVisionModel instead!\n"\
"Otherwise, make an issue to https://github.com/unslothai/unsloth!",
return FastModel.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length, # [TODO] No effect
dtype = dtype,
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
token = token,
device_map = device_map,
rope_scaling = rope_scaling, # [TODO] No effect
fix_tokenizer = fix_tokenizer, # [TODO] No effect
trust_remote_code = trust_remote_code,
use_gradient_checkpointing = use_gradient_checkpointing,
resize_model_vocab = resize_model_vocab, # [TODO] No effect
revision = revision,
return_logits = return_logits, # Return logits
fullgraph = fullgraph, # No graph breaks
use_exact_model_name = use_exact_model_name,
*args, **kwargs,
)
pass
@ -284,6 +324,11 @@ class FastLanguageModel(FastLlamaModel):
pass
if fast_inference:
import platform
if platform.system().lower() == 'windows':
print("Unsloth: vLLM does not work in Windows! Will use Unsloth inference!")
fast_inference = False
pass
from unsloth_zoo.vllm_utils import (
patch_vllm,
vllm_dynamic_quant_supported,
@ -392,6 +437,8 @@ class FastModel(FastBaseModel):
max_seq_length = None, # [TODO] No effect
dtype = None,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
token = None,
device_map = "sequential",
rope_scaling = None, # [TODO] No effect
@ -413,6 +460,21 @@ class FastModel(FastBaseModel):
if use_gradient_checkpointing == "unsloth":
patch_unsloth_smart_gradient_checkpointing(dtype = dtype)
if full_finetuning and (load_in_4bit or load_in_8bit):
print("Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA.")
load_in_4bit = False
load_in_8bit = False
pass
if load_in_4bit and load_in_8bit:
raise RuntimeError("Unsloth: Can only load in 4bit or 8bit, not both!")
if load_in_4bit: pass
elif load_in_8bit: pass
elif not load_in_4bit and not load_in_8bit and not full_finetuning:
print("Unsloth: LoRA, QLoRA and full finetuning all not selected. Switching to QLoRA.")
load_in_4bit = True
pass
old_model_name = model_name
if not use_exact_model_name:
model_name = get_model_name(model_name, load_in_4bit)
@ -569,6 +631,8 @@ class FastModel(FastBaseModel):
max_seq_length = max_seq_length,
dtype = _get_dtype(dtype),
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
full_finetuning = full_finetuning,
token = token,
device_map = device_map,
trust_remote_code = trust_remote_code,
@ -576,6 +640,7 @@ class FastModel(FastBaseModel):
model_types = model_types,
tokenizer_name = tokenizer_name,
auto_model = auto_model,
use_gradient_checkpointing = use_gradient_checkpointing,
*args, **kwargs,
)
@ -623,7 +688,7 @@ class FastModel(FastBaseModel):
trust_remote_code = trust_remote_code,
)
# Patch it as well!
model = FastBaseModel.patch_peft_model(model, use_gradient_checkpointing)
model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing)
pass
return model, tokenizer
pass

17
unsloth/models/mapper.py
View file

@ -492,6 +492,18 @@ __INT_TO_FLOAT_MAPPER = \
"unsloth/Qwen2-VL-72B-Instruct",
"Qwen/Qwen2-VL-72B-Instruct",
),
"unsloth/Qwen2-VL-2B-bnb-4bit" : (
"unsloth/Qwen2-VL-2B",
"Qwen/Qwen2-VL-2B",
),
"unsloth/Qwen2-VL-7B-bnb-4bit" : (
"unsloth/Qwen2-VL-7B",
"Qwen/Qwen2-VL-7B",
),
"unsloth/Qwen2-VL-72B-bnb-4bit" : (
"unsloth/Qwen2-VL-72B",
"Qwen/Qwen2-VL-72B",
),
"unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit" : (
"unsloth/Llama-3.2-11B-Vision-Instruct",
"meta-llama/Llama-3.2-11B-Vision-Instruct",
@ -626,6 +638,11 @@ __INT_TO_FLOAT_MAPPER = \
"Qwen/QwQ-32B",
"unsloth/QwQ-32B-bnb-4bit",
),
"unsloth/Phi-4-mini-instruct-unsloth-bnb-4bit" : (
"unsloth/Phi-4-mini-instruct",
"microsoft/Phi-4-mini-instruct",
"unsloth/Phi-4-mini-instruct",
),
}
INT_TO_FLOAT_MAPPER = {}

14
unsloth/models/rl.py
View file

@ -234,6 +234,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
mixed_precision = \
"use_bf16 = getattr(args, 'bf16', False)\n"\
"use_fp16 = getattr(args, 'fp16', False)\n"\
"mixed_precision_dtype = os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32')\n"\
"dtype = getattr(model.config, 'torch_dtype', None)\n"\
"if dtype is None: dtype = model.get_input_embeddings().dtype\n"\
"from unsloth_zoo.utils import _get_dtype\n"\
@ -241,10 +242,14 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
"float16 = dtype == torch.float16\n"\
"if float16 and use_bf16: raise TypeError('Unsloth: Model is in float16 precision but you want to use bfloat16 precision. Set fp16 to `True` and bf16 to `False`')\n"\
"if not float16 and use_fp16: raise TypeError('Unsloth: Model is in bfloat16 precision but you want to use float16 precision. Set fp16 to `False` and bf16 to `True`')\n"\
"if not use_bf16 and not use_fp16:\n"\
"if (not use_bf16 and not use_fp16) and mixed_precision_dtype == 'float32':\n"\
" args.fp16 = float16\n"\
" args.bf16 = not float16\n"\
" os.environ['ACCELERATE_MIXED_PRECISION'] = 'fp16' if float16 else 'bf16'\n"
"elif mixed_precision_dtype == 'bfloat16':\n"\
" args.fp16 = False\n"\
" args.bf16 = False\n"\
" os.environ['ACCELERATE_MIXED_PRECISION'] = 'no'\n"
extra_args += mixed_precision
pass
@ -280,7 +285,12 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
"bf16_full_eval = getattr(args, 'bf16_full_eval', False)\n"\
"if args.fp16 and bf16_full_eval: args.bf16_full_eval = False; args.fp16_full_eval = True\n"\
"if args.bf16 and fp16_full_eval: args.bf16_full_eval = True; args.fp16_full_eval = False\n"\
"if not bf16_full_eval and not fp16_full_eval: args.bf16_full_eval = args.bf16; args.fp16_full_eval = args.fp16\n"
"if os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32') == 'bfloat16':\n"\
" args.bf16_full_eval = True\n"\
" args.fp16_full_eval = False\n"\
"elif not bf16_full_eval and not fp16_full_eval:\n"\
" args.bf16_full_eval = args.bf16\n"\
" args.fp16_full_eval = args.fp16\n"
extra_args += eval_changes
pass

65
unsloth/models/vision.py
View file

@ -35,6 +35,7 @@ from unsloth_zoo.peft_utils import (
from triton import __version__ as triton_version
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.patching_utils import patch_model_and_tokenizer
from unsloth_zoo.training_utils import prepare_model_for_training
import types
import functools
@ -90,12 +91,15 @@ class FastBaseModel:
max_seq_length = None,
dtype = None,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
token = None,
device_map = "sequential",
trust_remote_code = False,
model_types = None,
tokenizer_name = None,
auto_model = AutoModelForVision2Seq,
use_gradient_checkpointing = "unsloth",
**kwargs,
):
if trust_remote_code:
@ -141,6 +145,14 @@ class FastBaseModel:
assert(dtype == torch.float16 or dtype == torch.bfloat16 or dtype == torch.float32)
bnb_config = None
if full_finetuning and (load_in_4bit or load_in_8bit):
print("Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA.")
load_in_4bit = False
load_in_8bit = False
pass
if load_in_4bit and load_in_8bit:
raise RuntimeError("Unsloth: Can only load in 4bit or 8bit, not both!")
if load_in_4bit:
bnb_config = BitsAndBytesConfig(
load_in_4bit = True,
@ -149,6 +161,21 @@ class FastBaseModel:
bnb_4bit_compute_dtype = dtype,
llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES,
)
elif load_in_8bit:
bnb_config = BitsAndBytesConfig(
load_in_8bit = True,
llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES,
)
elif not load_in_4bit and not load_in_8bit and not full_finetuning:
print("Unsloth: LoRA, QLoRA and full finetuning all not selected. Switching to QLoRA.")
load_in_4bit = True
pass
if full_finetuning:
if dtype == torch.bfloat16:
print("Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.")
else:
print("Unsloth: Float16 full finetuning uses more memory since we upcast weights to float32.")
pass
kwargs.pop("attn_implementation", None); # No need since we auto call it
@ -209,18 +236,29 @@ class FastBaseModel:
while hasattr(m, "model"):
m._saved_temp_tokenizer = tokenizer
# Also set is_loaded_in_8bit to disable incorrect DDP
m.is_loaded_in_8bit = True
m.is_loaded_in_8bit = True if not full_finetuning else False
m = m.model
pass
m._saved_temp_tokenizer = tokenizer
# Also set is_loaded_in_8bit to disable incorrect DDP
m.is_loaded_in_8bit = True
m.is_loaded_in_8bit = True if not full_finetuning else False
# Patch generate
if model.generate.__name__ != "unsloth_base_fast_generate":
model._old_generate = model.generate
unsloth_base_fast_generate.__doc__ = model._old_generate.__doc__
model.generate = types.MethodType(unsloth_base_fast_generate, model)
# Post patches
model = FastBaseModel.post_patch_model(
model,
use_gradient_checkpointing = use_gradient_checkpointing,
)
# Clear deleted GPU items
for _ in range(3):
gc.collect()
torch.cuda.empty_cache()
pass
return model, tokenizer
pass
@ -299,7 +337,7 @@ class FastBaseModel:
# Enable gradients on modules which are trainable
requires_grad_for_gradient_checkpointing(model)
model = FastBaseModel.patch_peft_model(model, use_gradient_checkpointing)
model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing)
# Clear deleted GPU items
for _ in range(3):
@ -316,7 +354,7 @@ class FastBaseModel:
@staticmethod
def patch_peft_model(
def post_patch_model(
model,
use_gradient_checkpointing = True,
):
@ -325,11 +363,22 @@ class FastBaseModel:
"Unsloth: Your model needs to call `.get_peft_model` first!"
)
pass
full_finetuning = hasattr(model.config, "quantization_config", None) is not None
model = prepare_model_for_kbit_training(
float32_mixed_precision = True
if _get_dtype(model.config.torch_dtype) == torch.bfloat16:
# Use bfloat16 precision for full finetuning
float32_mixed_precision = False
model = prepare_model_for_training(
model,
use_gradient_checkpointing = use_gradient_checkpointing,
use_reentrant = True,
use_reentrant = True,
full_finetuning = full_finetuning,
train_layernorms = full_finetuning,
train_embedding = full_finetuning,
train_lm_head = full_finetuning,
float32_mixed_precision = float32_mixed_precision,
)
from transformers.trainer import Trainer
@ -350,14 +399,14 @@ class FastBaseModel:
m._saved_temp_tokenizer.tokenizer.padding_side = "right"
pass
# Also set is_loaded_in_8bit to disable incorrect DDP
m.is_loaded_in_8bit = True
m.is_loaded_in_8bit = True if not full_finetuning else False
m = m.model
pass
if hasattr(m, "_saved_temp_tokenizer"):
m._saved_temp_tokenizer.tokenizer.padding_side = "right"
pass
# Also set is_loaded_in_8bit to disable incorrect DDP
m.is_loaded_in_8bit = True
m.is_loaded_in_8bit = True if not full_finetuning else False
# Clear deleted GPU items
for _ in range(3):