diff --git a/pyproject.toml b/pyproject.toml index 5b9dc8bb57..667901e76f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.3.8", + "unsloth_zoo>=2025.3.9", "packaging", "tyro", "transformers>=4.46.1,!=4.47.0", @@ -354,7 +354,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.3.8", + "unsloth_zoo>=2025.3.9", "packaging", "tyro", "transformers>=4.46.1,!=4.47.0", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 5bbb85d520..9bcdd5cf64 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -198,14 +198,19 @@ pass # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2025.3.8"): - try: - os.system("pip install --upgrade --no-cache-dir --no-deps unsloth_zoo") - except: + if Version(unsloth_zoo_version) < Version("2025.3.9"): + print( + "Unsloth: Updating Unsloth-Zoo utilies to the latest version.\n"\ + "To disable this, set os.environ['UNSLOTH_DISABLE_AUTO_UPDATES'] = '1'" + ) + if os.environ.get("UNSLOTH_DISABLE_AUTO_UPDATES", "0") == "0": try: - os.system("pip install --upgrade --no-cache-dir --no-deps --user unsloth_zoo") + os.system("pip install --upgrade --no-cache-dir --no-deps unsloth_zoo") except: - raise ImportError("Unsloth: Please update unsloth_zoo via `pip install --upgrade --no-cache-dir --no-deps unsloth_zoo`") + try: + os.system("pip install --upgrade --no-cache-dir --no-deps --user unsloth_zoo") + except: + raise ImportError("Unsloth: Please update unsloth_zoo via `pip install --upgrade --no-cache-dir --no-deps unsloth_zoo`") import unsloth_zoo except: raise ImportError("Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo`") diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 25fa788099..50dbe7cae6 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.3.9" +__version__ = "2025.3.10" __all__ = [ "SUPPORTS_BFLOAT16", @@ -109,6 +109,9 @@ from unsloth_zoo.compiler import ( get_transformers_model_type, unsloth_compile_transformers as _unsloth_compile_transformers, ) +from unsloth_zoo.training_utils import ( + prepare_model_for_training, +) # ============================================= # Disable some warnings which can get annoying @@ -509,67 +512,16 @@ def prepare_model_for_kbit_training( use_gradient_checkpointing : Optional = True, use_reentrant : Optional[bool] = True, ) -> Any: - """ - Calculates where to place the gradient checkpoints given n_layers. - We also freeze all other layers's gradients - - Args: - model: Any LlamaModel with layers. - use_gradient_checkpointing (`bool`, *optional*): - Default enabled. Provides memory savings by not saving all activations, - but only some. - use_reentrant (`bool`, *optional*): - https://github.com/pytorch/pytorch/blob/main/torch/utils/checkpoint.py#L354 - Optimal gradient checkpointing algorithm which will be the default in - future Pytorch versions. - """ - - # Freeze all parameters except LoRA - with torch.no_grad(): - for name, param in model.named_parameters(): - if ".lora_A." in name or ".lora_B." in name or ".lora_magnitude_vector" in name: - param.requires_grad_(True) - # Also must be in float32! - if param.dtype != torch.float32: - name = name.replace("base_model", "model", 1) - layer_number = re.search(r"\.[\d]{1,}\.", name).group(0) - name = name.replace(layer_number, f"[{layer_number[1:-1]}].") - name = name.replace(".weight", "", 1) - exec(f"{name}.to(torch.float32)") - pass - else: - param.requires_grad_(False) - pass - pass - - # Gradient checkpointing! - if use_gradient_checkpointing == "unsloth": - - # Saves VRAM! - original_model = model - while hasattr(original_model, "model"): - original_model._offloaded_gradient_checkpointing = True - original_model = original_model.model - pass - original_model._offloaded_gradient_checkpointing = True - - model.gradient_checkpointing_enable() - - elif use_gradient_checkpointing == True: - model.gradient_checkpointing_enable() - pass - - # If use_reentrant = True which is the Pytorch default, we just make the input requires_grad. - if use_reentrant: - if hasattr(model, "enable_input_require_grads"): - model.enable_input_require_grads() - else: - def make_inputs_require_grad(module, input, output): - output.requires_grad_(True) - model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) - pass - - return model + return prepare_model_for_training( + model = model, + use_gradient_checkpointing = use_gradient_checkpointing, + use_reentrant = use_reentrant, + full_finetuning = False, + train_layernorms = False, + train_embedding = False, + train_lm_head = False, + float32_mixed_precision = True, + ) pass # ============================================= diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 7062c481cf..445658b77d 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -73,6 +73,8 @@ class FastLanguageModel(FastLlamaModel): max_seq_length = None, dtype = None, load_in_4bit = True, + load_in_8bit = False, + full_finetuning = False, token = None, device_map = "sequential", rope_scaling = None, @@ -91,6 +93,28 @@ class FastLanguageModel(FastLlamaModel): disable_log_stats = True, *args, **kwargs, ): + if load_in_8bit or full_finetuning: + return FastModel.from_pretrained( + model_name = model_name, + max_seq_length = max_seq_length, # [TODO] No effect + dtype = dtype, + load_in_4bit = load_in_4bit, + load_in_8bit = load_in_8bit, + token = token, + device_map = device_map, + rope_scaling = rope_scaling, # [TODO] No effect + fix_tokenizer = fix_tokenizer, # [TODO] No effect + trust_remote_code = trust_remote_code, + use_gradient_checkpointing = use_gradient_checkpointing, + resize_model_vocab = resize_model_vocab, # [TODO] No effect + revision = revision, + return_logits = return_logits, # Return logits + fullgraph = fullgraph, # No graph breaks + use_exact_model_name = use_exact_model_name, + *args, **kwargs, + ) + pass + if token is None: token = get_token() assert (dtype is None or dtype == torch.float16 or dtype == torch.bfloat16) @@ -150,7 +174,7 @@ class FastLanguageModel(FastLlamaModel): # Old transformers versions check both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32 - + # New transformers need to check manually. if SUPPORTS_LLAMA32: # Check if folder exists locally @@ -261,15 +285,31 @@ class FastLanguageModel(FastLlamaModel): dispatch_model = FastGemma2Model elif model_type == "qwen2": dispatch_model = FastQwen2Model - elif model_type == "cohere": - dispatch_model = FastCohereModel - elif model_type == "granite": - dispatch_model = FastGraniteModel + # Temporary disable optimized Cohere until errors match + # elif model_type == "cohere": + # dispatch_model = FastCohereModel + # Temporary disable optimized Granite until errors match + # elif model_type == "granite": + # dispatch_model = FastGraniteModel else: - raise NotImplementedError( - f"Unsloth: {model_name} not supported yet!\n"\ - "Maybe you're doing vision finetuning? Please use FastVisionModel instead!\n"\ - "Otherwise, make an issue to https://github.com/unslothai/unsloth!", + return FastModel.from_pretrained( + model_name = model_name, + max_seq_length = max_seq_length, # [TODO] No effect + dtype = dtype, + load_in_4bit = load_in_4bit, + load_in_8bit = load_in_8bit, + token = token, + device_map = device_map, + rope_scaling = rope_scaling, # [TODO] No effect + fix_tokenizer = fix_tokenizer, # [TODO] No effect + trust_remote_code = trust_remote_code, + use_gradient_checkpointing = use_gradient_checkpointing, + resize_model_vocab = resize_model_vocab, # [TODO] No effect + revision = revision, + return_logits = return_logits, # Return logits + fullgraph = fullgraph, # No graph breaks + use_exact_model_name = use_exact_model_name, + *args, **kwargs, ) pass @@ -284,6 +324,11 @@ class FastLanguageModel(FastLlamaModel): pass if fast_inference: + import platform + if platform.system().lower() == 'windows': + print("Unsloth: vLLM does not work in Windows! Will use Unsloth inference!") + fast_inference = False + pass from unsloth_zoo.vllm_utils import ( patch_vllm, vllm_dynamic_quant_supported, @@ -392,6 +437,8 @@ class FastModel(FastBaseModel): max_seq_length = None, # [TODO] No effect dtype = None, load_in_4bit = True, + load_in_8bit = False, + full_finetuning = False, token = None, device_map = "sequential", rope_scaling = None, # [TODO] No effect @@ -413,6 +460,21 @@ class FastModel(FastBaseModel): if use_gradient_checkpointing == "unsloth": patch_unsloth_smart_gradient_checkpointing(dtype = dtype) + if full_finetuning and (load_in_4bit or load_in_8bit): + print("Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA.") + load_in_4bit = False + load_in_8bit = False + pass + + if load_in_4bit and load_in_8bit: + raise RuntimeError("Unsloth: Can only load in 4bit or 8bit, not both!") + if load_in_4bit: pass + elif load_in_8bit: pass + elif not load_in_4bit and not load_in_8bit and not full_finetuning: + print("Unsloth: LoRA, QLoRA and full finetuning all not selected. Switching to QLoRA.") + load_in_4bit = True + pass + old_model_name = model_name if not use_exact_model_name: model_name = get_model_name(model_name, load_in_4bit) @@ -569,6 +631,8 @@ class FastModel(FastBaseModel): max_seq_length = max_seq_length, dtype = _get_dtype(dtype), load_in_4bit = load_in_4bit, + load_in_8bit = load_in_8bit, + full_finetuning = full_finetuning, token = token, device_map = device_map, trust_remote_code = trust_remote_code, @@ -576,6 +640,7 @@ class FastModel(FastBaseModel): model_types = model_types, tokenizer_name = tokenizer_name, auto_model = auto_model, + use_gradient_checkpointing = use_gradient_checkpointing, *args, **kwargs, ) @@ -623,7 +688,7 @@ class FastModel(FastBaseModel): trust_remote_code = trust_remote_code, ) # Patch it as well! - model = FastBaseModel.patch_peft_model(model, use_gradient_checkpointing) + model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing) pass return model, tokenizer pass diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index a2e609f203..001152183e 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -492,6 +492,18 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/Qwen2-VL-72B-Instruct", "Qwen/Qwen2-VL-72B-Instruct", ), + "unsloth/Qwen2-VL-2B-bnb-4bit" : ( + "unsloth/Qwen2-VL-2B", + "Qwen/Qwen2-VL-2B", + ), + "unsloth/Qwen2-VL-7B-bnb-4bit" : ( + "unsloth/Qwen2-VL-7B", + "Qwen/Qwen2-VL-7B", + ), + "unsloth/Qwen2-VL-72B-bnb-4bit" : ( + "unsloth/Qwen2-VL-72B", + "Qwen/Qwen2-VL-72B", + ), "unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit" : ( "unsloth/Llama-3.2-11B-Vision-Instruct", "meta-llama/Llama-3.2-11B-Vision-Instruct", @@ -626,6 +638,11 @@ __INT_TO_FLOAT_MAPPER = \ "Qwen/QwQ-32B", "unsloth/QwQ-32B-bnb-4bit", ), + "unsloth/Phi-4-mini-instruct-unsloth-bnb-4bit" : ( + "unsloth/Phi-4-mini-instruct", + "microsoft/Phi-4-mini-instruct", + "unsloth/Phi-4-mini-instruct", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index f13f7ef61b..cf5eb9cfe7 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -234,6 +234,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): mixed_precision = \ "use_bf16 = getattr(args, 'bf16', False)\n"\ "use_fp16 = getattr(args, 'fp16', False)\n"\ + "mixed_precision_dtype = os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32')\n"\ "dtype = getattr(model.config, 'torch_dtype', None)\n"\ "if dtype is None: dtype = model.get_input_embeddings().dtype\n"\ "from unsloth_zoo.utils import _get_dtype\n"\ @@ -241,10 +242,14 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "float16 = dtype == torch.float16\n"\ "if float16 and use_bf16: raise TypeError('Unsloth: Model is in float16 precision but you want to use bfloat16 precision. Set fp16 to `True` and bf16 to `False`')\n"\ "if not float16 and use_fp16: raise TypeError('Unsloth: Model is in bfloat16 precision but you want to use float16 precision. Set fp16 to `False` and bf16 to `True`')\n"\ - "if not use_bf16 and not use_fp16:\n"\ + "if (not use_bf16 and not use_fp16) and mixed_precision_dtype == 'float32':\n"\ " args.fp16 = float16\n"\ " args.bf16 = not float16\n"\ " os.environ['ACCELERATE_MIXED_PRECISION'] = 'fp16' if float16 else 'bf16'\n" + "elif mixed_precision_dtype == 'bfloat16':\n"\ + " args.fp16 = False\n"\ + " args.bf16 = False\n"\ + " os.environ['ACCELERATE_MIXED_PRECISION'] = 'no'\n" extra_args += mixed_precision pass @@ -280,7 +285,12 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "bf16_full_eval = getattr(args, 'bf16_full_eval', False)\n"\ "if args.fp16 and bf16_full_eval: args.bf16_full_eval = False; args.fp16_full_eval = True\n"\ "if args.bf16 and fp16_full_eval: args.bf16_full_eval = True; args.fp16_full_eval = False\n"\ - "if not bf16_full_eval and not fp16_full_eval: args.bf16_full_eval = args.bf16; args.fp16_full_eval = args.fp16\n" + "if os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32') == 'bfloat16':\n"\ + " args.bf16_full_eval = True\n"\ + " args.fp16_full_eval = False\n"\ + "elif not bf16_full_eval and not fp16_full_eval:\n"\ + " args.bf16_full_eval = args.bf16\n"\ + " args.fp16_full_eval = args.fp16\n" extra_args += eval_changes pass diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index ff07ef6917..56da240b40 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -35,6 +35,7 @@ from unsloth_zoo.peft_utils import ( from triton import __version__ as triton_version from unsloth_zoo.utils import _get_dtype from unsloth_zoo.patching_utils import patch_model_and_tokenizer +from unsloth_zoo.training_utils import prepare_model_for_training import types import functools @@ -90,12 +91,15 @@ class FastBaseModel: max_seq_length = None, dtype = None, load_in_4bit = True, + load_in_8bit = False, + full_finetuning = False, token = None, device_map = "sequential", trust_remote_code = False, model_types = None, tokenizer_name = None, auto_model = AutoModelForVision2Seq, + use_gradient_checkpointing = "unsloth", **kwargs, ): if trust_remote_code: @@ -141,6 +145,14 @@ class FastBaseModel: assert(dtype == torch.float16 or dtype == torch.bfloat16 or dtype == torch.float32) bnb_config = None + if full_finetuning and (load_in_4bit or load_in_8bit): + print("Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA.") + load_in_4bit = False + load_in_8bit = False + pass + + if load_in_4bit and load_in_8bit: + raise RuntimeError("Unsloth: Can only load in 4bit or 8bit, not both!") if load_in_4bit: bnb_config = BitsAndBytesConfig( load_in_4bit = True, @@ -149,6 +161,21 @@ class FastBaseModel: bnb_4bit_compute_dtype = dtype, llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES, ) + elif load_in_8bit: + bnb_config = BitsAndBytesConfig( + load_in_8bit = True, + llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES, + ) + elif not load_in_4bit and not load_in_8bit and not full_finetuning: + print("Unsloth: LoRA, QLoRA and full finetuning all not selected. Switching to QLoRA.") + load_in_4bit = True + pass + + if full_finetuning: + if dtype == torch.bfloat16: + print("Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.") + else: + print("Unsloth: Float16 full finetuning uses more memory since we upcast weights to float32.") pass kwargs.pop("attn_implementation", None); # No need since we auto call it @@ -209,18 +236,29 @@ class FastBaseModel: while hasattr(m, "model"): m._saved_temp_tokenizer = tokenizer # Also set is_loaded_in_8bit to disable incorrect DDP - m.is_loaded_in_8bit = True + m.is_loaded_in_8bit = True if not full_finetuning else False m = m.model pass m._saved_temp_tokenizer = tokenizer # Also set is_loaded_in_8bit to disable incorrect DDP - m.is_loaded_in_8bit = True + m.is_loaded_in_8bit = True if not full_finetuning else False # Patch generate if model.generate.__name__ != "unsloth_base_fast_generate": model._old_generate = model.generate unsloth_base_fast_generate.__doc__ = model._old_generate.__doc__ model.generate = types.MethodType(unsloth_base_fast_generate, model) + + # Post patches + model = FastBaseModel.post_patch_model( + model, + use_gradient_checkpointing = use_gradient_checkpointing, + ) + # Clear deleted GPU items + for _ in range(3): + gc.collect() + torch.cuda.empty_cache() + pass return model, tokenizer pass @@ -299,7 +337,7 @@ class FastBaseModel: # Enable gradients on modules which are trainable requires_grad_for_gradient_checkpointing(model) - model = FastBaseModel.patch_peft_model(model, use_gradient_checkpointing) + model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing) # Clear deleted GPU items for _ in range(3): @@ -316,7 +354,7 @@ class FastBaseModel: @staticmethod - def patch_peft_model( + def post_patch_model( model, use_gradient_checkpointing = True, ): @@ -325,11 +363,22 @@ class FastBaseModel: "Unsloth: Your model needs to call `.get_peft_model` first!" ) pass + full_finetuning = hasattr(model.config, "quantization_config", None) is not None - model = prepare_model_for_kbit_training( + float32_mixed_precision = True + if _get_dtype(model.config.torch_dtype) == torch.bfloat16: + # Use bfloat16 precision for full finetuning + float32_mixed_precision = False + + model = prepare_model_for_training( model, use_gradient_checkpointing = use_gradient_checkpointing, - use_reentrant = True, + use_reentrant = True, + full_finetuning = full_finetuning, + train_layernorms = full_finetuning, + train_embedding = full_finetuning, + train_lm_head = full_finetuning, + float32_mixed_precision = float32_mixed_precision, ) from transformers.trainer import Trainer @@ -350,14 +399,14 @@ class FastBaseModel: m._saved_temp_tokenizer.tokenizer.padding_side = "right" pass # Also set is_loaded_in_8bit to disable incorrect DDP - m.is_loaded_in_8bit = True + m.is_loaded_in_8bit = True if not full_finetuning else False m = m.model pass if hasattr(m, "_saved_temp_tokenizer"): m._saved_temp_tokenizer.tokenizer.padding_side = "right" pass # Also set is_loaded_in_8bit to disable incorrect DDP - m.is_loaded_in_8bit = True + m.is_loaded_in_8bit = True if not full_finetuning else False # Clear deleted GPU items for _ in range(3):