From f7bad2a50b5b88ccb1a6bb4ad875d88f86178d1a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 8 Mar 2025 03:02:45 -0800 Subject: [PATCH] FastModel --- unsloth/models/__init__.py | 2 +- unsloth/models/_utils.py | 2 +- unsloth/models/llama.py | 4 +- unsloth/models/loader.py | 31 +++++++--- unsloth/models/vision.py | 112 +++++++++++++++++++------------------ 5 files changed, 86 insertions(+), 65 deletions(-) diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py index a187ee577a..317525c793 100644 --- a/unsloth/models/__init__.py +++ b/unsloth/models/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. from .llama import FastLlamaModel -from .loader import FastLanguageModel, FastVisionModel +from .loader import FastLanguageModel, FastVisionModel, FastTextModel, FastModel from .mistral import FastMistralModel from .qwen2 import FastQwen2Model from .granite import FastGraniteModel diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 37c69ef877..03eb21f4eb 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.3.8" +__version__ = "2025.3.9" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 3504037b66..888015b10e 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1981,8 +1981,8 @@ class FastLlamaModel: # Patch generate if model.generate.__name__ != "unsloth_fast_generate": model._old_generate = model.generate - unsloth_fast_generate.__doc__ = model._old_generate.__doc__ model.generate = types.MethodType(unsloth_fast_generate, model) + model.generate.__doc__ = model._old_generate.__doc__ pass return model, tokenizer pass @@ -2420,8 +2420,8 @@ class FastLlamaModel: # Patch generate if model.generate.__name__ != "unsloth_fast_generate": model._old_generate = model.generate - unsloth_fast_generate.__doc__ = model._old_generate.__doc__ model.generate = types.MethodType(unsloth_fast_generate, model) + model.generate.__doc__ = model._old_generate.__doc__ return model pass diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 30128cd134..b4639f27b1 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -383,10 +383,13 @@ from ..kernels import ( patch_loss_functions, post_patch_loss_function, ) -from .vision import FastBaseVisionModel +from .vision import FastBaseModel +from transformers import ( + AutoModelForVision2Seq, + AutoModelForCausalLM, +) - -class FastVisionModel(FastBaseVisionModel): +class FastModel(FastBaseModel): @staticmethod def from_pretrained( model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit", @@ -413,7 +416,7 @@ class FastVisionModel(FastBaseVisionModel): patch_compiling_bitsandbytes() if use_gradient_checkpointing == "unsloth": patch_unsloth_smart_gradient_checkpointing(dtype = dtype) - + old_model_name = model_name if not use_exact_model_name: model_name = get_model_name(model_name, load_in_4bit) @@ -427,7 +430,7 @@ class FastVisionModel(FastBaseVisionModel): from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled was_disabled = are_progress_bars_disabled() disable_progress_bars() - + autoconfig_error = None peft_error = None try: @@ -458,7 +461,7 @@ class FastVisionModel(FastBaseVisionModel): # Old transformers versions check both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32 - + # New transformers need to check manually. if SUPPORTS_LLAMA32: # Check if folder exists locally @@ -559,7 +562,12 @@ class FastVisionModel(FastBaseVisionModel): tokenizer_name = None pass - model, tokenizer = FastBaseVisionModel.from_pretrained( + # Check if VLM + is_vlm = (x.endswith("ForConditionalGeneration") for x in model_config.architectures) + is_vlm = is_vlm or hasattr(model_config, "vision_config") + auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM + + model, tokenizer = FastBaseModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, dtype = _get_dtype(dtype), @@ -570,6 +578,7 @@ class FastVisionModel(FastBaseVisionModel): revision = revision if not is_peft else None, model_types = model_types, tokenizer_name = tokenizer_name, + auto_model = auto_model, *args, **kwargs, ) @@ -617,8 +626,14 @@ class FastVisionModel(FastBaseVisionModel): trust_remote_code = trust_remote_code, ) # Patch it as well! - model = FastBaseVisionModel.patch_peft_model(model, use_gradient_checkpointing) + model = FastBaseModel.patch_peft_model(model, use_gradient_checkpointing) pass return model, tokenizer pass pass + +class FastVisionModel(FastModel): + pass + +class FastTextModel(FastModel): + pass diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 22b6ffcce8..9eb7f6e99f 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -17,6 +17,8 @@ from transformers import ( BitsAndBytesConfig, AutoModelForVision2Seq, AutoProcessor, + AutoTokenizer, + AutoModelForCausalLM, ) from .llama import * from ..kernels import ( @@ -32,26 +34,33 @@ from unsloth_zoo.peft_utils import ( ) from triton import __version__ as triton_version from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.patching_utils import patch_model_and_tokenizer import types import functools __all__ = [ - "FastBaseVisionModel", + "FastBaseModel", ] -def unsloth_vision_fast_generate( +def unsloth_base_fast_generate( self, *args, **kwargs, ): - FastBaseVisionModel.for_inference(self) - + FastBaseModel.for_inference(self) dtype = _get_dtype(self.config.torch_dtype) + # Check if VLM + is_vlm = (x.endswith("ForConditionalGeneration") for x in self.config.architectures) + is_vlm = is_vlm or hasattr(self.config, "vision_config") + # Remove token_type_ids kwargs.pop("token_type_ids", None) + # VLMs do not allow logits_to_keep + if not is_vlm: kwargs["logits_to_keep"] = 1 + # Check pad_token model_eos_token_id = getattr(model.config, "eos_token_id", None) if model_eos_token_id is not None and hasattr(model_eos_token_id, "__iter__"): @@ -59,27 +68,25 @@ def unsloth_vision_fast_generate( kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id) - try: - kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype) - except: - pass + # Get pixel values for VLMs + try: kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype) + except: pass # Mixed precision autocast with torch.inference_mode(), torch.autocast(device_type = "cuda", dtype = dtype): output = self._old_generate(*args, **kwargs) pass - FastBaseVisionModel.for_training(self) - + FastBaseModel.for_training(self) return output pass -class FastBaseVisionModel: +class FastBaseModel: @staticmethod def from_pretrained( - model_name = "unsloth/llama-3-8b-bnb-4bit", + model_name = "unsloth/Llama-3.2-1B-Instruct", max_seq_length = None, dtype = None, load_in_4bit = True, @@ -88,6 +95,7 @@ class FastBaseVisionModel: trust_remote_code = False, model_types = None, tokenizer_name = None, + auto_model = AutoModelForVision2Seq, **kwargs, ): if trust_remote_code: @@ -148,7 +156,7 @@ class FastBaseVisionModel: # Cannot be None, since HF now checks for the config if load_in_4bit: kwargs["quantization_config"] = bnb_config - model = AutoModelForVision2Seq.from_pretrained( + model = auto_model.from_pretrained( model_name, device_map = device_map, torch_dtype = dtype, @@ -163,26 +171,25 @@ class FastBaseVisionModel: # Counteract saved tokenizers tokenizer_name = model_name if tokenizer_name is None else tokenizer_name - tokenizer = AutoProcessor.from_pretrained( + auto_processor = AutoProcessor if auto_model is AutoModelForVision2Seq else AutoTokenizer + tokenizer = auto_processor.from_pretrained( tokenizer_name, padding_side = "right", token = token, ) # Add padding side as well - tokenizer.tokenizer.padding_side = "right" + if hasattr(tokenizer, "tokenizer"): + tokenizer.tokenizer.padding_side = "right" model, tokenizer = patch_tokenizer(model, tokenizer) model = post_patch_loss_function(model) - - # Fix up config for transformers uploading PEFT - # Not necessary anymore since we require transformers>=4.37! - if False: - name = model.config._name_or_path - if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): - name = name[:len(name) - len("-bnb-4bit")] - model.config.update({"_name_or_path" : name}) - pass - pass + # Fix other stuff like BnB compute data types + model, tokenizer = patch_model_and_tokenizer( + model, + tokenizer, + downcast_rope = False, + fix_embeddings = False, + ) # Log Unsloth version for future fastpaths for inference if hasattr(model, "config"): @@ -198,23 +205,22 @@ class FastBaseVisionModel: # Save tokenizer for inference purposes tokenizer.padding_side = "left" # Force inference tokenizer.tokenizer.padding_side = "left" # Force inference - internal_model = model - while hasattr(internal_model, "model"): - internal_model._saved_temp_tokenizer = tokenizer + m = model + while hasattr(m, "model"): + m._saved_temp_tokenizer = tokenizer # Also set is_loaded_in_8bit to disable incorrect DDP - internal_model.is_loaded_in_8bit = True - - internal_model = internal_model.model + m.is_loaded_in_8bit = True + m = m.model pass - internal_model._saved_temp_tokenizer = tokenizer + m._saved_temp_tokenizer = tokenizer # Also set is_loaded_in_8bit to disable incorrect DDP - internal_model.is_loaded_in_8bit = True + m.is_loaded_in_8bit = True # Patch generate - if model.generate.__name__ != "unsloth_vision_fast_generate": + if model.generate.__name__ != "unsloth_base_fast_generate": model._old_generate = model.generate - unsloth_vision_fast_generate.__doc__ = model._old_generate.__doc__ - model.generate = types.MethodType(unsloth_vision_fast_generate, model) + model.generate = types.MethodType(unsloth_base_fast_generate, model) + model.generate.__doc__ = model._old_generate.__doc__ return model, tokenizer pass @@ -293,7 +299,7 @@ class FastBaseVisionModel: # Enable gradients on modules which are trainable requires_grad_for_gradient_checkpointing(model) - model = FastBaseVisionModel.patch_peft_model(model, use_gradient_checkpointing) + model = FastBaseModel.patch_peft_model(model, use_gradient_checkpointing) # Clear deleted GPU items for _ in range(3): @@ -303,8 +309,8 @@ class FastBaseVisionModel: patch_saving_functions(model, vision = True) # Add for_inference and for_training - model.for_training = functools.partial(FastBaseVisionModel.for_training, model) - model.for_inference = functools.partial(FastBaseVisionModel.for_inference, model) + model.for_training = functools.partial(FastBaseModel.for_training, model) + model.for_inference = functools.partial(FastBaseModel.for_inference, model) return model pass @@ -338,20 +344,20 @@ class FastBaseVisionModel: patch_saving_functions(model, vision = True) # Patch tokenizer to pad to the right - internal_model = model - while hasattr(internal_model, "model"): - if hasattr(internal_model, "_saved_temp_tokenizer"): - internal_model._saved_temp_tokenizer.tokenizer.padding_side = "right" + m = model + while hasattr(m, "model"): + if hasattr(m, "_saved_temp_tokenizer"): + m._saved_temp_tokenizer.tokenizer.padding_side = "right" pass # Also set is_loaded_in_8bit to disable incorrect DDP - internal_model.is_loaded_in_8bit = True - internal_model = internal_model.model + m.is_loaded_in_8bit = True + m = m.model pass - if hasattr(internal_model, "_saved_temp_tokenizer"): - internal_model._saved_temp_tokenizer.tokenizer.padding_side = "right" + if hasattr(m, "_saved_temp_tokenizer"): + m._saved_temp_tokenizer.tokenizer.padding_side = "right" pass # Also set is_loaded_in_8bit to disable incorrect DDP - internal_model.is_loaded_in_8bit = True + m.is_loaded_in_8bit = True # Clear deleted GPU items for _ in range(3): @@ -359,14 +365,14 @@ class FastBaseVisionModel: torch.cuda.empty_cache() pass # Add for_inference and for_training - model.for_training = functools.partial(FastBaseVisionModel.for_training, model) - model.for_inference = functools.partial(FastBaseVisionModel.for_inference, model) + model.for_training = functools.partial(FastBaseModel.for_training, model) + model.for_inference = functools.partial(FastBaseModel.for_inference, model) # Patch generate - if model.generate.__name__ != "unsloth_vision_fast_generate": + if model.generate.__name__ != "unsloth_base_fast_generate": model._old_generate = model.generate - unsloth_vision_fast_generate.__doc__ = model._old_generate.__doc__ - model.generate = types.MethodType(unsloth_vision_fast_generate, model) + model.generate = types.MethodType(unsloth_base_fast_generate, model) + model.generate.__doc__ = model._old_generate.__doc__ return model pass