From 925a63120e73119fc55d234f5f587bcb49befe94 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 21 Nov 2024 01:48:20 -0800 Subject: [PATCH] Vision --- unsloth/models/_utils.py | 5 +- unsloth/models/loader.py | 400 ++++++++++++++++++++-------- unsloth/models/loader_utils.py | 114 ++++++++ unsloth/models/mapper.py | 8 + unsloth/models/vision.py | 460 +++++++++++---------------------- unsloth/save.py | 170 +++++++++++- 6 files changed, 732 insertions(+), 425 deletions(-) create mode 100644 unsloth/models/loader_utils.py diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index e1214a89e8..460b23bd0d 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2024.11.7" +__version__ = "2024.11.8" __all__ = [ "prepare_model_for_kbit_training", @@ -1120,7 +1120,6 @@ def unsloth_compile_transformers( revision = revision, trust_remote_code = trust_remote_code, ) - print(f"Unsloth: Automatic compiler will now patch {model_types}") for model_type in model_types: _unsloth_compile_transformers( model_type, @@ -1145,5 +1144,5 @@ def unsloth_compile_transformers( disable = disable, ) pass - return + return model_types pass diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 1d4bb71e85..b561239313 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -20,7 +20,7 @@ from .cohere import FastCohereModel from transformers import AutoConfig from transformers import __version__ as transformers_version from peft import PeftConfig, PeftModel -from .mapper import INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit +from .loader_utils import get_model_name import os try: from huggingface_hub.utils import get_token @@ -63,106 +63,6 @@ def _get_dtype(dtype): pass -def __get_model_name( - model_name, - load_in_4bit = True, - INT_TO_FLOAT_MAPPER = None, - FLOAT_TO_INT_MAPPER = None, - MAP_TO_UNSLOTH_16bit = None, -): - model_name = str(model_name) - lower_model_name = model_name.lower() - - if not SUPPORTS_FOURBIT and lower_model_name in INT_TO_FLOAT_MAPPER: - - model_name = INT_TO_FLOAT_MAPPER[lower_model_name] - logger.warning_once( - f"Unsloth: Your transformers version of {transformers_version} does not support native "\ - f"4bit loading.\nThe minimum required version is 4.37.\n"\ - f'Try `pip install --upgrade "transformers>=4.37"`\n'\ - f"to obtain the latest transformers build, then restart this session.\n"\ - f"For now, we shall load `{model_name}` instead (still 4bit, just slower downloading)." - ) - return model_name - - elif not load_in_4bit and lower_model_name in INT_TO_FLOAT_MAPPER: - - new_model_name = INT_TO_FLOAT_MAPPER[lower_model_name] - # logger.warning_once( - # f"Unsloth: You passed in `{model_name}` which is a 4bit model, yet you set\n"\ - # f"`load_in_4bit = False`. We shall load `{new_model_name}` instead." - # ) - return new_model_name - - elif not load_in_4bit and lower_model_name in MAP_TO_UNSLOTH_16bit: - - new_model_name = MAP_TO_UNSLOTH_16bit[lower_model_name] - return new_model_name - - elif load_in_4bit and SUPPORTS_FOURBIT and lower_model_name in FLOAT_TO_INT_MAPPER: - - new_model_name = FLOAT_TO_INT_MAPPER[lower_model_name] - # logger.warning_once( - # f"Unsloth: You passed in `{model_name}` and `load_in_4bit = True`.\n"\ - # f"We shall load `{new_model_name}` for 4x faster loading." - # ) - return new_model_name - pass - - return None -pass - - -def _get_new_mapper(): - try: - import requests - new_mapper = "https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/models/mapper.py" - with requests.get(new_mapper, timeout = 3) as new_mapper: new_mapper = new_mapper.text - new_mapper = new_mapper[new_mapper.find("__INT_TO_FLOAT_MAPPER"):] - new_mapper = new_mapper\ - .replace("INT_TO_FLOAT_MAPPER", "NEW_INT_TO_FLOAT_MAPPER")\ - .replace("FLOAT_TO_INT_MAPPER", "NEW_FLOAT_TO_INT_MAPPER")\ - .replace("MAP_TO_UNSLOTH_16bit", "NEW_MAP_TO_UNSLOTH_16bit") - - exec(new_mapper, globals()) - return NEW_INT_TO_FLOAT_MAPPER, NEW_FLOAT_TO_INT_MAPPER, NEW_MAP_TO_UNSLOTH_16bit - except: - return {}, {}, {} - pass -pass - - -def get_model_name(model_name, load_in_4bit = True): - new_model_name = __get_model_name( - model_name = model_name, - load_in_4bit = load_in_4bit, - INT_TO_FLOAT_MAPPER = INT_TO_FLOAT_MAPPER, - FLOAT_TO_INT_MAPPER = FLOAT_TO_INT_MAPPER, - MAP_TO_UNSLOTH_16bit = MAP_TO_UNSLOTH_16bit, - ) - if new_model_name is None and model_name.count("/") == 1 and model_name[0].isalnum(): - # Try checking if a new Unsloth version allows it! - NEW_INT_TO_FLOAT_MAPPER, NEW_FLOAT_TO_INT_MAPPER, NEW_MAP_TO_UNSLOTH_16bit = _get_new_mapper() - upgraded_model_name = __get_model_name( - model_name = model_name, - load_in_4bit = load_in_4bit, - INT_TO_FLOAT_MAPPER = NEW_INT_TO_FLOAT_MAPPER, - FLOAT_TO_INT_MAPPER = NEW_FLOAT_TO_INT_MAPPER, - MAP_TO_UNSLOTH_16bit = NEW_MAP_TO_UNSLOTH_16bit, - ) - if upgraded_model_name is not None: - raise NotImplementedError( - f"Unsloth: {model_name} is not supported in your current Unsloth version! Please update Unsloth via:\n\n"\ - 'pip uninstall unsloth unsloth_zoo -y\n'\ - 'pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n'\ - 'pip install --upgrade --no-cache-dir "git+https://github.com/unslothai/unsloth-zoo.git"\n'\ - ) - pass - pass - return new_model_name if new_model_name is not None else model_name -pass - - class FastLanguageModel(FastLlamaModel): @staticmethod def from_pretrained( @@ -275,6 +175,302 @@ class FastLanguageModel(FastLlamaModel): model_type = model_config.model_type + if model_type == "llama": + scaling_type = None + if getattr(model_config, "rope_scaling", None) is not None: + scaling_type1 = model_config.rope_scaling.get("type", None) + scaling_type2 = model_config.rope_scaling.get("rope_type", None) + scaling_type = scaling_type1 if scaling_type1 is not None else scaling_type2 + pass + + if scaling_type == "llama3" and not SUPPORTS_LLAMA31: + raise ImportError( + f"Unsloth: Your transformers version of {transformers_version} does not support Llama 3.1.\n"\ + f"The minimum required version is 4.43.2\n"\ + f'Try `pip install --upgrade "transformers>=4.43.2"`\n'\ + f"to obtain the latest transformers build, then restart this session."\ + ) + + dispatch_model = FastLlamaModel + + elif model_type == "mistral": dispatch_model = FastMistralModel + elif model_type == "gemma": + if not SUPPORTS_GEMMA: + raise ImportError( + f"Unsloth: Your transformers version of {transformers_version} does not support Gemma.\n"\ + f"The minimum required version is 4.38.\n"\ + f'Try `pip install --upgrade "transformers>=4.38"`\n'\ + f"to obtain the latest transformers build, then restart this session."\ + ) + dispatch_model = FastGemmaModel + elif model_type == "gemma2": + if not SUPPORTS_GEMMA2: + raise ImportError( + f"Unsloth: Your transformers version of {transformers_version} does not support Gemma2.\n"\ + f"The minimum required version is 4.42.3.\n"\ + f'Try `pip install --upgrade "transformers>=4.42.3"`\n'\ + f"to obtain the latest transformers build, then restart this session."\ + ) + # Also check for softcapping support in flash-attn which is faster! + if is_bfloat16_supported() and not HAS_FLASH_ATTENTION: + print( + "Unsloth: If you want to finetune Gemma 2, install flash-attn to make it faster!\n"\ + "To install flash-attn, do the below:\n"\ + '\npip install --no-deps --upgrade "flash-attn>=2.6.3"' + ) + elif HAS_FLASH_ATTENTION and not HAS_FLASH_ATTENTION_SOFTCAPPING: + print( + "Unsloth: If you want to finetune Gemma 2, upgrade flash-attn to version 2.6.3 or higher!\n"\ + "Newer versions support faster and less memory usage kernels for Gemma 2's attention softcapping!\n"\ + "To update flash-attn, do the below:\n"\ + '\npip install --no-deps --upgrade "flash-attn>=2.6.3"' + ) + + dispatch_model = FastGemma2Model + elif model_type == "qwen2": + dispatch_model = FastQwen2Model + elif model_type == "cohere": + dispatch_model = FastCohereModel + else: + raise NotImplementedError( + f"Unsloth: {model_name} not supported yet!\n"\ + "Maybe you're doing vision finetuning? Please use FastVisionModel instead!\n"\ + "Otherwise, make an issue to https://github.com/unslothai/unsloth!", + ) + pass + + # Check if this is local model since the tokenizer gets overwritten + if os.path.exists(os.path.join(old_model_name, "tokenizer_config.json")) and \ + os.path.exists(os.path.join(old_model_name, "tokenizer.json")) and \ + os.path.exists(os.path.join(old_model_name, "special_tokens_map.json")): + + tokenizer_name = old_model_name + else: + tokenizer_name = None + pass + + model, tokenizer = dispatch_model.from_pretrained( + model_name = model_name, + max_seq_length = max_seq_length, + dtype = _get_dtype(dtype), + load_in_4bit = load_in_4bit, + token = token, + device_map = device_map, + rope_scaling = rope_scaling, + fix_tokenizer = fix_tokenizer, + model_patcher = dispatch_model, + tokenizer_name = tokenizer_name, + trust_remote_code = trust_remote_code, + revision = revision if not is_peft else None, + *args, **kwargs, + ) + + if resize_model_vocab is not None: + model.resize_token_embeddings(resize_model_vocab) + pass + + # In case the model supports tagging, add the unsloth tag. + if hasattr(model, "add_model_tags"): + model.add_model_tags(["unsloth",]) + pass + if hasattr(tokenizer, "add_model_tags"): + tokenizer.add_model_tags(["unsloth",]) + pass + + if load_in_4bit: + # Fix up bitsandbytes config + quantization_config = \ + { + # Sometimes torch_dtype is not a string!! + "bnb_4bit_compute_dtype" : model.config.to_dict()["torch_dtype"], + "bnb_4bit_quant_type" : "nf4", + "bnb_4bit_use_double_quant" : True, + "llm_int8_enable_fp32_cpu_offload" : False, + "llm_int8_has_fp16_weight" : False, + "llm_int8_skip_modules" : None, + "llm_int8_threshold" : 6.0, + "load_in_4bit" : True, + "load_in_8bit" : False, + "quant_method" : "bitsandbytes", + } + model.config.update({"quantization_config" : quantization_config}) + pass + + if is_peft: + # From https://github.com/huggingface/peft/issues/184 + # Now add PEFT adapters + model.enable_input_require_grads() + model = PeftModel.from_pretrained( + model, + old_model_name, + token = token, + revision = revision, + is_trainable = True, + trust_remote_code = trust_remote_code, + ) + # Patch it as well! + model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing) + pass + return model, tokenizer + pass +pass + + +from ._utils import ( + patch_compiling_bitsandbytes, + patch_model_and_tokenizer, + prepare_model_for_kbit_training, + patch_unsloth_smart_gradient_checkpointing, + patch_compiled_autograd, + process_vision_info, + unsloth_compile_transformers, +) +from ..kernels import ( + patch_loss_functions, + post_patch_loss_function, +) + +class FastVisionModel: + @staticmethod + def from_pretrained( + model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit", + max_seq_length = None, # [TODO] No effect + dtype = None, + load_in_4bit = True, + token = None, + device_map = "sequential", + rope_scaling = None, # [TODO] No effect + fix_tokenizer = True, # [TODO] No effect + trust_remote_code = False, + use_gradient_checkpointing = "unsloth", + resize_model_vocab = None, # [TODO] No effect + revision = None, + *args, **kwargs, + ): + if token is None: token = get_token() + + patch_compiled_autograd() + patch_loss_functions(torch_compile = False) + patch_compiling_bitsandbytes() + if use_gradient_checkpointing == "unsloth": + patch_unsloth_smart_gradient_checkpointing() + + old_model_name = model_name + model_name = get_model_name(model_name, load_in_4bit) + + model_types = unsloth_compile_transformers( + model_name = model_name, + sdpa_dynamic_mask = True, + sdpa_bool_masks = True, + sdpa_gqa_replace = True, + sdpa_dynamic_compile = True, + compile_attention = True, + disable_causal_masks = True, + compile_torch_modules = True, + compile_custom_modules = True, + compile_function_calls = True, + fuse_lm_head = True, + gradient_checkpointing = True, + manual_replacements = True, + epilogue_fusion = True, + max_autotune = False, + shape_padding = True, + cudagraphs = False, + debug = False, + import_from_cache = False, + disable = False, + ) + + # First check if it's a normal model via AutoConfig + from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled + was_disabled = are_progress_bars_disabled() + disable_progress_bars() + + autoconfig_error = None + peft_error = None + try: + model_config = AutoConfig.from_pretrained( + model_name, + token = token, + revision = revision, + trust_remote_code = trust_remote_code, + ) + is_model = True + except Exception as error: + autoconfig_error = str(error) + is_model = False + try: + peft_config = PeftConfig.from_pretrained( + model_name, + token = token, + revision = revision, + trust_remote_code = trust_remote_code, + ) + is_peft = True + except Exception as error: + peft_error = str(error) + is_peft = False + pass + + # Both config.json and adapter_config.json should not exist! + + # Old transformers versions check + both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32 + + # New transformers need to check manually. + if SUPPORTS_LLAMA32: + # Check if folder exists locally + if os.path.isdir(model_name): + exist_adapter_config = os.path.exists(os.path.join(model_name, "adapter_config.json")) + exist_config = os.path.exists(os.path.join(model_name, "config.json")) + both_exist = exist_adapter_config and exist_config + else: + files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json")) + files = (os.path.split(x)[-1] for x in files) + if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2: + both_exist = True + pass + pass + pass + + # Error out if both LoRA and normal model config exists. + if both_exist: + raise RuntimeError( + "Unsloth: Your repo has a LoRA adapter and a base model.\n"\ + "You have 2 files `config.json` and `adapter_config.json`.\n"\ + "We must only allow one config file.\n"\ + "Please separate the LoRA and base models to 2 repos." + ) + + elif not is_model and not is_peft: + error = autoconfig_error or peft_error + # Old transformers version + if "rope_scaling" in error.lower() and not SUPPORTS_LLAMA31: + raise ImportError( + f"Unsloth: Your transformers version of {transformers_version} does not support new RoPE scaling methods.\n"\ + f"This includes Llama 3.1. The minimum required version is 4.43.2\n"\ + f'Try `pip install --upgrade "transformers>=4.43.2"`\n'\ + f"to obtain the latest transformers build, then restart this session."\ + ) + raise RuntimeError(autoconfig_error or peft_error) + pass + + # Get base model for PEFT: + if is_peft: + # Check base model again for PEFT + model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) + model_config = AutoConfig.from_pretrained( + model_name, + token = token, + revision = revision, + trust_remote_code = trust_remote_code, + ) + pass + + if not was_disabled: enable_progress_bars() + + model_type = model_config.model_type + if model_type == "llama": scaling_type = None if getattr(model_config, "rope_scaling", None) is not None: @@ -412,4 +608,4 @@ class FastLanguageModel(FastLlamaModel): pass return model, tokenizer pass -pass \ No newline at end of file +pass diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py new file mode 100644 index 0000000000..dcb17b41dc --- /dev/null +++ b/unsloth/models/loader_utils.py @@ -0,0 +1,114 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .mapper import INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit + +def __get_model_name( + model_name, + load_in_4bit = True, + INT_TO_FLOAT_MAPPER = None, + FLOAT_TO_INT_MAPPER = None, + MAP_TO_UNSLOTH_16bit = None, +): + model_name = str(model_name) + lower_model_name = model_name.lower() + + if not SUPPORTS_FOURBIT and lower_model_name in INT_TO_FLOAT_MAPPER: + + model_name = INT_TO_FLOAT_MAPPER[lower_model_name] + print( + f"Unsloth: Your transformers version of {transformers_version} does not support native "\ + f"4bit loading.\nThe minimum required version is 4.37.\n"\ + f'Try `pip install --upgrade "transformers>=4.37"`\n'\ + f"to obtain the latest transformers build, then restart this session.\n"\ + f"For now, we shall load `{model_name}` instead (still 4bit, just slower downloading)." + ) + return model_name + + elif not load_in_4bit and lower_model_name in INT_TO_FLOAT_MAPPER: + + new_model_name = INT_TO_FLOAT_MAPPER[lower_model_name] + # logger.warning_once( + # f"Unsloth: You passed in `{model_name}` which is a 4bit model, yet you set\n"\ + # f"`load_in_4bit = False`. We shall load `{new_model_name}` instead." + # ) + return new_model_name + + elif not load_in_4bit and lower_model_name in MAP_TO_UNSLOTH_16bit: + + new_model_name = MAP_TO_UNSLOTH_16bit[lower_model_name] + return new_model_name + + elif load_in_4bit and SUPPORTS_FOURBIT and lower_model_name in FLOAT_TO_INT_MAPPER: + + new_model_name = FLOAT_TO_INT_MAPPER[lower_model_name] + # logger.warning_once( + # f"Unsloth: You passed in `{model_name}` and `load_in_4bit = True`.\n"\ + # f"We shall load `{new_model_name}` for 4x faster loading." + # ) + return new_model_name + pass + + return None +pass + + +def _get_new_mapper(): + try: + import requests + new_mapper = "https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/models/mapper.py" + with requests.get(new_mapper, timeout = 3) as new_mapper: new_mapper = new_mapper.text + new_mapper = new_mapper[new_mapper.find("__INT_TO_FLOAT_MAPPER"):] + new_mapper = new_mapper\ + .replace("INT_TO_FLOAT_MAPPER", "NEW_INT_TO_FLOAT_MAPPER")\ + .replace("FLOAT_TO_INT_MAPPER", "NEW_FLOAT_TO_INT_MAPPER")\ + .replace("MAP_TO_UNSLOTH_16bit", "NEW_MAP_TO_UNSLOTH_16bit") + + exec(new_mapper, globals()) + return NEW_INT_TO_FLOAT_MAPPER, NEW_FLOAT_TO_INT_MAPPER, NEW_MAP_TO_UNSLOTH_16bit + except: + return {}, {}, {} + pass +pass + + +def get_model_name(model_name, load_in_4bit = True): + new_model_name = __get_model_name( + model_name = model_name, + load_in_4bit = load_in_4bit, + INT_TO_FLOAT_MAPPER = INT_TO_FLOAT_MAPPER, + FLOAT_TO_INT_MAPPER = FLOAT_TO_INT_MAPPER, + MAP_TO_UNSLOTH_16bit = MAP_TO_UNSLOTH_16bit, + ) + if new_model_name is None and model_name.count("/") == 1 and model_name[0].isalnum(): + # Try checking if a new Unsloth version allows it! + NEW_INT_TO_FLOAT_MAPPER, NEW_FLOAT_TO_INT_MAPPER, NEW_MAP_TO_UNSLOTH_16bit = _get_new_mapper() + upgraded_model_name = __get_model_name( + model_name = model_name, + load_in_4bit = load_in_4bit, + INT_TO_FLOAT_MAPPER = NEW_INT_TO_FLOAT_MAPPER, + FLOAT_TO_INT_MAPPER = NEW_FLOAT_TO_INT_MAPPER, + MAP_TO_UNSLOTH_16bit = NEW_MAP_TO_UNSLOTH_16bit, + ) + if upgraded_model_name is not None: + raise NotImplementedError( + f"Unsloth: {model_name} is not supported in your current Unsloth version! Please update Unsloth via:\n\n"\ + 'pip uninstall unsloth unsloth_zoo -y\n'\ + 'pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n'\ + 'pip install --upgrade --no-cache-dir "git+https://github.com/unslothai/unsloth-zoo.git"\n'\ + ) + pass + pass + return new_model_name if new_model_name is not None else model_name +pass diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 03047b8103..fc1dc8cdb0 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -484,6 +484,14 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/Pixtral-12B-Base-2409", "mistralai/Pixtral-12B-Base-2409", ), + "unsloth/llava-1.5-7b-hf-bnb-4bit" : ( + "unsloth/llava-1.5-7b-hf", + "llava-hf/llava-1.5-7b-hf", + ), + "unsloth/llava-v1.6-mistral-7b-hf-bnb-4bit" : ( + "unsloth/llava-v1.6-mistral-7b-hf", + "llava-hf/llava-v1.6-mistral-7b-hf", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 0b8c08a371..ccfc261c94 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -1,58 +1,49 @@ +# Unsloth Zoo - Utilities for Unsloth # Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +import torch +from transformers import ( + BitsAndBytesConfig, + AutoModelForVision2Seq, + AutoProcessor, +) from .llama import * -from ..kernels import patch_layernorm, unpatch_layernorm -from ..kernels import patch_rms_layernorm, unpatch_rms_layernorm -from ..kernels import patch_llama_for_causal_lm, unpatch_llama_for_causal_lm -from ._utils import patch_gradient_checkpointing +from ..kernels import ( + post_patch_loss_function, +) +from peft import LoraConfig, TaskType, get_peft_model +from transformers import set_seed as transformers_set_seed +from unsloth_zoo.peft_utils import ( + get_peft_regex, + merge_and_overwrite_lora, +) -from transformers import AutoProcessor -try: - from transformers import MllamaForConditionalGeneration -except: - raise ImportError( - "Unsloth: Please update your transformers version to 4.46.0 for Llama 3.2 support!" - ) -pass - -class FastVisionModel: - - def pre_patch(self): - patch_gradient_checkpointing() - patch_layernorm() - patch_rms_layernorm() - patch_llama_for_causal_lm() - pass - - def post_unpatch(self): - unpatch_layernorm() - unpatch_rms_layernorm() - unpatch_llama_for_causal_lm() - pass +class FastBaseVisionModel: @staticmethod def from_pretrained( - model_name = "llava-hf/llava-1.5-7b-hf", + model_name = "unsloth/llama-3-8b-bnb-4bit", max_seq_length = None, dtype = None, load_in_4bit = True, token = None, device_map = "sequential", - rope_scaling = None, trust_remote_code = False, + model_types = None, **kwargs, ): if trust_remote_code: @@ -67,7 +58,7 @@ class FastVisionModel: max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) statistics = \ - f"==((====))== Unsloth {__version__}: Fast {model_patcher.__name__[4:-5]} patching. Transformers = {transformers_version}.\n"\ + f"==((====))== Unsloth {__version__}: Fast {model_types[0]} vision patching. Transformers = {transformers_version}.\n"\ f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\ f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. FA [Xformers = {xformers_version}. FA2 = {HAS_FLASH_ATTENTION}]\n"\ @@ -81,7 +72,9 @@ class FastVisionModel: pass # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer + os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" + model_patcher.pre_patch() get_statistics() # For debugging - we use a download counter to see if environments are not breaking if dtype is None: @@ -105,160 +98,36 @@ class FastVisionModel: ) pass + kwargs.pop("attn_implementation", None); # No need since we auto call it + # Cannot be None, since HF now checks for the config if load_in_4bit: kwargs["quantization_config"] = bnb_config - self.pre_patch() - model = MllamaForConditionalGeneration.from_pretrained( + model = AutoModelForVision2Seq.from_pretrained( model_name, device_map = device_map, torch_dtype = dtype, - # quantization_config = bnb_config, + # quantization_config = bnb_config, token = token, - max_position_embeddings = max_position_embeddings, trust_remote_code = trust_remote_code, - attn_implementation = "sdpa", + # attn_implementation = "sdpa", [TODO] Pixtral for eg fails **kwargs, ) - self.post_unpatch() - # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer # We currently only support NVIDIA GPUs - AMD / Intel is a work in progress! post_check = check_nvidia() # Counteract saved tokenizers + tokenizer_name = model_name if tokenizer_name is None else tokenizer_name tokenizer = AutoProcessor.from_pretrained( - model_name, - ) - model = FastVisionModel.post_patch(model) - - # Patch Trainer - from transformers.trainer import Trainer - try: - if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": - inner_training_loop = inspect.getsource(Trainer._inner_training_loop) - Trainer._original_training_loop = inner_training_loop - else: - inner_training_loop = Trainer._original_training_loop - except: - raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') - pass - - if ((post_check - pre_check) >= 1).sum() > 1: - raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') - - import transformers.trainer - items_in_trainer = dir(transformers.trainer) - good_items = [] - for item in items_in_trainer: - # TODO: Support Deepspeed - if item.startswith(("deepspeed", "xm", "met", "smp")): continue - if item in inner_training_loop: good_items.append(item) - pass - exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")", globals()) - - start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0] - end = inner_training_loop.find("\n\n", start) - original_debug = inner_training_loop[start:end] - spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:] - front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0) - - debug_info = """debug_info = \\ - f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs = {args.world_size}\\n"\\ - f" \\\\\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\ - f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\ - f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\ - f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}' - logger.warning(debug_info) - import subprocess, re, gc, numpy as np - a = np.array([0,]) - try: - a = subprocess.check_output('nvidia-smi --query-gpu=memory.used --format=csv', shell = True) - a = re.findall(rb'([\\d]{1,})[\\s]{1,}M', a) - a = np.array([int(x.decode('utf-8'))/1024 for x in a]) - except: - if not torch.cuda.is_available(): - raise RuntimeError('Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!') - if ((a - PRE_CHECK) >= 1).sum() > 1: - raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') - for _ in range(3): - gc.collect() - torch.cuda.empty_cache()""" - - debug_info = debug_info.split('\n') - debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]]) - inner_training_loop = inner_training_loop.replace(original_debug, debug_info) - - debug_info = """n_total_devices = total_train_batch_size // \\ - args.gradient_accumulation_steps // self._train_batch_size - if n_total_devices > 1: - logger.warning_once('Unsloth currently does not support multi GPU setups - but we are working on it!') - debug_info =""" - debug_info = debug_info.split('\n') - debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]]) - inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1) - - front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0) - inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags = re.MULTILINE) - inner_training_loop = inner_training_loop.replace( - "train_dataloader = tpu_spmd_dataloader(train_dataloader)", - "raise RuntimeError('Unsloth: TPUs are not yet supported!')" - ) - inner_training_loop = inner_training_loop.replace( - "self.accelerator.free_memory()", - "self.accelerator.free_memory()\n" + \ - front_spaces + "if self.is_deepspeed_enabled:"\ - "raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1, + tokenizer_name, + padding_side = "right", + token = token, ) - check_batches = """train_dataloader = self.get_train_dataloader() - ga = args.gradient_accumulation_steps - bsz = self._train_batch_size - total_batches = bsz * ga * args.world_size - n_total_devices = total_batches // ga // bsz - if n_total_devices > 1: - logger.warning_once('Unsloth currently does not support multi GPU setups - but we are working on it!') - divisor = n_total_devices / 1 - bsz = self._train_batch_size = max(int(bsz / divisor), 1) - if total_batches // ga // bsz > 1: - divisor = n_total_devices / 1 - ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)""" - check_batches = check_batches.split('\n') - check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]]) - inner_training_loop = inner_training_loop.replace( - "train_dataloader = self.get_train_dataloader()", - check_batches, 1, - ) - inner_training_loop = inner_training_loop.replace( - "_inner_training_loop", - "_fast_inner_training_loop", 1, - ) - exec(inner_training_loop, globals()) - - Trainer._inner_training_loop = _fast_inner_training_loop - inner_training_loop = inner_training_loop.replace( - "is_torch_tpu_available()", - "False", - ) - if "n_total_devices >" not in inner_training_loop: - raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') - pass - inner_training_loop = inner_training_loop.replace( - "is_sagemaker_mp_enabled()", - "False", - ) - exec(inner_training_loop, globals()) - Trainer._inner_training_loop = _fast_inner_training_loop - - # Save max_seq_length - model.max_seq_length = max_position_embeddings - internal_model = model - while hasattr(internal_model, "model"): - internal_model.max_seq_length = max_position_embeddings - internal_model = internal_model.model - pass - internal_model.max_seq_length = max_position_embeddings + model, tokenizer = patch_tokenizer(model, tokenizer) + model = post_patch_loss_function(model) # Fix up config for transformers uploading PEFT # Not necessary anymore since we require transformers>=4.37! @@ -271,115 +140,80 @@ class FastVisionModel: pass # Log Unsloth version for future fastpaths for inference - model.config.update({"unsloth_version" : __version__}) + if hasattr(model, "config"): + model.config.update({"unsloth_version" : __version__}) + pass + patch_saving_functions(model, vision = True) + patch_saving_functions(tokenizer, vision = True) - # Add save modules - patch_saving_functions(model) - Trainer._inner_training_loop = _fast_inner_training_loop + # Fix gradient accumulation + from transformers.trainer import Trainer + patch_gradient_accumulation_fix(Trainer) - # Also fix torch_dtype + # Save tokenizer for inference purposes + tokenizer.padding_side = "left" # Force inference internal_model = model while hasattr(internal_model, "model"): - if hasattr(internal_model, "config"): - if internal_model.config.torch_dtype == "float32": - internal_model.config.torch_dtype = torch.float32 - elif internal_model.config.torch_dtype == "bfloat16": - internal_model.config.torch_dtype = torch.bfloat16 - elif internal_model.config.torch_dtype == "float16": - internal_model.config.torch_dtype = torch.float16 - pass - pass + internal_model._saved_temp_tokenizer = tokenizer internal_model = internal_model.model pass - if hasattr(internal_model, "config"): - if internal_model.config.torch_dtype == "float32": - internal_model.config.torch_dtype = torch.float32 - elif internal_model.config.torch_dtype == "bfloat16": - internal_model.config.torch_dtype = torch.bfloat16 - elif internal_model.config.torch_dtype == "float16": - internal_model.config.torch_dtype = torch.float16 - pass - pass + internal_model._saved_temp_tokenizer = tokenizer return model, tokenizer pass - @staticmethod - def post_patch(model): - # Patch model - layers = model.model.layers - lm_head = model.get_output_embeddings().weight - - # Also patch all dtypes - BnB seems to not allocate the correct type? - # BnB default dtype seems to be float16! - correct_dtype = lm_head.weight.dtype - - for name, module in model.named_modules(): - if isinstance(module, (Bnb_Linear4bit, Peft_Linear4bit)): - weight = module.weight - quant_state = weight.quant_state - - if type(quant_state) is list: - # BnB seems to have float16 as default! - module.weight.quant_state[2] = correct_dtype # Cast to correct dtype - else: - # https://github.com/TimDettmers/bitsandbytes/pull/763/files - quant_state.dtype = correct_dtype - pass - pass - pass - - # Clear deleted GPU items - for _ in range(3): - gc.collect() - torch.cuda.empty_cache() - return model - pass - - @staticmethod def get_peft_model( model, - r = 16, - target_modules = "all-linear", - lora_alpha = 16, - lora_dropout = 0, - bias = "none", - layers_to_transform = None, - layers_pattern = None, + r = 16, + target_modules = None, + lora_alpha = 16, + lora_dropout = 0, + bias = "none", + finetune_vision_layers = True, + finetune_language_layers = True, + finetune_attention_modules = True, + finetune_mlp_modules = True, + layers_to_transform = None, + layers_pattern = None, use_gradient_checkpointing = True, - random_state = 3407, - max_seq_length = 2048, # not used anymore - use_rslora = False, - modules_to_save = None, - init_lora_weights = True, - loftq_config = {}, - temporary_location = "_unsloth_temporary_saved_buffers", + random_state = 3407, + max_seq_length = 2048, # not used anymore + use_rslora = False, + modules_to_save = None, + init_lora_weights = True, + loftq_config = {}, + temporary_location = "_unsloth_temporary_saved_buffers", **kwargs, ): transformers_set_seed(random_state) - # Get LoRA - arguments = dict( - r = r, - lora_alpha = lora_alpha, - target_modules = target_modules, - lora_dropout = lora_dropout, - bias = bias, - layers_to_transform = layers_to_transform, - init_lora_weights = init_lora_weights, - # loftq_config = loftq_config, - # use_rslora = use_rslora, - modules_to_save = modules_to_save, - **kwargs, - ) + if type(r) is not int: + raise TypeError(f"Unsloth: Rank of {str(r)} must be an integer.") + if r <= 0: + raise TypeError(f"Unsloth: Rank of {str(r)} must be larger than 0.") - lora_config = LoraConfig(**arguments) + if isinstance(model, PeftModelForCausalLM): + raise RuntimeError("Unsloth: You already added LoRA adapters to your model!") - model = _get_peft_model(model, lora_config) - - model = FastVisionModel.patch_peft_model(model, use_gradient_checkpointing) + if target_modules == "all-linear": + finetune_vision_layers = True + finetune_language_layers = True + finetune_attention_modules = True + finetune_mlp_modules = True + pass + if target_modules is None: + target_modules = get_peft_regex( + model, + finetune_vision_layers = finetune_vision_layers, + finetune_language_layers = finetune_language_layers, + finetune_attention_modules = finetune_attention_modules, + finetune_mlp_modules = finetune_mlp_modules, + ) + else: + assert(type(target_modules) in (list, tuple,)) + pass # Clear deleted GPU items for _ in range(3): @@ -387,35 +221,23 @@ class FastVisionModel: torch.cuda.empty_cache() pass - return model - pass - - - @staticmethod - def patch_peft_model( - model, - use_gradient_checkpointing = True, - ): - + lora_config = LoraConfig( + r = r, + lora_alpha = lora_alpha, + target_modules = target_modules, + lora_dropout = lora_dropout, + bias = bias, + task_type = TaskType.CAUSAL_LM, + ) + model = prepare_model_for_kbit_training( + model, + use_gradient_checkpointing = use_gradient_checkpointing, + ) + model = get_peft_model(model, lora_config) model = prepare_model_for_kbit_training( model, use_gradient_checkpointing = use_gradient_checkpointing, - use_reentrant = True, ) - - # Fix up config for transformers uploading PEFT - for active_adapter in model.peft_config.keys(): - # Not necessary since we requires transformers >= 4.37 - if False: - name = model.peft_config[active_adapter].base_model_name_or_path - if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): - name = name[:len(name) - len("-bnb-4bit")] - model.peft_config[active_adapter].base_model_name_or_path = name - pass - # Add revision to enable future fast inference paths - # [TODO] Bugs out!see https://github.com/unslothai/unsloth/issues/492 - # model.peft_config[active_adapter].revision = f"unsloth" - pass from transformers.trainer import Trainer if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": @@ -427,24 +249,6 @@ class FastVisionModel: ) pass - logger.warning_once( - f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "\ - f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.", - ) - patch_saving_functions(model) - - # Patch cross entropy loss labels - # Fixes https://github.com/unslothai/unsloth/issues/10 - max_seq_length = model.max_seq_length - extra_ignored_labels = torch.full((max_seq_length, 1), -100, device = "cuda:0") - model.model.extra_ignored_labels = extra_ignored_labels - internal_model = model - while hasattr(internal_model, "model"): - internal_model.max_seq_length = max_seq_length - internal_model = internal_model.model - pass - internal_model.max_seq_length = max_seq_length - # Patch tokenizer to pad to the right internal_model = model while hasattr(internal_model, "model"): @@ -462,6 +266,8 @@ class FastVisionModel: gc.collect() torch.cuda.empty_cache() pass + patch_saving_functions(model, vision = True) + return model pass @@ -500,6 +306,24 @@ class FastVisionModel: elif dtype == "bfloat16": dtype = torch.bfloat16 pass + # Wrap model.generate + if model.generate.__name__ != "_fast_generate": + model._unwrapped_old_generate = model.generate + model.generate = _wrap_fast_inference(model.generate, device_type, dtype, model) + pass + + # Patch tokenizer to pad to the left + internal_model = model + while hasattr(internal_model, "model"): + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "left" + pass + internal_model = internal_model.model + pass + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "left" + pass + # Also disable training for embeddings for NEFTune if hasattr(model, "get_input_embeddings"): embeddings = model.get_input_embeddings() @@ -520,12 +344,6 @@ class FastVisionModel: internal_model.gradient_checkpointing = use_gradient_checkpointing internal_model.training = True - # Delete all fast inference loras - for param in model.parameters(): - if hasattr(param, "_fast_lora"): - del param._fast_lora - pass - while hasattr(internal_model, "model"): internal_model = internal_model.model internal_model.gradient_checkpointing = use_gradient_checkpointing @@ -535,6 +353,24 @@ class FastVisionModel: internal_model.training = True pass + # Also revert model.generate + if hasattr(model, "_unwrapped_old_generate"): + model.generate = model._unwrapped_old_generate + del model._unwrapped_old_generate + pass + + # Patch tokenizer to pad to the right + internal_model = model + while hasattr(internal_model, "model"): + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "right" + pass + internal_model = internal_model.model + pass + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "right" + pass + # Also re-enable training for embeddings for NEFTune if hasattr(model, "get_input_embeddings"): embeddings = model.get_input_embeddings() @@ -548,3 +384,5 @@ class FastVisionModel: return model pass pass + + diff --git a/unsloth/save.py b/unsloth/save.py index b4c6b499cf..fe03368a6e 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2041,8 +2041,152 @@ def unsloth_convert_lora_to_ggml_and_save_locally( print("Unsloth: Done.") print(f"Unsloth: Conversion completed! Output file: {output_file}") print("\nThis GGML making function was made by Maheswar. Ping him @Maheswar on the Unsloth Discord or on HuggingFace (@mahiatlinux) if you like this!") +pass -def patch_saving_functions(model): + +from unsloth_zoo.peft_utils import merge_and_overwrite_lora +from .loader_utils import get_model_name + +@torch.inference_mode +def unsloth_generic_save( + model, + tokenizer, + save_directory : Union[str, os.PathLike] = "unsloth_finetuned_merge", + save_method : str = "lora", # ["lora", "merged_16bit", "merged_4bit"] + push_to_hub : bool = False, + token : Optional[Union[str, bool]] = None, + is_main_process : bool = True, + state_dict : Optional[dict] = None, + save_function : Callable = torch.save, + max_shard_size : Union[int, str] = "5GB", + safe_serialization : bool = True, + variant : Optional[str] = None, + save_peft_format : bool = True, + + # Push to hub + use_temp_dir : Optional[bool] = None, + commit_message : Optional[str] = "Trained with Unsloth", + private : Optional[bool] = None, + create_pr : bool = False, + revision : str = None, + commit_description : str = "Upload model trained with Unsloth 2x faster", + tags : List[str] = None, + + # Our functions + temporary_location : str = "_unsloth_temporary_saved_buffers", + maximum_memory_usage : float = 0.9, +): + if token is None: token = get_token() + merge_and_overwrite_lora( + get_model_name, + create_huggingface_repo, + model, + save_location = save_directory, + push_to_hub = push_to_hub, + token = token, + upload_location = save_directory, + low_disk_space_usage = True, + private = private, + ) + return +pass + + +def unsloth_generic_save_pretrained_merged( + self, + save_directory : Union[str, os.PathLike], + tokenizer = None, + save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"] + push_to_hub : bool = False, + token : Optional[Union[str, bool]] = None, + is_main_process : bool = True, + state_dict : Optional[dict] = None, + save_function : Callable = torch.save, + max_shard_size : Union[int, str] = "5GB", + safe_serialization : bool = True, + variant : Optional[str] = None, + save_peft_format : bool = True, + tags : List[str] = None, + temporary_location : str = "_unsloth_temporary_saved_buffers", + maximum_memory_usage : float = 0.75, +): + """ + Same as .push_to_hub(...) except 4bit weights are auto + converted to float16 with as few overhead as possible. + + Choose for `save_method` to be either: + 1. `16bit`: Merge LoRA into float16 weights. Useful for GGUF / llama.cpp. + 2. `4bit`: Merge LoRA into int4 weights. Useful for DPO / HF inference. + 3. `lora`: Save LoRA adapters with no merging. Useful for HF inference. + """ + if tokenizer is None: + logger.warning_once( + "Unsloth: You're not saving a tokenizer as well?\n"\ + "You can do it separately via `tokenizer.save_pretrained(...)`" + ) + pass + + arguments = dict(locals()) + arguments["model"] = self + del arguments["self"] + unsloth_generic_save(**arguments) + for _ in range(3): + gc.collect() +pass + + +def unsloth_generic_push_to_hub_merged( + self, + repo_id : str, + tokenizer = None, + save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"] + use_temp_dir : Optional[bool] = None, + commit_message : Optional[str] = "Trained with Unsloth", + private : Optional[bool] = None, + token : Union[bool, str, None] = None, + max_shard_size : Union[int, str, None] = "5GB", + create_pr : bool = False, + safe_serialization : bool = True, + revision : str = None, + commit_description : str = "Upload model trained with Unsloth 2x faster", + tags : Optional[List[str]] = None, + temporary_location : str = "_unsloth_temporary_saved_buffers", + maximum_memory_usage : float = 0.75, +): + """ + Same as .push_to_hub(...) except 4bit weights are auto + converted to float16 with as few overhead as possible. + + Choose for `save_method` to be either: + 1. `16bit`: Merge LoRA into float16 weights. Useful for GGUF / llama.cpp. + 2. `4bit`: Merge LoRA into int4 weights. Useful for DPO / HF inference. + 3. `lora`: Save LoRA adapters with no merging. Useful for HF inference. + """ + if tokenizer is None: + logger.warning_once( + "Unsloth: You're not saving a tokenizer as well?\n"\ + "You can do it separately via `tokenizer.push_to_hub(...)`" + ) + pass + + arguments = dict(locals()) + arguments["model"] = self + arguments["save_directory"] = repo_id + arguments["push_to_hub"] = True + del arguments["self"] + del arguments["repo_id"] + unsloth_generic_save(**arguments) + for _ in range(3): + gc.collect() +pass + + +def not_implemented_save(*args, **kwargs): + raise NotImplementedError("Unsloth: Sorry GGUF is currently not supported for vision models!") +pass + + +def patch_saving_functions(model, vision = False): import inspect import types from typing import Callable, Optional, Union, List @@ -2131,14 +2275,22 @@ def patch_saving_functions(model): pass # Add saving methods to top level model - if hasattr(model, "config"): - # Counteract tokenizers - model.push_to_hub_merged = types.MethodType(unsloth_push_to_hub_merged, model) - model.save_pretrained_merged = types.MethodType(unsloth_save_pretrained_merged, model) - model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) - model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model) - model.push_to_hub_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_push_to_hub, model) - model.save_pretrained_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_save_locally, model) + if not vision: + if hasattr(model, "config"): + # Counteract tokenizers + model.push_to_hub_merged = types.MethodType(unsloth_push_to_hub_merged, model) + model.save_pretrained_merged = types.MethodType(unsloth_save_pretrained_merged, model) + model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) + model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model) + model.push_to_hub_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_push_to_hub, model) + model.save_pretrained_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_save_locally, model) + pass + else: + # Vision only 1 option + model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model) + model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, model) + model.push_to_hub_gguf = types.MethodType(not_implemented_save, model) + model.save_pretrained_gguf = types.MethodType(not_implemented_save, model) pass return model pass