From 4cae1aa0bed2ab4bae565dbad1aad022923f357f Mon Sep 17 00:00:00 2001 From: Itsuro Tajima Date: Tue, 26 Nov 2024 20:20:34 +0900 Subject: [PATCH 1/2] use exact model name --- unsloth/models/loader.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 232fe6acff..19747cb4ef 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -78,12 +78,14 @@ class FastLanguageModel(FastLlamaModel): use_gradient_checkpointing = "unsloth", resize_model_vocab = None, revision = None, + use_exact_model_name = False, *args, **kwargs, ): if token is None: token = get_token() old_model_name = model_name - model_name = get_model_name(model_name, load_in_4bit) + if not use_exact_model_name: + model_name = get_model_name(model_name, load_in_4bit) # First check if it's a normal model via AutoConfig from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled @@ -162,7 +164,10 @@ class FastLanguageModel(FastLlamaModel): # Get base model for PEFT: if is_peft: # Check base model again for PEFT - model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) + if not use_exact_model_name: + model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) + else: + model_name = peft_config.base_model_name_or_path model_config = AutoConfig.from_pretrained( model_name, token = token, @@ -249,6 +254,8 @@ class FastLanguageModel(FastLlamaModel): tokenizer_name = None pass + original_kwargs = kwargs.copy() + model, tokenizer = dispatch_model.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, @@ -262,7 +269,7 @@ class FastLanguageModel(FastLlamaModel): tokenizer_name = tokenizer_name, trust_remote_code = trust_remote_code, revision = revision if not is_peft else None, - *args, **kwargs, + *args, **original_kwargs, ) if resize_model_vocab is not None: @@ -347,6 +354,7 @@ class FastVisionModel(FastBaseVisionModel): use_gradient_checkpointing = "unsloth", resize_model_vocab = None, # [TODO] No effect revision = None, + use_exact_model_name = False, *args, **kwargs, ): if token is None: token = get_token() @@ -357,7 +365,8 @@ class FastVisionModel(FastBaseVisionModel): patch_unsloth_smart_gradient_checkpointing() old_model_name = model_name - model_name = get_model_name(model_name, load_in_4bit) + if not use_exact_model_name: + model_name = get_model_name(model_name, load_in_4bit) with contextlib.redirect_stdout(open(os.devnull, "w")): patch_loss_functions(torch_compile = False) @@ -462,7 +471,10 @@ class FastVisionModel(FastBaseVisionModel): # Get base model for PEFT: if is_peft: # Check base model again for PEFT - model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) + if not use_exact_model_name: + model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) + else: + model_name = peft_config.base_model_name_or_path model_config = AutoConfig.from_pretrained( model_name, token = token, @@ -483,6 +495,8 @@ class FastVisionModel(FastBaseVisionModel): tokenizer_name = None pass + original_kwargs = kwargs.copy() + model, tokenizer = FastBaseVisionModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, @@ -494,7 +508,7 @@ class FastVisionModel(FastBaseVisionModel): revision = revision if not is_peft else None, model_types = model_types, tokenizer_name = tokenizer_name, - *args, **kwargs, + *args, **original_kwargs, ) if resize_model_vocab is not None: From 3e1c5ec3a039cade71c1191bd8c0ef62cd19c0a2 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 4 Jan 2025 22:03:11 -0800 Subject: [PATCH 2/2] Update loader.py --- unsloth/models/loader.py | 114 ++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 62 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 19747cb4ef..a881146692 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -13,6 +13,7 @@ # limitations under the License. from ._utils import is_bfloat16_supported, HAS_FLASH_ATTENTION, HAS_FLASH_ATTENTION_SOFTCAPPING +from .granite import FastGraniteModel from .llama import FastLlamaModel, logger from .mistral import FastMistralModel from .qwen2 import FastQwen2Model @@ -31,13 +32,14 @@ pass from huggingface_hub import HfFileSystem # https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading! -from packaging.version import Version +from unsloth_zoo.utils import Version, _get_dtype transformers_version = Version(transformers_version) SUPPORTS_FOURBIT = transformers_version >= Version("4.37") SUPPORTS_GEMMA = transformers_version >= Version("4.38") SUPPORTS_GEMMA2 = transformers_version >= Version("4.42") SUPPORTS_LLAMA31 = transformers_version >= Version("4.43.2") SUPPORTS_LLAMA32 = transformers_version > Version("4.45.0") +SUPPORTS_GRANITE = transformers_version >= Version("4.46.0") if SUPPORTS_GEMMA: from .gemma import FastGemmaModel if SUPPORTS_GEMMA2: @@ -45,28 +47,11 @@ if SUPPORTS_GEMMA2: pass import torch -def _get_dtype(dtype): - __DTYPE_MAP = { - "float32": torch.float32, - torch.float32: torch.float32, - "float16": torch.float16, - torch.float16: torch.float16, - "bfloat16": torch.bfloat16, - torch.bfloat16: torch.bfloat16, - } - if dtype is None or dtype == None: return None - elif dtype in __DTYPE_MAP: return __DTYPE_MAP[dtype] - else: - print(f"Unsloth: {dtype} is not recognized, so we'll default to None") - return None - pass -pass - class FastLanguageModel(FastLlamaModel): @staticmethod def from_pretrained( - model_name = "unsloth/llama-3-8b-bnb-4bit", + model_name = "unsloth/Llama-3.2-1B-Instruct", max_seq_length = None, dtype = None, load_in_4bit = True, @@ -131,7 +116,8 @@ class FastLanguageModel(FastLlamaModel): exist_config = os.path.exists(os.path.join(model_name, "config.json")) both_exist = exist_adapter_config and exist_config else: - files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json")) + # Because HfFileSystem assumes linux paths, we need to set the path with forward slashes, even on Windows. + files = HfFileSystem(token = token).glob(f"{model_name}/*.json") files = (os.path.split(x)[-1] for x in files) if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2: both_exist = True @@ -164,10 +150,9 @@ class FastLanguageModel(FastLlamaModel): # Get base model for PEFT: if is_peft: # Check base model again for PEFT + model_name = peft_config.base_model_name_or_path if not use_exact_model_name: - model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) - else: - model_name = peft_config.base_model_name_or_path + model_name = get_model_name(model_name, load_in_4bit) model_config = AutoConfig.from_pretrained( model_name, token = token, @@ -180,7 +165,7 @@ class FastLanguageModel(FastLlamaModel): model_type = model_config.model_type - if model_type == "llama": + if model_type == "llama": scaling_type = None if getattr(model_config, "rope_scaling", None) is not None: scaling_type1 = model_config.rope_scaling.get("type", None) @@ -236,6 +221,8 @@ class FastLanguageModel(FastLlamaModel): dispatch_model = FastQwen2Model elif model_type == "cohere": dispatch_model = FastCohereModel + elif model_type == "granite": + dispatch_model = FastGraniteModel else: raise NotImplementedError( f"Unsloth: {model_name} not supported yet!\n"\ @@ -254,8 +241,6 @@ class FastLanguageModel(FastLlamaModel): tokenizer_name = None pass - original_kwargs = kwargs.copy() - model, tokenizer = dispatch_model.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, @@ -269,7 +254,7 @@ class FastLanguageModel(FastLlamaModel): tokenizer_name = tokenizer_name, trust_remote_code = trust_remote_code, revision = revision if not is_peft else None, - *args, **original_kwargs, + *args, **kwargs, ) if resize_model_vocab is not None: @@ -354,6 +339,8 @@ class FastVisionModel(FastBaseVisionModel): use_gradient_checkpointing = "unsloth", resize_model_vocab = None, # [TODO] No effect revision = None, + return_logits = False, # Return logits + fullgraph = True, # No graph breaks use_exact_model_name = False, *args, **kwargs, ): @@ -362,43 +349,17 @@ class FastVisionModel(FastBaseVisionModel): patch_compiled_autograd() patch_compiling_bitsandbytes() if use_gradient_checkpointing == "unsloth": - patch_unsloth_smart_gradient_checkpointing() + patch_unsloth_smart_gradient_checkpointing(dtype = dtype) old_model_name = model_name if not use_exact_model_name: model_name = get_model_name(model_name, load_in_4bit) - with contextlib.redirect_stdout(open(os.devnull, "w")): - patch_loss_functions(torch_compile = False) - model_types = unsloth_compile_transformers( - model_name = model_name, - sdpa_dynamic_mask = True, - sdpa_bool_masks = True, - sdpa_gqa_replace = True, - sdpa_dynamic_compile = True, - compile_attention = True, - disable_causal_masks = True, - compile_torch_modules = True, - compile_custom_modules = True, - compile_function_calls = True, - fuse_lm_head = True, - gradient_checkpointing = True, - manual_replacements = True, - epilogue_fusion = True, - max_autotune = False, - shape_padding = True, - cudagraphs = False, - debug = False, - import_from_cache = False, - disable = False, - ) - pass - # First check if it's a normal model via AutoConfig from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled was_disabled = are_progress_bars_disabled() disable_progress_bars() - + autoconfig_error = None peft_error = None try: @@ -438,7 +399,7 @@ class FastVisionModel(FastBaseVisionModel): exist_config = os.path.exists(os.path.join(model_name, "config.json")) both_exist = exist_adapter_config and exist_config else: - files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json")) + files = HfFileSystem(token = token).glob(f"{model_name}/*.json") files = (os.path.split(x)[-1] for x in files) if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2: both_exist = True @@ -471,10 +432,10 @@ class FastVisionModel(FastBaseVisionModel): # Get base model for PEFT: if is_peft: # Check base model again for PEFT + model_name = peft_config.base_model_name_or_path if not use_exact_model_name: - model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) - else: - model_name = peft_config.base_model_name_or_path + model_name = get_model_name(model_name, load_in_4bit) + model_config = AutoConfig.from_pretrained( model_name, token = token, @@ -485,6 +446,37 @@ class FastVisionModel(FastBaseVisionModel): if not was_disabled: enable_progress_bars() + with contextlib.redirect_stdout(open(os.devnull, "w")): + patch_loss_functions(torch_compile = False) + model_types = unsloth_compile_transformers( + model_name = model_name, + sdpa_dynamic_mask = True, + sdpa_bool_masks = True, + sdpa_gqa_replace = True, + sdpa_dynamic_compile = True, + compile_attention = True, + disable_causal_masks = True, + compile_torch_modules = True, + compile_custom_modules = True, + compile_function_calls = True, + fuse_lm_head = True, + gradient_checkpointing = True, + manual_replacements = True, + fast_lora_forwards = True, + fast_residual_stream = False, + accurate_accumulation = True, + epilogue_fusion = True, + max_autotune = False, + shape_padding = True, + cudagraphs = False, + debug = False, + fullgraph = fullgraph, + import_from_cache = False, + disable = False, + return_logits = return_logits, + ) + pass + # Check if this is local model since the tokenizer gets overwritten if os.path.exists(os.path.join(old_model_name, "tokenizer_config.json")) and \ os.path.exists(os.path.join(old_model_name, "tokenizer.json")) and \ @@ -495,8 +487,6 @@ class FastVisionModel(FastBaseVisionModel): tokenizer_name = None pass - original_kwargs = kwargs.copy() - model, tokenizer = FastBaseVisionModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, @@ -508,7 +498,7 @@ class FastVisionModel(FastBaseVisionModel): revision = revision if not is_peft else None, model_types = model_types, tokenizer_name = tokenizer_name, - *args, **original_kwargs, + *args, **kwargs, ) if resize_model_vocab is not None: