Update loader.py
This commit is contained in:
Daniel Han
GitHub
parent
4cae1aa0be
commit
3e1c5ec3a0
1 changed files with 52 additions and 62 deletions
|
|
@ -13,6 +13,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
from ._utils import is_bfloat16_supported, HAS_FLASH_ATTENTION, HAS_FLASH_ATTENTION_SOFTCAPPING
|
||||
from .granite import FastGraniteModel
|
||||
from .llama import FastLlamaModel, logger
|
||||
from .mistral import FastMistralModel
|
||||
from .qwen2 import FastQwen2Model
|
||||
|
|
@ -31,13 +32,14 @@ pass
|
|||
from huggingface_hub import HfFileSystem
|
||||
|
||||
# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
|
||||
from packaging.version import Version
|
||||
from unsloth_zoo.utils import Version, _get_dtype
|
||||
transformers_version = Version(transformers_version)
|
||||
SUPPORTS_FOURBIT = transformers_version >= Version("4.37")
|
||||
SUPPORTS_GEMMA = transformers_version >= Version("4.38")
|
||||
SUPPORTS_GEMMA2 = transformers_version >= Version("4.42")
|
||||
SUPPORTS_LLAMA31 = transformers_version >= Version("4.43.2")
|
||||
SUPPORTS_LLAMA32 = transformers_version > Version("4.45.0")
|
||||
SUPPORTS_GRANITE = transformers_version >= Version("4.46.0")
|
||||
if SUPPORTS_GEMMA:
|
||||
from .gemma import FastGemmaModel
|
||||
if SUPPORTS_GEMMA2:
|
||||
|
|
@ -45,28 +47,11 @@ if SUPPORTS_GEMMA2:
|
|||
pass
|
||||
import torch
|
||||
|
||||
def _get_dtype(dtype):
|
||||
__DTYPE_MAP = {
|
||||
"float32": torch.float32,
|
||||
torch.float32: torch.float32,
|
||||
"float16": torch.float16,
|
||||
torch.float16: torch.float16,
|
||||
"bfloat16": torch.bfloat16,
|
||||
torch.bfloat16: torch.bfloat16,
|
||||
}
|
||||
if dtype is None or dtype == None: return None
|
||||
elif dtype in __DTYPE_MAP: return __DTYPE_MAP[dtype]
|
||||
else:
|
||||
print(f"Unsloth: {dtype} is not recognized, so we'll default to None")
|
||||
return None
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class FastLanguageModel(FastLlamaModel):
|
||||
@staticmethod
|
||||
def from_pretrained(
|
||||
model_name = "unsloth/llama-3-8b-bnb-4bit",
|
||||
model_name = "unsloth/Llama-3.2-1B-Instruct",
|
||||
max_seq_length = None,
|
||||
dtype = None,
|
||||
load_in_4bit = True,
|
||||
|
|
@ -131,7 +116,8 @@ class FastLanguageModel(FastLlamaModel):
|
|||
exist_config = os.path.exists(os.path.join(model_name, "config.json"))
|
||||
both_exist = exist_adapter_config and exist_config
|
||||
else:
|
||||
files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json"))
|
||||
# Because HfFileSystem assumes linux paths, we need to set the path with forward slashes, even on Windows.
|
||||
files = HfFileSystem(token = token).glob(f"{model_name}/*.json")
|
||||
files = (os.path.split(x)[-1] for x in files)
|
||||
if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2:
|
||||
both_exist = True
|
||||
|
|
@ -164,10 +150,9 @@ class FastLanguageModel(FastLlamaModel):
|
|||
# Get base model for PEFT:
|
||||
if is_peft:
|
||||
# Check base model again for PEFT
|
||||
model_name = peft_config.base_model_name_or_path
|
||||
if not use_exact_model_name:
|
||||
model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit)
|
||||
else:
|
||||
model_name = peft_config.base_model_name_or_path
|
||||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
model_config = AutoConfig.from_pretrained(
|
||||
model_name,
|
||||
token = token,
|
||||
|
|
@ -180,7 +165,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
|
||||
model_type = model_config.model_type
|
||||
|
||||
if model_type == "llama":
|
||||
if model_type == "llama":
|
||||
scaling_type = None
|
||||
if getattr(model_config, "rope_scaling", None) is not None:
|
||||
scaling_type1 = model_config.rope_scaling.get("type", None)
|
||||
|
|
@ -236,6 +221,8 @@ class FastLanguageModel(FastLlamaModel):
|
|||
dispatch_model = FastQwen2Model
|
||||
elif model_type == "cohere":
|
||||
dispatch_model = FastCohereModel
|
||||
elif model_type == "granite":
|
||||
dispatch_model = FastGraniteModel
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Unsloth: {model_name} not supported yet!\n"\
|
||||
|
|
@ -254,8 +241,6 @@ class FastLanguageModel(FastLlamaModel):
|
|||
tokenizer_name = None
|
||||
pass
|
||||
|
||||
original_kwargs = kwargs.copy()
|
||||
|
||||
model, tokenizer = dispatch_model.from_pretrained(
|
||||
model_name = model_name,
|
||||
max_seq_length = max_seq_length,
|
||||
|
|
@ -269,7 +254,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
tokenizer_name = tokenizer_name,
|
||||
trust_remote_code = trust_remote_code,
|
||||
revision = revision if not is_peft else None,
|
||||
*args, **original_kwargs,
|
||||
*args, **kwargs,
|
||||
)
|
||||
|
||||
if resize_model_vocab is not None:
|
||||
|
|
@ -354,6 +339,8 @@ class FastVisionModel(FastBaseVisionModel):
|
|||
use_gradient_checkpointing = "unsloth",
|
||||
resize_model_vocab = None, # [TODO] No effect
|
||||
revision = None,
|
||||
return_logits = False, # Return logits
|
||||
fullgraph = True, # No graph breaks
|
||||
use_exact_model_name = False,
|
||||
*args, **kwargs,
|
||||
):
|
||||
|
|
@ -362,43 +349,17 @@ class FastVisionModel(FastBaseVisionModel):
|
|||
patch_compiled_autograd()
|
||||
patch_compiling_bitsandbytes()
|
||||
if use_gradient_checkpointing == "unsloth":
|
||||
patch_unsloth_smart_gradient_checkpointing()
|
||||
patch_unsloth_smart_gradient_checkpointing(dtype = dtype)
|
||||
|
||||
old_model_name = model_name
|
||||
if not use_exact_model_name:
|
||||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
|
||||
with contextlib.redirect_stdout(open(os.devnull, "w")):
|
||||
patch_loss_functions(torch_compile = False)
|
||||
model_types = unsloth_compile_transformers(
|
||||
model_name = model_name,
|
||||
sdpa_dynamic_mask = True,
|
||||
sdpa_bool_masks = True,
|
||||
sdpa_gqa_replace = True,
|
||||
sdpa_dynamic_compile = True,
|
||||
compile_attention = True,
|
||||
disable_causal_masks = True,
|
||||
compile_torch_modules = True,
|
||||
compile_custom_modules = True,
|
||||
compile_function_calls = True,
|
||||
fuse_lm_head = True,
|
||||
gradient_checkpointing = True,
|
||||
manual_replacements = True,
|
||||
epilogue_fusion = True,
|
||||
max_autotune = False,
|
||||
shape_padding = True,
|
||||
cudagraphs = False,
|
||||
debug = False,
|
||||
import_from_cache = False,
|
||||
disable = False,
|
||||
)
|
||||
pass
|
||||
|
||||
# First check if it's a normal model via AutoConfig
|
||||
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled
|
||||
was_disabled = are_progress_bars_disabled()
|
||||
disable_progress_bars()
|
||||
|
||||
|
||||
autoconfig_error = None
|
||||
peft_error = None
|
||||
try:
|
||||
|
|
@ -438,7 +399,7 @@ class FastVisionModel(FastBaseVisionModel):
|
|||
exist_config = os.path.exists(os.path.join(model_name, "config.json"))
|
||||
both_exist = exist_adapter_config and exist_config
|
||||
else:
|
||||
files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json"))
|
||||
files = HfFileSystem(token = token).glob(f"{model_name}/*.json")
|
||||
files = (os.path.split(x)[-1] for x in files)
|
||||
if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2:
|
||||
both_exist = True
|
||||
|
|
@ -471,10 +432,10 @@ class FastVisionModel(FastBaseVisionModel):
|
|||
# Get base model for PEFT:
|
||||
if is_peft:
|
||||
# Check base model again for PEFT
|
||||
model_name = peft_config.base_model_name_or_path
|
||||
if not use_exact_model_name:
|
||||
model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit)
|
||||
else:
|
||||
model_name = peft_config.base_model_name_or_path
|
||||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
|
||||
model_config = AutoConfig.from_pretrained(
|
||||
model_name,
|
||||
token = token,
|
||||
|
|
@ -485,6 +446,37 @@ class FastVisionModel(FastBaseVisionModel):
|
|||
|
||||
if not was_disabled: enable_progress_bars()
|
||||
|
||||
with contextlib.redirect_stdout(open(os.devnull, "w")):
|
||||
patch_loss_functions(torch_compile = False)
|
||||
model_types = unsloth_compile_transformers(
|
||||
model_name = model_name,
|
||||
sdpa_dynamic_mask = True,
|
||||
sdpa_bool_masks = True,
|
||||
sdpa_gqa_replace = True,
|
||||
sdpa_dynamic_compile = True,
|
||||
compile_attention = True,
|
||||
disable_causal_masks = True,
|
||||
compile_torch_modules = True,
|
||||
compile_custom_modules = True,
|
||||
compile_function_calls = True,
|
||||
fuse_lm_head = True,
|
||||
gradient_checkpointing = True,
|
||||
manual_replacements = True,
|
||||
fast_lora_forwards = True,
|
||||
fast_residual_stream = False,
|
||||
accurate_accumulation = True,
|
||||
epilogue_fusion = True,
|
||||
max_autotune = False,
|
||||
shape_padding = True,
|
||||
cudagraphs = False,
|
||||
debug = False,
|
||||
fullgraph = fullgraph,
|
||||
import_from_cache = False,
|
||||
disable = False,
|
||||
return_logits = return_logits,
|
||||
)
|
||||
pass
|
||||
|
||||
# Check if this is local model since the tokenizer gets overwritten
|
||||
if os.path.exists(os.path.join(old_model_name, "tokenizer_config.json")) and \
|
||||
os.path.exists(os.path.join(old_model_name, "tokenizer.json")) and \
|
||||
|
|
@ -495,8 +487,6 @@ class FastVisionModel(FastBaseVisionModel):
|
|||
tokenizer_name = None
|
||||
pass
|
||||
|
||||
original_kwargs = kwargs.copy()
|
||||
|
||||
model, tokenizer = FastBaseVisionModel.from_pretrained(
|
||||
model_name = model_name,
|
||||
max_seq_length = max_seq_length,
|
||||
|
|
@ -508,7 +498,7 @@ class FastVisionModel(FastBaseVisionModel):
|
|||
revision = revision if not is_peft else None,
|
||||
model_types = model_types,
|
||||
tokenizer_name = tokenizer_name,
|
||||
*args, **original_kwargs,
|
||||
*args, **kwargs,
|
||||
)
|
||||
|
||||
if resize_model_vocab is not None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue