ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix saving and bnb-4bit

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-25 19:05:43 +11:00
commit 380f2fd6e4
4 changed files with 39 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

25
unsloth/models/llama.py
View file

@ -725,7 +725,7 @@ class FastLlamaModel:
f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\
f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\
f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth'
f' "-____-" Free Apache license: http://github.com/unslothai/unsloth'
logger.warning_once(statistics)
FastLlamaModel.pre_patch()
@ -814,10 +814,13 @@ class FastLlamaModel:
patch_saving_functions(tokenizer)
# Fix up config for transformers uploading PEFT
name = model.config._name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[:len(name) - len("-bnb-4bit")]
model.config.update({"_name_or_path" : name})
# Not necessary anymore since we require transformers>=4.37!
if False:
name = model.config._name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[:len(name) - len("-bnb-4bit")]
model.config.update({"_name_or_path" : name})
pass
pass
# Log Unsloth version for future fastpaths for inference
@ -1020,11 +1023,13 @@ class FastLlamaModel:
# Fix up config for transformers uploading PEFT
for active_adapter in model.peft_config.keys():
name = model.peft_config[active_adapter].base_model_name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[:len(name) - len("-bnb-4bit")]
model.peft_config[active_adapter].base_model_name_or_path = name
pass
# Not necessary since we requires transformers >= 4.37
if False:
name = model.peft_config[active_adapter].base_model_name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[:len(name) - len("-bnb-4bit")]
model.peft_config[active_adapter].base_model_name_or_path = name
pass
# Add revision to enable future fast inference paths
model.peft_config[active_adapter].revision = f"unsloth"
pass

2
unsloth/models/loader.py
View file

@ -34,7 +34,7 @@ def _get_model_name(model_name, load_in_4bit = True):
logger.warning_once(
f"Unsloth: Your transformers version of {transformers_version} does not support native "\
f"4bit loading.\nThe minimum required version is 4.37.\n"\
f'Try `pip install "git+https://github.com/huggingface/transformers.git"`\n'\
f'Try `pip install --upgrade "transformers>=4.37"`\n'\
f"to obtain the latest transformers build, then restart this session.\n"\
f"For now, we shall load `{model_name}` instead (still 4bit, just slower downloading)."
)

12
unsloth/models/mistral.py
View file

@ -364,11 +364,13 @@ class FastMistralModel(FastLlamaModel):
patch_saving_functions(tokenizer)
# Fix up config for transformers uploading PEFT
name = model.config._name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[:len(name) - len("-bnb-4bit")]
model.config.update({"_name_or_path" : name})
pass
# Not necessary anymore since we require transformers>=4.37
if False:
name = model.config._name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[:len(name) - len("-bnb-4bit")]
model.config.update({"_name_or_path" : name})
pass
# Log Unsloth version for future fastpaths for inference
model.config.update({"unsloth_version" : __version__})

19
unsloth/save.py
View file

@ -135,6 +135,17 @@ def unsloth_save_model(
temporary_location : str = "_unsloth_temporary_saved_buffers",
maximum_memory_usage : float = 0.9,
):
if save_method == "merged_4bit":
raise RuntimeError(
"Unsloth: Merging into 4bit will cause your model to lose accuracy if you plan\n"\
"to merge to GGUF or others later on. I suggest you to do this as a final step\n"\
"if you're planning to do multiple saves.\n"\
"If you are certain, change `save_method` to `merged_4bit_forced`."
)
elif save_method == "merged_4bit_forced":
save_method = "merged_4bit"
pass
save_pretrained_settings = dict(locals())
for deletion in ("model", "tokenizer", "save_method", "temporary_location", "maximum_memory_usage"):
del save_pretrained_settings[deletion]
@ -457,6 +468,8 @@ pass
def install_llama_cpp_make_non_blocking():
env = { **os.environ, "LLAMA_CUBLAS": "1", }
n_jobs = max(int(psutil.cpu_count()*1.5), 1)
# Force make clean
os.system("make clean -C llama.cpp")
full_command = ["make", "-j", str(n_jobs), "-C", "llama.cpp"]
run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
return run_installer
@ -487,8 +500,8 @@ pass
def save_to_gguf(
model_directory : str = "unsloth_finetuned_model",
quantization_method : str = "fast_quantized",
model_directory : str = "unsloth_finetuned_model",
quantization_method : str = "fast_quantized",
_run_installer = None, # Non blocking install of llama.cpp
):
from transformers.models.llama.modeling_llama import logger
@ -566,7 +579,7 @@ def unsloth_save_pretrained_merged(
self,
save_directory : Union[str, os.PathLike],
tokenizer = None,
save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"]
save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"]
push_to_hub : bool = False,
token : Optional[Union[str, bool]] = None,
is_main_process : bool = True,