Fix saving issues (#139)
* faster saving & inference
* Update llama.py
* Update save.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update mistral.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* fast inference
* Update llama.py
* Update save.py
* Update llama.py
* Mistral correct RoPE scaling
* Max sequence lengths
* Apache 2
* fast_linear_forward
* Update utils.py
* Update utils.py
* No print
* Update utils.py
* Update utils.py
* inference
* Update llama.py
* Fast inference RoPE
* Update llama.py
* Update llama.py
* RoPE
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* LoRA
* Fast LoRA saving
* Update llama.py
* hidden_states
* q_len == 1
* q_len issue
* Update mistral.py
* Update mistral.py
* incorrect inference
* Update to transformers 4.37
* Graceful FA2 error + torch 2.1.1
* Update mapper.py
* Update pyproject.toml
* Fix saving and bnb-4bit
* Update fast_lora.py
* Update fast_lora.py
* remove patching
* Update llama.py
* Update llama.py
* Update swiglu.py
* Repatch
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update llama.py
* Update fast_lora.py
* Update llama.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update swiglu.py
* Update fast_lora.py
* Update swiglu.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update save.py
* Update fast_lora.py
* Update utils.py
* Update llama.py
* Update fast_lora.py
* Update swiglu.py
* Update save.py
* Update save.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Revert "Update llama.py"
This reverts commit a208ec46e0.
* Update llama.py
* Works?
* Update pyproject.toml
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Swiglu
* Update swiglu.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update swiglu.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* attention_mask
* Update llama.py
* Update llama.py
* labels
* Update mistral.py
* Update llama.py
* attention mask
* Update save.py
* Update save.py
* Update mistral.py
* attention mask
* Update llama.py
* Update llama.py
* Update mistral.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update dpo.py
* Patch saving
* Update save.py
* Update save.py
* patch_saving_functions
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* print
This commit is contained in:
Daniel Han
GitHub
parent
af33224554
commit
a16bc73e80
4 changed files with 111 additions and 50 deletions
|
|
@ -101,10 +101,13 @@ pass
|
|||
|
||||
|
||||
def PatchDPOTrainer():
|
||||
# Patch DPO notebook printing
|
||||
NotebookTrainingTracker.write_line = NotebookTrainingTracker_write_line
|
||||
from transformers.trainer import DEFAULT_PROGRESS_CALLBACK
|
||||
DEFAULT_PROGRESS_CALLBACK.on_train_begin = NotebookProgressCallback_on_train_begin
|
||||
DEFAULT_PROGRESS_CALLBACK.on_log = NotebookProgressCallback_on_log
|
||||
from transformers.trainer import is_in_notebook
|
||||
if is_in_notebook():
|
||||
# Patch DPO notebook printing
|
||||
NotebookTrainingTracker.write_line = NotebookTrainingTracker_write_line
|
||||
from transformers.trainer import DEFAULT_PROGRESS_CALLBACK
|
||||
DEFAULT_PROGRESS_CALLBACK.on_train_begin = NotebookProgressCallback_on_train_begin
|
||||
DEFAULT_PROGRESS_CALLBACK.on_log = NotebookProgressCallback_on_log
|
||||
pass
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -486,6 +486,15 @@ def LlamaModel_fast_forward(
|
|||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
|
||||
# Fix up attention mask by setting elements to 0
|
||||
# Specifically for DPO
|
||||
if self._has_no_labels and attention_mask is not None:
|
||||
inputs_requires_grad = inputs_embeds.requires_grad
|
||||
if inputs_requires_grad: inputs_embeds.requires_grad_(False)
|
||||
inputs_embeds *= attention_mask.unsqueeze(0).transpose(0, 1).transpose(1, 2)
|
||||
if inputs_requires_grad: inputs_embeds.requires_grad_(True)
|
||||
pass
|
||||
|
||||
# Ignore attention_mask
|
||||
if attention_mask is None:
|
||||
padding_mask = None
|
||||
|
|
@ -617,6 +626,7 @@ def LlamaForCausalLM_fast_forward(
|
|||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||
|
||||
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
||||
self.model._has_no_labels = labels is None
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
causal_mask=causal_mask,
|
||||
|
|
@ -726,7 +736,7 @@ class FastLlamaModel:
|
|||
f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
|
||||
f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\
|
||||
f' "-____-" Free Apache license: http://github.com/unslothai/unsloth'
|
||||
logger.warning_once(statistics)
|
||||
print(statistics)
|
||||
FastLlamaModel.pre_patch()
|
||||
|
||||
if dtype is None:
|
||||
|
|
@ -826,6 +836,9 @@ class FastLlamaModel:
|
|||
# Log Unsloth version for future fastpaths for inference
|
||||
model.config.update({"unsloth_version" : __version__})
|
||||
|
||||
# Add save modules
|
||||
patch_saving_functions(model)
|
||||
|
||||
return model, tokenizer
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -195,6 +195,7 @@ def MistralForCausalLM_fast_forward(
|
|||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||
|
||||
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
||||
self.model._has_no_labels = labels is None
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
causal_mask=causal_mask,
|
||||
|
|
@ -282,7 +283,7 @@ class FastMistralModel(FastLlamaModel):
|
|||
f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
|
||||
f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\
|
||||
f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth'
|
||||
logger.warning_once(statistics)
|
||||
print(statistics)
|
||||
FastMistralModel.pre_patch()
|
||||
|
||||
if dtype is None:
|
||||
|
|
|
|||
130
unsloth/save.py
130
unsloth/save.py
|
|
@ -278,7 +278,7 @@ def unsloth_save_model(
|
|||
not hasattr(internal_model.model, "layers")
|
||||
):
|
||||
# Do general saving
|
||||
|
||||
print(type(model))
|
||||
# Edit save_pretrained_settings
|
||||
# [TODO] _create_repo has errors due to **kwargs getting accepted
|
||||
for deletion in \
|
||||
|
|
@ -483,7 +483,7 @@ def install_llama_cpp_make_non_blocking():
|
|||
n_jobs = max(int(psutil.cpu_count()*1.5), 1)
|
||||
# Force make clean
|
||||
os.system("make clean -C llama.cpp")
|
||||
full_command = ["make", "-j", str(n_jobs), "-C", "llama.cpp"]
|
||||
full_command = ["make", "all", "-j", str(n_jobs), "-C", "llama.cpp"]
|
||||
run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
|
||||
return run_installer
|
||||
pass
|
||||
|
|
@ -499,7 +499,7 @@ pass
|
|||
def install_llama_cpp_blocking():
|
||||
commands = [
|
||||
"git clone https://github.com/ggerganov/llama.cpp",
|
||||
f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make -j {psutil.cpu_count()*2}",
|
||||
f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j {psutil.cpu_count()*2}",
|
||||
"pip install gguf protobuf",
|
||||
]
|
||||
if os.path.exists("llama.cpp"): return
|
||||
|
|
@ -515,6 +515,7 @@ pass
|
|||
def save_to_gguf(
|
||||
model_directory : str = "unsloth_finetuned_model",
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
_run_installer = None, # Non blocking install of llama.cpp
|
||||
):
|
||||
from transformers.models.llama.modeling_llama import logger
|
||||
|
|
@ -539,6 +540,16 @@ def save_to_gguf(
|
|||
f' "-____-" In total, you will have to wait around 26 minutes.\n'
|
||||
print(print_info)
|
||||
|
||||
# Check first_conversion format
|
||||
if first_conversion == "f16" : pass
|
||||
elif first_conversion == "f32" : pass
|
||||
elif first_conversion == "q8_0": pass
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: `first_conversion` can only be one of ['f16', 'f32', 'q8_0'] and not `{first_conversion}`."
|
||||
)
|
||||
pass
|
||||
|
||||
print("Unsloth: [0] Installing llama.cpp. This will take 3 minutes...")
|
||||
if _run_installer is not None:
|
||||
_run_installer.wait()
|
||||
|
|
@ -546,11 +557,19 @@ def save_to_gguf(
|
|||
install_llama_cpp_blocking()
|
||||
pass
|
||||
|
||||
print("Unsloth: [1] Converting HF into GGUF format. This will take 3 minutes...")
|
||||
first_conversion = "f16"
|
||||
if quantization_method == "f32": first_conversion = "f32"
|
||||
elif quantization_method == "f16": first_conversion = "f16"
|
||||
elif quantization_method == "q8_0": first_conversion = "q8_0"
|
||||
else:
|
||||
# Quantized models must have f16 as the default argument
|
||||
if first_conversion == "f32" : pass
|
||||
elif first_conversion == "f16" : pass
|
||||
elif first_conversion == "q8_0":
|
||||
logger.warning_once("Unsloth: We must use f16 for quantization first.")
|
||||
first_conversion = "f16"
|
||||
pass
|
||||
pass
|
||||
print(f"Unsloth: [1] Converting HF into {first_conversion} GGUF format. This will take 3 minutes...")
|
||||
|
||||
n_cpus = psutil.cpu_count()*2
|
||||
# Concurrency from https://rentry.org/llama-cpp-conversions#merging-loras-into-a-model
|
||||
|
|
@ -566,6 +585,17 @@ def save_to_gguf(
|
|||
print(line.decode("utf-8"), flush = True, end = "")
|
||||
pass
|
||||
|
||||
# Check if quantization succeeded!
|
||||
if not os.path.isfile(final_location):
|
||||
raise RuntimeError(
|
||||
"Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
print(f"Unsloth: Conversion completed! Output location: {final_location}")
|
||||
|
||||
if quantization_method != first_conversion:
|
||||
|
|
@ -581,6 +611,19 @@ def save_to_gguf(
|
|||
for line in sp.stderr:
|
||||
print(line.decode("utf-8"), flush = True, end = "")
|
||||
pass
|
||||
|
||||
# Check if quantization succeeded!
|
||||
if not os.path.isfile(final_location):
|
||||
raise RuntimeError(
|
||||
"Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
|
||||
print(f"Unsloth: Conversion completed! Output location: {final_location}")
|
||||
pass
|
||||
|
||||
|
|
@ -765,6 +808,7 @@ def unsloth_save_pretrained_gguf(
|
|||
save_directory : Union[str, os.PathLike],
|
||||
tokenizer = None,
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
push_to_hub : bool = False,
|
||||
token : Optional[Union[str, bool]] = None,
|
||||
is_main_process : bool = True,
|
||||
|
|
@ -813,6 +857,7 @@ def unsloth_save_pretrained_gguf(
|
|||
arguments["save_method"] = "merged_16bit" # Must be 16bit
|
||||
del arguments["self"]
|
||||
del arguments["quantization_method"]
|
||||
del arguments["first_conversion"]
|
||||
|
||||
# Non blocking install GGUF first
|
||||
if not os.path.exists("llama.cpp"):
|
||||
|
|
@ -840,7 +885,7 @@ def unsloth_save_pretrained_gguf(
|
|||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, makefile)
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
|
||||
|
||||
if push_to_hub:
|
||||
print("Unsloth: Uploading GGUF to Huggingface Hub...")
|
||||
|
|
@ -861,6 +906,7 @@ def unsloth_push_to_hub_gguf(
|
|||
repo_id : str,
|
||||
tokenizer = None,
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
use_temp_dir : Optional[bool] = None,
|
||||
commit_message : Optional[str] = None,
|
||||
private : Optional[bool] = None,
|
||||
|
|
@ -911,6 +957,7 @@ def unsloth_push_to_hub_gguf(
|
|||
del arguments["self"]
|
||||
del arguments["repo_id"]
|
||||
del arguments["quantization_method"]
|
||||
del arguments["first_conversion"]
|
||||
|
||||
# Non blocking install GGUF first
|
||||
if not os.path.exists("llama.cpp"):
|
||||
|
|
@ -938,7 +985,7 @@ def unsloth_push_to_hub_gguf(
|
|||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, makefile)
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
|
||||
|
||||
print("Unsloth: Uploading GGUF to Huggingface Hub...")
|
||||
username = upload_to_huggingface(
|
||||
|
|
@ -960,6 +1007,23 @@ def patch_saving_functions(model):
|
|||
|
||||
if hasattr(model, "_original_push_to_hub"): return
|
||||
|
||||
# First check if this has already been called, and revert it
|
||||
original_model = model
|
||||
while True:
|
||||
if hasattr(original_model, "_original_push_to_hub"):
|
||||
original_model.push_to_hub = original_model._original_push_to_hub
|
||||
del original_model._original_push_to_hub
|
||||
if hasattr(original_model, "push_to_hub_merged"): del original_model.push_to_hub_merged
|
||||
if hasattr(original_model, "save_pretrained_merged"): del original_model.save_pretrained_merged
|
||||
if hasattr(original_model, "push_to_hub_gguf"): del original_model.push_to_hub_gguf
|
||||
if hasattr(original_model, "save_pretrained_gguf"): del original_model.save_pretrained_gguf
|
||||
pass
|
||||
|
||||
if hasattr(original_model, "model"): original_model = original_model.model
|
||||
else: break
|
||||
pass
|
||||
|
||||
# And now re add our saving methods!
|
||||
original_push_to_hub = model.push_to_hub
|
||||
signature = str(inspect.signature(original_push_to_hub)).replace("NoneType", "None")
|
||||
signature = signature[1:]
|
||||
|
|
@ -988,49 +1052,29 @@ def patch_saving_functions(model):
|
|||
pass
|
||||
'''
|
||||
exec(push_to_hub_text, globals())
|
||||
model.push_to_hub = types.MethodType(unsloth_push_to_hub, model)
|
||||
|
||||
if hasattr(model, "add_model_tags"):
|
||||
model.add_model_tags(["unsloth",])
|
||||
original_model = model
|
||||
while True:
|
||||
|
||||
if not hasattr(original_model, "_original_push_to_hub"):
|
||||
original_model._original_push_to_hub = original_model.push_to_hub
|
||||
original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model)
|
||||
|
||||
if hasattr(original_model, "add_model_tags"):
|
||||
original_model.add_model_tags(["unsloth",])
|
||||
pass
|
||||
|
||||
if hasattr(original_model, "model"): original_model = original_model.model
|
||||
else: break
|
||||
pass
|
||||
|
||||
# Add saving methods to top level model
|
||||
if hasattr(model, "config"):
|
||||
# Counteract tokenizers
|
||||
model.push_to_hub_merged = types.MethodType(unsloth_push_to_hub_merged, model)
|
||||
model.save_pretrained_merged = types.MethodType(unsloth_save_pretrained_merged, model)
|
||||
model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model)
|
||||
model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model)
|
||||
else:
|
||||
model.push_to_hub_merged = model.push_to_hub
|
||||
model.save_pretrained_merged = model.save_pretrained
|
||||
model.push_to_hub_gguf = model.push_to_hub
|
||||
model.save_pretrained_gguf = model.save_pretrained
|
||||
pass
|
||||
|
||||
original_model = model
|
||||
while hasattr(original_model, "model"):
|
||||
original_model = original_model.model
|
||||
if hasattr(original_model, "_original_push_to_hub"): continue
|
||||
|
||||
original_model._original_push_to_hub = original_model.push_to_hub
|
||||
original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model)
|
||||
|
||||
if hasattr(original_model, "add_model_tags"):
|
||||
original_model.add_model_tags(["unsloth",])
|
||||
|
||||
if hasattr(original_model, "config"):
|
||||
# Counteract tokenizers
|
||||
original_model.push_to_hub_merged = \
|
||||
types.MethodType(unsloth_push_to_hub_merged, original_model)
|
||||
|
||||
original_model.save_pretrained_merged = \
|
||||
types.MethodType(unsloth_save_pretrained_merged, original_model)
|
||||
|
||||
original_model.push_to_hub_gguf = \
|
||||
types.MethodType(unsloth_push_to_hub_gguf, original_model)
|
||||
|
||||
original_model.save_pretrained_gguf = \
|
||||
types.MethodType(unsloth_save_pretrained_gguf, original_model)
|
||||
pass
|
||||
pass
|
||||
return
|
||||
return model
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue