Fix llama-3 (#423)
* Fix prompt * Update chat_templates.py * fix_untrained_tokens * Update llama.py * add tokens * Update _utils.py * Update tokenizer_utils.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * pad_token * Update chat_templates.py * Update chat_templates.py * tokenizer * Update save.py * Update chat_templates.py * Update chat_templates.py * patch tokenizer padding * Update tokenizer_utils.py * Update save.py * Fix: loading models with resized vocabulary (#377) * new: vocab resize on load * new: gitignore * GGUF fix * Readme (#390) * Update README.md * Update README.md --------- Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> * Update README.md * Delete .gitignore * Phi-3 * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Fix reserved tokens * Update save.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py --------- Co-authored-by: Igor Kilbas <whitemarsstudios@gmail.com> Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
0560136e94
commit
4a802a4bb1
3 changed files with 119 additions and 16 deletions
|
|
@ -1503,10 +1503,16 @@ class FastLlamaModel:
|
|||
pass
|
||||
pass
|
||||
|
||||
# Check for Llama-3
|
||||
# if hasattr(model._saved_temp_tokenizer, "_using_llama3_template"):
|
||||
# if not train_embed_tokens and not train_lm_head:
|
||||
# raise RuntimeError("")
|
||||
|
||||
# First fix untrained tokens
|
||||
if train_embed_tokens or train_lm_head:
|
||||
fix_untrained_tokens(model, eps = 1e-16)
|
||||
pass
|
||||
# Wrong - can cause reserved tokens to pop out!!
|
||||
# if train_embed_tokens or train_lm_head:
|
||||
# fix_untrained_tokens(model, eps = 1e-16)
|
||||
# pass
|
||||
|
||||
# Check modules_to_save
|
||||
if modules_to_save is not None:
|
||||
|
|
@ -1547,7 +1553,7 @@ class FastLlamaModel:
|
|||
|
||||
lora_config = LoraConfig(**arguments)
|
||||
model = _get_peft_model(model, lora_config)
|
||||
|
||||
|
||||
model._saved_temp_tokenizer = _saved_temp_tokenizer
|
||||
|
||||
model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
|
||||
|
|
|
|||
|
|
@ -118,14 +118,14 @@ def _merge_lora(layer, name):
|
|||
W = fast_dequantize(W, quant_state)
|
||||
else:
|
||||
dtype = W.dtype
|
||||
# W = W.to(torch.float32).t()
|
||||
W = W.t()
|
||||
W = W.to(torch.float32).t()
|
||||
# W = W.t()
|
||||
|
||||
if A is not None:
|
||||
# sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32)))
|
||||
# W += sAB
|
||||
# W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)
|
||||
W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s)
|
||||
W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)
|
||||
# W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s)
|
||||
# if not torch.isfinite(W).all():
|
||||
maximum_element = torch.max(W.min().abs(), W.max())
|
||||
if not torch.isfinite(maximum_element).item():
|
||||
|
|
@ -696,12 +696,18 @@ pass
|
|||
|
||||
|
||||
def install_llama_cpp_make_non_blocking():
|
||||
env = { **os.environ, "LLAMA_CUDA": "1", }
|
||||
# https://github.com/ggerganov/llama.cpp/issues/7062
|
||||
# Weirdly GPU conversion for GGUF breaks??
|
||||
# env = { **os.environ, "LLAMA_CUDA": "1", }
|
||||
n_jobs = max(int(psutil.cpu_count()*1.5), 1)
|
||||
# Force make clean
|
||||
os.system("make clean -C llama.cpp")
|
||||
full_command = ["make", "all", "-j"+str(n_jobs), "-C", "llama.cpp"]
|
||||
run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
|
||||
|
||||
# https://github.com/ggerganov/llama.cpp/issues/7062
|
||||
# Weirdly GPU conversion for GGUF breaks??
|
||||
# run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
|
||||
run_installer = subprocess.Popen(full_command, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
|
||||
return run_installer
|
||||
pass
|
||||
|
||||
|
|
@ -764,12 +770,17 @@ pass
|
|||
|
||||
|
||||
def install_llama_cpp_blocking(use_cuda = True):
|
||||
use_cuda = "LLAMA_CUDA=1" if use_cuda else ""
|
||||
# https://github.com/ggerganov/llama.cpp/issues/7062
|
||||
# Weirdly GPU conversion for GGUF breaks??
|
||||
# use_cuda = "LLAMA_CUDA=1" if use_cuda else ""
|
||||
|
||||
commands = [
|
||||
"git clone --recursive https://github.com/ggerganov/llama.cpp",
|
||||
"make clean -C llama.cpp",
|
||||
f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
# https://github.com/ggerganov/llama.cpp/issues/7062
|
||||
# Weirdly GPU conversion for GGUF breaks??
|
||||
# f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
f"make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
"pip install gguf protobuf",
|
||||
]
|
||||
if os.path.exists("llama.cpp"): return
|
||||
|
|
@ -833,6 +844,12 @@ def save_to_gguf(
|
|||
first_conversion : str = "f16",
|
||||
_run_installer = None, # Non blocking install of llama.cpp
|
||||
):
|
||||
logger.warning(
|
||||
"WARNING: llama.cpp GGUF conversion is currently unstable, since llama.cpp is\n"\
|
||||
"undergoing some major bug fixes as at 5th of May 2024. This is not an Unsloth issue.\n"\
|
||||
"Please be patient - GGUF saving should still work, but might not work as well."
|
||||
)
|
||||
|
||||
from transformers.models.llama.modeling_llama import logger
|
||||
|
||||
if quantization_method.startswith("iq2"):
|
||||
|
|
@ -967,7 +984,7 @@ def save_to_gguf(
|
|||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone --recursive https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"cd llama.cpp && make clean && make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
|
|
@ -1007,7 +1024,7 @@ def save_to_gguf(
|
|||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone --recursive https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"cd llama.cpp && make clean && make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ __all__ = [
|
|||
"load_correct_tokenizer",
|
||||
"fix_sentencepiece_tokenizer",
|
||||
"check_tokenizer",
|
||||
"fix_untrained_tokens",
|
||||
"add_new_tokens",
|
||||
]
|
||||
|
||||
|
|
@ -518,6 +517,44 @@ def fix_untrained_tokens(model, eps = 1e-16):
|
|||
pass
|
||||
|
||||
|
||||
@torch.inference_mode
|
||||
def mean_of_trained_tokens(model, eps = 1e-16):
|
||||
"""
|
||||
Llama-3 for eg has untrained vectors in the base model.
|
||||
These include <|eot_id|>, <|start_header_id|>, <|end_header_id|>
|
||||
We reset them to the mean of the rest of the tokens
|
||||
"""
|
||||
embedding_matrix = model.get_input_embeddings ().weight.data.clone()
|
||||
lm_head_matrix = model.get_output_embeddings().weight.data.clone()
|
||||
|
||||
# Get untrained tokens
|
||||
indicator_untrained = torch.amax(embedding_matrix, axis = 1) <= eps
|
||||
where_untrained = torch.where(indicator_untrained)[0]
|
||||
n_untrained = where_untrained.shape[0]
|
||||
n_trained = embedding_matrix.shape[0] - n_untrained
|
||||
if n_untrained != 0:
|
||||
print(
|
||||
f"Unsloth: Not an error, but your model has {n_untrained} untrained tokens.\n"\
|
||||
"We shall set them to the mean of the other trained tokens."
|
||||
)
|
||||
pass
|
||||
|
||||
# First set untrained to all 0s - sometimes it's not! 1e-23 for bfloat16
|
||||
embedding_matrix[where_untrained] = 0
|
||||
lm_head_matrix [where_untrained] = 0
|
||||
|
||||
# Find sum
|
||||
sum_embedding = torch.sum(embedding_matrix, dtype = torch.float32, axis = 0)
|
||||
sum_lm_head = torch.sum(lm_head_matrix, dtype = torch.float32, axis = 0)
|
||||
|
||||
# Find correct average by dividing by sum of trained tokens
|
||||
mean_embedding = (sum_embedding / n_trained).to(embedding_matrix.dtype)
|
||||
mean_lm_head = (sum_lm_head / n_trained).to(lm_head_matrix .dtype)
|
||||
|
||||
return mean_embedding, mean_lm_head
|
||||
pass
|
||||
|
||||
|
||||
@torch.inference_mode
|
||||
def add_new_tokens(
|
||||
model,
|
||||
|
|
@ -547,7 +584,10 @@ def add_new_tokens(
|
|||
pass
|
||||
|
||||
# Get mean of trained tokens
|
||||
mean_embedding, mean_lm_head = fix_untrained_tokens(model)
|
||||
# mean_embedding, mean_lm_head = fix_untrained_tokens(model)
|
||||
|
||||
# Weirdly be careful reserved tokens can pop out
|
||||
mean_embedding, mean_lm_head = mean_of_trained_tokens(model)
|
||||
mean_embedding = mean_embedding.to(torch.float32)
|
||||
mean_lm_head = mean_lm_head .to(torch.float32)
|
||||
|
||||
|
|
@ -595,3 +635,43 @@ def add_new_tokens(
|
|||
|
||||
return
|
||||
pass
|
||||
|
||||
|
||||
from inspect import getsource
|
||||
import trl.trainer.sft_trainer
|
||||
from trl.trainer.sft_trainer import *
|
||||
|
||||
def fix_sft_trainer_tokenizer():
|
||||
"""
|
||||
Fixes double adding BOS tokens like in llama-3
|
||||
"""
|
||||
for function_name, replacer in (
|
||||
("_prepare_non_packed_dataloader", "def tokenize(element):",),
|
||||
# ("_prepare_packed_dataloader", "if dataset_text_field is not None",),
|
||||
):
|
||||
function = getsource(eval(f"trl.trainer.sft_trainer.SFTTrainer.{function_name}"))
|
||||
where = function.find("def")
|
||||
function = function.split("\n")
|
||||
function = "\n".join(x[where:] for x in function)
|
||||
|
||||
check_text = \
|
||||
"\n"\
|
||||
"test_text = dataset[0][dataset_text_field] if (formatting_func is None or not use_formatting_func) else formatting_func(dataset[0])\n"\
|
||||
"chat_template = getattr(tokenizer, 'chat_template', None)\n"\
|
||||
"chat_template = '' if chat_template is None else chat_template\n"\
|
||||
"has_bos_token_already = test_text.startswith(tokenizer.bos_token) or tokenizer.bos_token in chat_template\n"\
|
||||
"add_special_tokens = False if has_bos_token_already else add_special_tokens\n\n"
|
||||
|
||||
check_text = check_text.split("\n")
|
||||
check_text = "\n".join(" "*where + x for x in check_text)
|
||||
|
||||
function = function.replace(replacer, check_text + replacer)
|
||||
exec(function, globals())
|
||||
|
||||
# Replace TRL's SFTTrainer
|
||||
exec(f"trl.trainer.sft_trainer.SFTTrainer.{function_name} = {function_name}", globals())
|
||||
pass
|
||||
pass
|
||||
|
||||
# Fixes double adding BOS tokens like in llama-3
|
||||
fix_sft_trainer_tokenizer()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue