Ollama Chat Templates (#582)
* Update llama.py * offload * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * continued pretraining trainer * Update trainer.py * Update trainer.py * Update trainer.py * Update trainer.py * is_bfloat16_supported * Update __init__.py * Update README.md * Update llama.py * is_bfloat16_supported * Update __init__.py * Mistral v3 * Phi 3 medium * Update chat_templates.py * Update chat_templates.py * Phi-3 * Update save.py * Update README.md Mistral v3 to Mistral v0.3 * Untrained tokens * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update save.py * Update save.py * Update save.py * checkpoint * Update _utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * accelerate * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update tokenizer_utils.py * train_dataloader * Update llama.py * Update llama.py * Update llama.py * use_fast_convert * Update save.py * Update save.py * Update save.py * Update save.py * remove_special_tokens * Ollama * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update llama.py * Update chat_templates.py * Support bfloat16 GGUF * Update save.py * Update llama.py * fast_forward_inference * Update mapper.py * Update loader.py * Update llama.py * Update tokenizer_utils.py * info * edits * Create chat template * Fix tokenizer --------- Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
8d9bd0ea8b
commit
8a9e24ed4f
9 changed files with 1015 additions and 161 deletions
|
|
@ -16,6 +16,12 @@ __all__ = [
|
|||
"get_chat_template",
|
||||
"test_chat_templates",
|
||||
"test_hf_gguf_equivalence",
|
||||
"remove_special_tokens",
|
||||
"standardize_dataset",
|
||||
|
||||
"construct_chat_template",
|
||||
"test_construct_chat_template",
|
||||
"create_ollama_modelfile",
|
||||
]
|
||||
|
||||
from transformers import StoppingCriteria, StoppingCriteriaList
|
||||
|
|
@ -29,6 +35,7 @@ from .models._utils import patch_tokenizer
|
|||
|
||||
CHAT_TEMPLATES = {}
|
||||
|
||||
# =========================================== Unsloth
|
||||
# Unsloth efficient template leverages from Zephyr
|
||||
unsloth_template = \
|
||||
"{{ bos_token }}"\
|
||||
|
|
@ -51,10 +58,24 @@ unsloth_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ '>>> Assistant: ' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
unsloth_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}{{ .System }}
|
||||
{{ end }}{{ if .Prompt }}>>> User: {{ .Prompt }}
|
||||
{{ end }}>>> Assistant: {{ .Response }}{__EOS_TOKEN__}
|
||||
"""
|
||||
PARAMETER stop "{__EOS_TOKEN__}"
|
||||
SYSTEM """You are a helpful assistant to the user"""
|
||||
'''
|
||||
|
||||
unsloth_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["unsloth"] = (unsloth_template, unsloth_eos_token, False,)
|
||||
|
||||
CHAT_TEMPLATES["unsloth"] = (unsloth_template, unsloth_eos_token, False, unsloth_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== Zephyr
|
||||
# Zephyr has no BOS!
|
||||
zephyr_template = \
|
||||
"{% for message in messages %}"\
|
||||
|
|
@ -69,10 +90,26 @@ zephyr_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ '<|assistant|>\n' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
zephyr_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}<|system|>
|
||||
{{ .System }}{__EOS_TOKEN__}
|
||||
{{ end }}{{ if .Prompt }}<|user|>
|
||||
{{ .Prompt }}{__EOS_TOKEN__}
|
||||
{{ end }}<|assistant|>
|
||||
{{ .Response }}{__EOS_TOKEN__}
|
||||
"""
|
||||
PARAMETER stop "{__EOS_TOKEN__}"
|
||||
'''
|
||||
|
||||
zephyr_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["zephyr"] = (zephyr_template, zephyr_eos_token, False,)
|
||||
|
||||
CHAT_TEMPLATES["zephyr"] = (zephyr_template, zephyr_eos_token, False, zephyr_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== ChatML
|
||||
# ChatML has no BOS and not EOS! Rather <|im_start|> and <|im_end|> acts as BOS / EOS.
|
||||
chatml_template = \
|
||||
"{% for message in messages %}"\
|
||||
|
|
@ -87,10 +124,27 @@ chatml_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ '<|im_start|>assistant\n' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
chatml_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}<|im_start|>system
|
||||
{{ .System }}<|im_end|>
|
||||
{{ end }}{{ if .Prompt }}<|im_start|>user
|
||||
{{ .Prompt }}<|im_end|>
|
||||
{{ end }}<|im_start|>assistant
|
||||
{{ .Response }}<|im_end|>
|
||||
"""
|
||||
PARAMETER stop "<|im_start|>"
|
||||
PARAMETER stop "<|im_end|>"
|
||||
'''
|
||||
|
||||
chatml_eos_token = "<|im_end|>"
|
||||
CHAT_TEMPLATES["chatml"] = (chatml_template, chatml_eos_token, True,)
|
||||
|
||||
CHAT_TEMPLATES["chatml"] = (chatml_template, chatml_eos_token, True, chatml_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== Mistral-1
|
||||
# Mistral Instruct doesn't allow system prompts, so we append it to the user message.
|
||||
mistral_template = \
|
||||
"{{ bos_token }}"\
|
||||
|
|
@ -114,10 +168,21 @@ mistral_template = \
|
|||
"{{ raise_exception('Only user and assistant roles are supported!') }}"\
|
||||
"{% endif %}"\
|
||||
"{% endfor %}"
|
||||
pass
|
||||
|
||||
# Ollama from https://www.ollama.com/library/mistral
|
||||
mistral_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """[INST] {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} [/INST]"""
|
||||
PARAMETER stop "{__EOS_TOKEN__}"
|
||||
'''
|
||||
|
||||
mistral_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["mistral"] = (mistral_template, mistral_eos_token, False,)
|
||||
|
||||
CHAT_TEMPLATES["mistral"] = (mistral_template, mistral_eos_token, False, mistral_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== Llama-2
|
||||
# Adds BOS to every convo! And weird <<SYS>> system messages.
|
||||
llama_template = \
|
||||
"{% if messages[0]['role'] == 'system' %}"\
|
||||
|
|
@ -140,10 +205,23 @@ llama_template = \
|
|||
"{{ raise_exception('Only user and assistant roles are supported!') }}"\
|
||||
"{% endif %}"\
|
||||
"{% endfor %}"
|
||||
pass
|
||||
|
||||
# Ollama from https://www.ollama.com/library/llama3
|
||||
llama_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """[INST] <<SYS>>{{ .System }}<</SYS>>
|
||||
|
||||
{{ .Prompt }} [/INST]"""
|
||||
PARAMETER stop "{__EOS_TOKEN__}"
|
||||
'''
|
||||
|
||||
llama_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["llama"] = (llama_template, llama_eos_token, False,)
|
||||
|
||||
CHAT_TEMPLATES["llama"] = (llama_template, llama_eos_token, False, llama_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== Vicuna
|
||||
# https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template
|
||||
vicuna_template = \
|
||||
"{{ bos_token }}"\
|
||||
|
|
@ -166,10 +244,21 @@ vicuna_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ 'ASSISTANT:' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
# Ollama from https://www.ollama.com/library/vicuna
|
||||
vicuna_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}USER: {{ .Prompt }} {{ end }}ASSISTANT: {{ .Response }} {__EOS_TOKEN__}"""
|
||||
PARAMETER stop "{__EOS_TOKEN__}"
|
||||
'''
|
||||
|
||||
vicuna_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["vicuna"] = (vicuna_template, vicuna_eos_token, False,)
|
||||
|
||||
CHAT_TEMPLATES["vicuna"] = (vicuna_template, vicuna_eos_token, False, vicuna_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== Vicuna Old
|
||||
# https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template
|
||||
vicuna_old_template = \
|
||||
"{{ bos_token }}"\
|
||||
|
|
@ -192,10 +281,24 @@ vicuna_old_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ '### Assistant:' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
vicuna_old_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}{{ .System }}
|
||||
{{ end }}{{ if .Prompt }}### Human: {{ .Prompt }}
|
||||
{{ end }}### Assistant: {{ .Response }}{__EOS_TOKEN__}
|
||||
"""
|
||||
PARAMETER stop "{__EOS_TOKEN__}"
|
||||
SYSTEM """A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."""
|
||||
'''
|
||||
|
||||
vicuna_old_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["vicuna_old"] = (vicuna_old_template, vicuna_old_eos_token, False,)
|
||||
|
||||
CHAT_TEMPLATES["vicuna_old"] = (vicuna_old_template, vicuna_old_eos_token, False, vicuna_old_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== Alpaca multi turn
|
||||
# https://github.com/tatsu-lab/stanford_alpaca Changed for multi-turn convos
|
||||
alpaca_template = \
|
||||
"{{ bos_token }}"\
|
||||
|
|
@ -203,7 +306,7 @@ alpaca_template = \
|
|||
"{{ messages[0]['content'] + '\n\n' }}"\
|
||||
"{% set loop_messages = messages[1:] %}"\
|
||||
"{% else %}"\
|
||||
"{{ 'Below are some instructions that describes some tasks. Write responses that appropriately completes each request.\n\n' }}"\
|
||||
"{{ 'Below are some instructions that describe some tasks. Write responses that appropriately complete each request.\n\n' }}"\
|
||||
"{% set loop_messages = messages %}"\
|
||||
"{% endif %}"\
|
||||
"{% for message in loop_messages %}"\
|
||||
|
|
@ -218,42 +321,100 @@ alpaca_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ '### Response:\n' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
alpaca_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}{{ .System }}
|
||||
|
||||
{{ end }}{{ if .Prompt }}### Instruction:
|
||||
{{ .Prompt }}{{ end }}
|
||||
|
||||
### Response:
|
||||
{{ .Response }}{__EOS_TOKEN__}
|
||||
|
||||
"""
|
||||
PARAMETER stop "{__EOS_TOKEN__}"
|
||||
SYSTEM """Below are some instructions that describe some tasks. Write responses that appropriately complete each request."""
|
||||
'''
|
||||
|
||||
alpaca_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token, False,)
|
||||
|
||||
CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token, False, alpaca_ollama,)
|
||||
pass
|
||||
|
||||
# =========================================== Gemma
|
||||
# https://huggingface.co/google/gemma-7b-it
|
||||
# Notice we must use |trim for lstrip and rstrip. <start_of_turn> maps to 106.
|
||||
# <end_of_turn> maps to 107. user and model are normal 1 word tokens.
|
||||
gemma_template = \
|
||||
"{{ bos_token }}"\
|
||||
"{% if messages[0]['role'] == 'system' %}"\
|
||||
"{{'<start_of_turn>user\n' + messages[0]['content'] | trim + ' ' + messages[1]['content'] | trim + '<end_of_turn>\n'}}"\
|
||||
"{% set loop_messages = messages[2:] %}"\
|
||||
"{% endif %}"\
|
||||
"{% for message in messages %}"\
|
||||
"{% if message['role'] == 'user' %}"\
|
||||
"{{'<start_of_turn>user\n' + message['content'] | trim + '<end_of_turn>\n'}}"\
|
||||
"{% elif message['role'] == 'assistant' %}"\
|
||||
"{{'<start_of_turn>model\n' + message['content'] | trim + '<end_of_turn>\n' }}"\
|
||||
"{% else %}"\
|
||||
"{{ '<start_of_turn>system\n' + message['content'] | trim + '<end_of_turn>\n' }}"\
|
||||
"{{ raise_exception('Only user and assistant roles are supported!') }}"\
|
||||
"{% endif %}"\
|
||||
"{% endfor %}"\
|
||||
"{% if add_generation_prompt %}"\
|
||||
"{{ '<start_of_turn>model\n' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
# Ollama from https://www.ollama.com/library/gemma
|
||||
gemma_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """<start_of_turn>user
|
||||
{{ if .System }}{{ .System }} {{ end }}{{ .Prompt }}<end_of_turn>
|
||||
<start_of_turn>model
|
||||
{{ .Response }}<end_of_turn>
|
||||
"""
|
||||
PARAMETER repeat_penalty 1
|
||||
PARAMETER stop "<start_of_turn>"
|
||||
PARAMETER stop "<end_of_turn>"
|
||||
PARAMETER penalize_newline false
|
||||
'''
|
||||
|
||||
gemma_eos_token = "<end_of_turn>"
|
||||
CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token, True,)
|
||||
CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token, True, gemma_ollama,)
|
||||
pass
|
||||
|
||||
|
||||
# Gemma with ChatML instead
|
||||
# =========================================== Gemma with ChatML instead
|
||||
# We find using <eos> is still more appropriate!
|
||||
gemma_chatml_template = "{{ bos_token }}" + chatml_template
|
||||
pass
|
||||
|
||||
gemma_chatml_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}<|im_start|>system
|
||||
{{ .System }}<|im_end|>
|
||||
{{ end }}{{ if .Prompt }}<|im_start|>user
|
||||
{{ .Prompt }}<|im_end|>
|
||||
{{ end }}<|im_start|>assistant
|
||||
{{ .Response }}<|im_end|>
|
||||
"""
|
||||
PARAMETER repeat_penalty 1
|
||||
PARAMETER stop "<|im_start|>"
|
||||
PARAMETER stop "<|im_end|>"
|
||||
PARAMETER penalize_newline false
|
||||
'''
|
||||
|
||||
gemma_chatml_eos_token = (
|
||||
{"<start_of_turn>" : "<|im_start|>", "<eos>" : "<|im_end|>"},
|
||||
"<|im_end|>",
|
||||
)
|
||||
CHAT_TEMPLATES["gemma_chatml"] = (gemma_chatml_template, gemma_chatml_eos_token, True,)
|
||||
CHAT_TEMPLATES["gemma_chatml"] = (gemma_chatml_template, gemma_chatml_eos_token, True, gemma_chatml_ollama,)
|
||||
pass
|
||||
|
||||
|
||||
# Llama-3
|
||||
# =========================================== Llama-3
|
||||
# Weirdly \n\n is needed?
|
||||
llama3_template = \
|
||||
"{{ bos_token }}"\
|
||||
|
|
@ -269,11 +430,30 @@ llama3_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
# Ollama from https://www.ollama.com/library/llama3
|
||||
llama3_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>
|
||||
|
||||
{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
|
||||
|
||||
{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
{{ .Response }}<|eot_id|>"""
|
||||
PARAMETER stop "<|start_header_id|>"
|
||||
PARAMETER stop "<|end_header_id|>"
|
||||
PARAMETER stop "<|eot_id|>"
|
||||
'''
|
||||
|
||||
llama3_template_eos_token = "eos_token"
|
||||
CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token, False,)
|
||||
CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,)
|
||||
pass
|
||||
|
||||
|
||||
# Phi-3
|
||||
# =========================================== Phi-3
|
||||
phi3_template = \
|
||||
"{{ bos_token }}"\
|
||||
"{% for message in messages %}"\
|
||||
|
|
@ -288,8 +468,27 @@ phi3_template = \
|
|||
"{% if add_generation_prompt %}"\
|
||||
"{{ '<|assistant|>\n' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
# Ollama from https://www.ollama.com/library/phi3
|
||||
phi3_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .System }}<|system|>
|
||||
{{ .System }}<|end|>
|
||||
{{ end }}{{ if .Prompt }}<|user|>
|
||||
{{ .Prompt }}<|end|>
|
||||
{{ end }}<|assistant|>
|
||||
{{ .Response }}<|end|>
|
||||
"""
|
||||
PARAMETER stop "<|end|>"
|
||||
PARAMETER stop "<|user|>"
|
||||
PARAMETER stop "<|assistant|>"
|
||||
'''
|
||||
|
||||
phi3_template_eos_token = "<|end|>"
|
||||
CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token, False,)
|
||||
CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token, False, phi3_ollama,)
|
||||
pass
|
||||
|
||||
|
||||
def get_chat_template(
|
||||
|
|
@ -297,6 +496,7 @@ def get_chat_template(
|
|||
chat_template = "chatml",
|
||||
mapping = {"role" : "role", "content" : "content", "user" : "user", "assistant" : "assistant"},
|
||||
map_eos_token = True,
|
||||
system_message = None,
|
||||
):
|
||||
assert(type(map_eos_token) is bool)
|
||||
old_tokenizer = tokenizer
|
||||
|
|
@ -331,7 +531,7 @@ def get_chat_template(
|
|||
|
||||
elif type(chat_template) is str:
|
||||
|
||||
chat_template, stop_word, yes_map_eos_token = CHAT_TEMPLATES[chat_template]
|
||||
chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template]
|
||||
|
||||
# Check mapping to eos_token
|
||||
if not map_eos_token and yes_map_eos_token: map_eos_token = True
|
||||
|
|
@ -496,10 +696,421 @@ def get_chat_template(
|
|||
# Patch saving functions
|
||||
tokenizer = patch_saving_functions(tokenizer)
|
||||
|
||||
# Add Ollama
|
||||
tokenizer._ollama_modelfile = ollama_modelfile
|
||||
tokenizer._system_message = system_message
|
||||
return tokenizer#, stopping_criteria
|
||||
pass
|
||||
|
||||
|
||||
def remove_special_tokens(tokenizer, prompt):
|
||||
# Removes double BOS token
|
||||
if prompt.startswith(tokenizer.bos_token):
|
||||
prompt = prompt[len(tokenizer.bos_token):]
|
||||
pass
|
||||
return prompt
|
||||
pass
|
||||
|
||||
|
||||
def standardize_dataset(
|
||||
dataset,
|
||||
conversation_key = "conversations",
|
||||
system_message = None,
|
||||
aliases_for_system = ["system",],
|
||||
aliases_for_user = ["user", "human", "input",],
|
||||
aliases_for_assistant = ["gpt", "assistant", "output",],
|
||||
):
|
||||
"""
|
||||
Standardizes ShareGPT and other formats to user/assistant Hugging Face format.
|
||||
"""
|
||||
import collections
|
||||
import itertools
|
||||
|
||||
convos = dataset[:10][conversation_key]
|
||||
uniques = collections.defaultdict(list)
|
||||
for convo in convos:
|
||||
for message in convo:
|
||||
for key, value in message.items():
|
||||
uniques[key].append(value)
|
||||
pass
|
||||
|
||||
# Must be only 2 entries
|
||||
assert(len(uniques.keys()) == 2)
|
||||
|
||||
keys = list(uniques.keys())
|
||||
length_first = len(set(uniques[keys[0]]))
|
||||
length_second = len(set(uniques[keys[1]]))
|
||||
|
||||
if length_first < length_second:
|
||||
# Role is assigned to the first element
|
||||
role_key = keys[0]
|
||||
content_key = keys[1]
|
||||
else:
|
||||
role_key = keys[1]
|
||||
content_key = keys[0]
|
||||
pass
|
||||
|
||||
# Check roles are in aliases
|
||||
all_aliases = set(aliases_for_system + aliases_for_user + aliases_for_assistant)
|
||||
roles = set(uniques[role_key])
|
||||
leftover_aliases = (all_aliases | roles) - all_aliases
|
||||
if len(leftover_aliases) != 0:
|
||||
raise TypeError(
|
||||
f"Unsloth: {list(leftover_aliases)} are not in aliases. Please update aliases."
|
||||
)
|
||||
pass
|
||||
|
||||
# Mapping for aliases
|
||||
aliases_mapping = {}
|
||||
for x in aliases_for_system: aliases_mapping[x] = "system"
|
||||
for x in aliases_for_user: aliases_mapping[x] = "user"
|
||||
for x in aliases_for_assistant: aliases_mapping[x] = "assistant"
|
||||
|
||||
def _standardize_dataset(examples):
|
||||
convos = examples[conversation_key]
|
||||
all_convos = []
|
||||
for convo in convos:
|
||||
new_convo = []
|
||||
if len(convo) == 0: continue
|
||||
has_system = aliases_mapping[convo[0][role_key]] == "system"
|
||||
if not has_system and system_message is not None:
|
||||
new_convo.append({ "role" : "system", "content" : system_message, })
|
||||
for message in convo:
|
||||
role = aliases_mapping[message[role_key]]
|
||||
new_convo.append({ "role" : role, "content" : message[content_key], })
|
||||
pass
|
||||
all_convos.append(new_convo)
|
||||
pass
|
||||
return { conversation_key : all_convos, }
|
||||
pass
|
||||
|
||||
return dataset.map(_standardize_dataset, batched = True,)
|
||||
pass
|
||||
|
||||
|
||||
def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):
|
||||
added_tokens_decoder = tokenizer.added_tokens_decoder.values()
|
||||
added_tokens_decoder = [str(x) for x in added_tokens_decoder]
|
||||
|
||||
# Remove added_tokens_decoder duplicates
|
||||
added_tokens_decoder = list(set(added_tokens_decoder) - set(extra_eos_tokens))
|
||||
|
||||
# Remove BOS
|
||||
if getattr(tokenizer, "bos_token", None) is not None:
|
||||
added_tokens_decoder = [x for x in added_tokens_decoder if x != tokenizer.bos_token]
|
||||
pass
|
||||
|
||||
repeatted_tokens = []
|
||||
# Join all vocab
|
||||
joined_text = "\x01\x00".join(added_tokens_decoder)
|
||||
for token in added_tokens_decoder:
|
||||
n = len(token)
|
||||
repeatted_counts = joined_text.count(token[:n//2])
|
||||
# Try finding longer than 1/2 of the token in the rest
|
||||
# For eg <|reserved_special_token_0|>, <|reserved_special_token_1|>
|
||||
if repeatted_counts > 2:
|
||||
for j in range(n//2+1, n):
|
||||
if joined_text.count(token[:j]) < repeatted_counts:
|
||||
j -= 1
|
||||
# Remove repeatted tokens to reduce search space
|
||||
joined_text = joined_text.replace(token[:j], "")
|
||||
repeatted_tokens.append(token[:j])
|
||||
break
|
||||
pass
|
||||
pass
|
||||
pass
|
||||
|
||||
# Remove duplicates
|
||||
splitted = joined_text.split("\x01\x00")
|
||||
final_eos_tokens = []
|
||||
for old, new in zip(added_tokens_decoder, splitted):
|
||||
if old == new: final_eos_tokens.append(old)
|
||||
pass
|
||||
final_eos_tokens += extra_eos_tokens
|
||||
final_eos_tokens += repeatted_tokens
|
||||
return final_eos_tokens
|
||||
pass
|
||||
|
||||
|
||||
def construct_chat_template( \
|
||||
|
||||
tokenizer = None,
|
||||
|
||||
template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
|
||||
|
||||
{SYSTEM}<|eot_id|><|start_header_id|>user<|end_header_id|>
|
||||
|
||||
{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
{OUTPUT}<|eot_id|><|start_header_id|>user<|end_header_id|>
|
||||
|
||||
{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
{OUTPUT}<|eot_id|>""",
|
||||
|
||||
default_system_message = \
|
||||
"Below are some instructions that describe some tasks. Write responses that appropriately complete each request.",
|
||||
|
||||
extra_eos_tokens = None,
|
||||
|
||||
):
|
||||
"""
|
||||
Creates a Ollama modelfile and a HF Jinja template from a custom
|
||||
template. You must provide 2x examples of an input & output.
|
||||
There is an optional system message as well.
|
||||
|
||||
You must use {INPUT}, {OUTPUT} twice, and {SYSTEM} is optional.
|
||||
"""
|
||||
assert(tokenizer is not None)
|
||||
|
||||
if extra_eos_tokens is None: extra_eos_tokens = []
|
||||
|
||||
vocab = tokenizer.get_vocab()
|
||||
for extra_eos in extra_eos_tokens:
|
||||
assert(type(extra_eos) is str)
|
||||
if extra_eos not in vocab:
|
||||
raise ValueError(f"Unsloth: `{extra_eos}` is not a singular token in the tokenizer.")
|
||||
pass
|
||||
pass
|
||||
|
||||
error_msg = \
|
||||
"Unsloth: Your prompt template must have 2 examples showing the user input {INPUT} "\
|
||||
"and the assistant output {OUTPUT}\n\n"\
|
||||
"For example what is not allowed is just:\n"\
|
||||
"### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n\n\n"\
|
||||
"What is required is 2x of this:\n"\
|
||||
"### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n"\
|
||||
"### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n"
|
||||
|
||||
# O(N^2) search finding 2 repeatted pieces of text
|
||||
j = len(template)-1
|
||||
at_least_one = False
|
||||
while j > 0:
|
||||
found = template.rfind(template[j:], 0, j)
|
||||
if found == -1: break
|
||||
j -= 1
|
||||
at_least_one = True
|
||||
pass
|
||||
if j > 0: j += 1
|
||||
else: raise RuntimeError(error_msg)
|
||||
|
||||
|
||||
if not at_least_one: raise RuntimeError(error_msg)
|
||||
|
||||
# Repeatted text
|
||||
instruction_response = template[j:]
|
||||
if instruction_response.count("{INPUT}") != 1 or instruction_response.count("{OUTPUT}") != 1:
|
||||
raise RuntimeError(error_msg)
|
||||
pass
|
||||
|
||||
# 1st System, Instruction, Output pair
|
||||
left = template[:j]
|
||||
# 2nd Instruction, Output pair
|
||||
right = template[j:]
|
||||
|
||||
# Isolate input
|
||||
extra_eos_tokens_regex = "|".join(f"(?:{re.escape(x)})" for x in extra_eos_tokens)
|
||||
if len(extra_eos_tokens_regex) != 0:
|
||||
find_end = f"(?:{extra_eos_tokens_regex})?"
|
||||
else:
|
||||
find_end = ""
|
||||
find_end = r"\{INPUT\}[\s\n]{0,}" + find_end
|
||||
input_end = list(re.finditer(find_end, right))
|
||||
assert(len(input_end) == 1)
|
||||
input_end = input_end[0]
|
||||
input_end = input_end.span(0)[1]
|
||||
input_part = right[:input_end]
|
||||
|
||||
# Isolate output
|
||||
output_part = right[input_end:]
|
||||
|
||||
# Isolate system
|
||||
system_part = left[:left.find(input_part)]
|
||||
|
||||
# Check if the user provided a correct prompt
|
||||
combined = system_part + input_part + output_part
|
||||
if combined != left:
|
||||
combined_changed = combined.replace('\n', '\\n')
|
||||
left_changed = left .replace('\n', '\\n')
|
||||
raise RuntimeError(
|
||||
"Unsloth: The prompt template you provided isn't correct. You gave:\n"\
|
||||
f"{combined_changed}\n\n"\
|
||||
"But we require the following:\n"\
|
||||
f"{left_changed}"
|
||||
)
|
||||
pass
|
||||
|
||||
# Ollama modelfile parts
|
||||
|
||||
# Check bos_token is in system prompt
|
||||
ollama_system = system_part
|
||||
has_bos_token = False
|
||||
if tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None):
|
||||
if ollama_system.startswith(tokenizer.bos_token):
|
||||
has_bos_token = True
|
||||
ollama_system = ollama_system[len(tokenizer.bos_token):]
|
||||
pass
|
||||
pass
|
||||
system_modelfile = "{{ if .System }}" + ollama_system.replace("{SYSTEM}", "{{ .System }}") + "{{ end }}"
|
||||
input_modelfile = "{{ if .Prompt }}" + input_part .replace("{INPUT}", "{{ .Prompt }}") + "{{ end }}"
|
||||
output_modelfile = output_part.replace("{OUTPUT}", "{{ .Response }}")
|
||||
|
||||
# Check if EOS token is at the end of the output
|
||||
if not output_modelfile.endswith(tuple(extra_eos_tokens)):
|
||||
output_modelfile += "{__EOS_TOKEN__}"
|
||||
pass
|
||||
|
||||
# Ollama EOS
|
||||
ollama_eos = get_ollama_eos_tokens(tokenizer, extra_eos_tokens)
|
||||
ollama_eos = '\n'.join(f'PARAMETER stop "{eos}"' for eos in ollama_eos)
|
||||
|
||||
# Ollama modelfile
|
||||
modelfile = 'FROM {__FILE_LOCATION__}\n\n'\
|
||||
'TEMPLATE """' + system_modelfile + input_modelfile + output_modelfile + \
|
||||
'"""\n\n' + ollama_eos
|
||||
|
||||
# HF Jinja Chat template
|
||||
def process(part, which, content = "message['content']"):
|
||||
if part.endswith(which):
|
||||
part = "'" + part[:part.find(which)] + f"' + {content}"
|
||||
elif part.startswith(which):
|
||||
part = f"{content} + '" + part[part.find(which):] + "'"
|
||||
else:
|
||||
part = "'" + part.replace(which, f"' + {content} + '") + "'"
|
||||
if part.startswith("'' + "): part = part[5:]
|
||||
return part
|
||||
pass
|
||||
input_jinja = process(input_part, "{INPUT}")
|
||||
output_jinja = process(output_part, "{OUTPUT}")
|
||||
pass
|
||||
|
||||
jinja_template = \
|
||||
"{% for message in loop_messages %}"\
|
||||
"{% if message['role'] == 'user' %}"\
|
||||
"{{ " + input_jinja + " }}"\
|
||||
"{% elif message['role'] == 'assistant' %}"\
|
||||
"{{ " + output_jinja + " }}"\
|
||||
"{% else %}"\
|
||||
"{{ raise_exception('Only user and assistant roles are supported!') }}"\
|
||||
"{% endif %}"\
|
||||
"{% endfor %}"\
|
||||
"{% if add_generation_prompt %}"\
|
||||
"{{ '" + output_part[:output_part.find("{OUTPUT}")] + "' }}"\
|
||||
"{% endif %}"
|
||||
pass
|
||||
|
||||
# Now add system prompt to jinja
|
||||
if len(system_part) != 0:
|
||||
partial_system = process(system_part, "{SYSTEM}", "messages[0]['content']")
|
||||
partial_system = partial_system.replace("{SYSTEM}", "")
|
||||
|
||||
# Separate the BOS
|
||||
if has_bos_token:
|
||||
partial_system = partial_system.replace(tokenizer.bos_token, "", 1)
|
||||
pass
|
||||
|
||||
partial_system = \
|
||||
"{% if messages[0]['role'] == 'system' %}"\
|
||||
"{{ " + partial_system + " }}"\
|
||||
"{% set loop_messages = messages[1:] %}"
|
||||
if default_system_message is not None:
|
||||
partial_system += "{% else %}"\
|
||||
"{{ '" + system_part.replace("{SYSTEM}", default_system_message) + "' }}"\
|
||||
"{% set loop_messages = messages %}"\
|
||||
"{% endif %}"
|
||||
else:
|
||||
partial_system += "{% endif %}"
|
||||
pass
|
||||
|
||||
jinja_template = partial_system + jinja_template
|
||||
|
||||
if has_bos_token:
|
||||
jinja_template = "{{ bos_token }}" + jinja_template
|
||||
pass
|
||||
|
||||
return modelfile, jinja_template
|
||||
pass
|
||||
|
||||
|
||||
def test_construct_chat_template():
|
||||
token = "hf_"
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct", token = token)
|
||||
|
||||
template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
|
||||
|
||||
{SYSTEM}<|eot_id|><|start_header_id|>user<|end_header_id|>
|
||||
|
||||
{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
{OUTPUT}<|eot_id|><|start_header_id|>user<|end_header_id|>
|
||||
|
||||
{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
{OUTPUT}<|eot_id|>"""
|
||||
|
||||
default_system_message = \
|
||||
"Below are some instructions that describe some tasks. Write responses that appropriately complete each request."
|
||||
|
||||
extra_eos_tokens = None
|
||||
|
||||
modelfile, jinja_template = construct_chat_template(template, default_system_message, extra_eos_tokens)
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are an assistant"},
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{"role": "assistant", "content": "It's 4."},
|
||||
{"role": "user", "content": "Ok!"},
|
||||
{"role": "assistant", "content": "Anything else?"},
|
||||
{"role": "user", "content": "What's 2x2?"},
|
||||
]
|
||||
correct_output = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
|
||||
|
||||
tokenizer.chat_template = jinja_template
|
||||
new_output = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
|
||||
|
||||
assert(correct_output == new_output)
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
def create_ollama_modelfile(tokenizer, gguf_location):
|
||||
"""
|
||||
Creates an Ollama Modelfile.
|
||||
Use ollama.create(model = "new_ollama_model", modelfile = modelfile)
|
||||
"""
|
||||
modelfile = getattr(tokenizer, "_ollama_modelfile", None)
|
||||
if modelfile is None:
|
||||
raise RuntimeError(
|
||||
"Unsloth: Tokenizer does not have a `ollama_modelfile` attribute.\n"\
|
||||
"Please use get_chat_template(...)."
|
||||
)
|
||||
pass
|
||||
|
||||
system_message = getattr(tokenizer, "_system_message", None)
|
||||
if system_message is None:
|
||||
__SYSTEM_MESSAGE__ = ""
|
||||
else:
|
||||
__SYSTEM_MESSAGE__ = f'SYSTEM """{system_message}"""'
|
||||
pass
|
||||
|
||||
modelfile = modelfile\
|
||||
.replace("{{", "⚫@✅#🦥")\
|
||||
.replace("}}", "⚡@🦥#⛵")\
|
||||
.format(
|
||||
__FILE_LOCATION__ = gguf_location,
|
||||
__SYSTEM_MESSAGE__ = __SYSTEM_MESSAGE__,
|
||||
__EOS_TOKEN__ = tokenizer.eos_token,
|
||||
)\
|
||||
.replace("⚫@✅#🦥", "{{")\
|
||||
.replace("⚡@🦥#⛵", "}}")\
|
||||
.rstrip()
|
||||
pass
|
||||
|
||||
return modelfile
|
||||
pass
|
||||
|
||||
|
||||
def create_stopping_criteria(tokenizer, stop_word = "eos_token"):
|
||||
class StoppingCriteriaSub(StoppingCriteria):
|
||||
__slots__ = "stop_token", "single_match", "length",
|
||||
|
|
@ -670,7 +1281,8 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf")
|
|||
if tokenizer.chat_template is not None:
|
||||
prompt = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
|
||||
prompt = prompt.replace("'", "") # Subprocess does not like ''
|
||||
prompts.append(prompts)
|
||||
prompt = remove_special_tokens(tokenizer, prompt)
|
||||
prompts.append(prompt)
|
||||
pass
|
||||
|
||||
for prompt in prompts:
|
||||
|
|
@ -688,9 +1300,9 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf")
|
|||
gguf_tokenized = re.findall("([\d]{1,}) \-\> \'([^\']{1,})\'", gguf_tokens, flags = re.MULTILINE)
|
||||
gguf_tokenized = [(int(x[0]), x[1],) for x in gguf_tokenized]
|
||||
input_ids = tokenizer(prompt).input_ids
|
||||
|
||||
tokens = tokenizer.batch_decode(input_ids)
|
||||
hf_tokenized = list(zip(input_ids, tokens))
|
||||
print(gguf_tokenized[:5])
|
||||
|
||||
# Compare to Huggingface
|
||||
for j, (hf_token, gguf_token) in enumerate(zip(hf_tokenized, gguf_tokenized)):
|
||||
|
|
@ -698,9 +1310,10 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf")
|
|||
print("Failed GGUF != HF at", j)
|
||||
print("HF =", hf_token)
|
||||
print("GGUF =", gguf_token)
|
||||
print(hf_tokenized[:j+1])
|
||||
print(gguf_tokenized[:j+1])
|
||||
print(gguf_tokens)
|
||||
print(hf_tokenized)
|
||||
print()
|
||||
print(gguf_tokenized)
|
||||
print()
|
||||
raise RuntimeError("Failed comparing GGUF to HF.")
|
||||
pass
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ import numpy as np
|
|||
import os
|
||||
import psutil
|
||||
|
||||
__version__ = "2024.5"
|
||||
__version__ = "2024.6"
|
||||
|
||||
# Get Flash Attention v2 if Ampere (RTX 30xx, A100)
|
||||
major_version, minor_version = torch.cuda.get_device_capability()
|
||||
|
|
|
|||
|
|
@ -209,8 +209,9 @@ def LlamaAttention_fast_forward_inference(
|
|||
|
||||
# Attention
|
||||
if bsz == 1:
|
||||
Qn *= self.scalar # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963
|
||||
# It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows
|
||||
A = torch.matmul(Qn, Knn.transpose(2, 3), out = self.attention[:,:,:,:cached_len])
|
||||
A *= self.scalar
|
||||
# if attention_mask is not None: A += attention_mask # Must add attention_mask for batched
|
||||
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)#.to(A.dtype)
|
||||
A = torch.matmul(A, Vnn, out = Qn)
|
||||
|
|
@ -791,7 +792,7 @@ def CausalLM_fast_forward(fast_forward_inference):
|
|||
*args, **kwargs,
|
||||
) -> Union[Tuple, CausalLMOutputWithPast]:
|
||||
|
||||
if past_key_values is not None and self.config.model_type != "qwen2":
|
||||
if past_key_values is not None:
|
||||
outputs = fast_forward_inference(
|
||||
self,
|
||||
input_ids,
|
||||
|
|
@ -1195,7 +1196,13 @@ class FastLlamaModel:
|
|||
f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\
|
||||
f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}'
|
||||
logger.warning(debug_info)
|
||||
import gc
|
||||
import subprocess, re, gc
|
||||
output = subprocess.check_output(
|
||||
'nvidia-smi --query-gpu=memory.used --format=csv', shell = True)
|
||||
output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output)
|
||||
output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output)
|
||||
if output > 1: raise RuntimeError(
|
||||
'Error: More than 1 GPUs have a lot of VRAM usage. Please obtain a commercial license.')
|
||||
for _ in range(3):
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()"""
|
||||
|
|
@ -1206,12 +1213,12 @@ class FastLlamaModel:
|
|||
|
||||
debug_info = """n_total_devices = total_train_batch_size // \\
|
||||
args.gradient_accumulation_steps // self._train_batch_size
|
||||
if n_total_devices > 2:
|
||||
if n_total_devices > 1:
|
||||
logger.warning_once(
|
||||
"Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||
"The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n"
|
||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
"* Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||
"* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n"
|
||||
"* We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||
"* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
)
|
||||
debug_info ="""
|
||||
debug_info = debug_info.split('\n')
|
||||
|
|
@ -1236,17 +1243,17 @@ class FastLlamaModel:
|
|||
bsz = self._train_batch_size
|
||||
total_batches = bsz * ga * args.world_size
|
||||
n_total_devices = total_batches // ga // bsz
|
||||
if n_total_devices > 2:
|
||||
if n_total_devices > 1:
|
||||
logger.warning_once(
|
||||
"Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||
"The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n"
|
||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
"* Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||
"* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n"
|
||||
"* We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||
"* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
)
|
||||
divisor = n_total_devices / 2
|
||||
divisor = n_total_devices / 1
|
||||
bsz = self._train_batch_size = max(int(bsz / divisor), 1)
|
||||
if total_batches // ga // bsz > 2:
|
||||
divisor = n_total_devices / 2
|
||||
if total_batches // ga // bsz > 1:
|
||||
divisor = n_total_devices / 1
|
||||
ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)"""
|
||||
check_batches = check_batches.split('\n')
|
||||
check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]])
|
||||
|
|
@ -1830,10 +1837,10 @@ class FastLlamaModel:
|
|||
|
||||
@staticmethod
|
||||
def for_inference(model):
|
||||
if model.config.model_type == "qwen2":
|
||||
FastLlamaModel.for_training(model)
|
||||
return
|
||||
pass
|
||||
# if model.config.model_type == "qwen2":
|
||||
# FastLlamaModel.for_training(model)
|
||||
# return
|
||||
# pass
|
||||
|
||||
internal_model = model
|
||||
internal_model.gradient_checkpointing = False
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ del major, minor
|
|||
|
||||
def _get_model_name(model_name, load_in_4bit = True):
|
||||
|
||||
# First try replacing lowercase 'b' with uppercase 'B'
|
||||
model_name = model_name.lower()
|
||||
|
||||
if not SUPPORTS_FOURBIT and model_name in INT_TO_FLOAT_MAPPER:
|
||||
model_name = INT_TO_FLOAT_MAPPER[model_name]
|
||||
logger.warning_once(
|
||||
|
|
|
|||
|
|
@ -197,4 +197,12 @@ for key, values in __INT_TO_FLOAT_MAPPER.items():
|
|||
for value in values:
|
||||
FLOAT_TO_INT_MAPPER[value] = key
|
||||
pass
|
||||
|
||||
# Get lowercased
|
||||
lowered_key = key.lower()
|
||||
INT_TO_FLOAT_MAPPER[lowered_key] = values[0].lower()
|
||||
|
||||
for value in values:
|
||||
FLOAT_TO_INT_MAPPER[value.lower()] = lowered_key
|
||||
pass
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -393,21 +393,6 @@ class FastMistralModel(FastLlamaModel):
|
|||
layer.self_attn.apply_o = original_apply_o
|
||||
pass
|
||||
|
||||
# Patch Trainer
|
||||
from transformers.trainer import Trainer
|
||||
if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
|
||||
try:
|
||||
inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
|
||||
except:
|
||||
raise RuntimeError(
|
||||
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
)
|
||||
pass
|
||||
pass
|
||||
|
||||
# Patch Trainer
|
||||
from transformers.trainer import Trainer
|
||||
try:
|
||||
|
|
@ -419,7 +404,7 @@ class FastMistralModel(FastLlamaModel):
|
|||
except:
|
||||
raise RuntimeError(
|
||||
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
||||
"The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\n"
|
||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
)
|
||||
|
|
@ -447,7 +432,17 @@ class FastMistralModel(FastLlamaModel):
|
|||
f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\
|
||||
f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\
|
||||
f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}'
|
||||
logger.warning_once(debug_info)"""
|
||||
logger.warning(debug_info)
|
||||
import subprocess, re, gc
|
||||
output = subprocess.check_output(
|
||||
'nvidia-smi --query-gpu=memory.used --format=csv', shell = True)
|
||||
output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output)
|
||||
output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output)
|
||||
if output > 1: raise RuntimeError(
|
||||
'Error: More than 1 GPUs have a lot of VRAM usage. Please obtain a commercial license.')
|
||||
for _ in range(3):
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()"""
|
||||
|
||||
debug_info = debug_info.split('\n')
|
||||
debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]])
|
||||
|
|
@ -455,12 +450,12 @@ class FastMistralModel(FastLlamaModel):
|
|||
|
||||
debug_info = """n_total_devices = total_train_batch_size // \\
|
||||
args.gradient_accumulation_steps // self._train_batch_size
|
||||
if n_total_devices > 2:
|
||||
if n_total_devices > 1:
|
||||
logger.warning_once(
|
||||
"Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n"
|
||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
"* Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||
"* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n"
|
||||
"* We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||
"* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
)
|
||||
debug_info ="""
|
||||
debug_info = debug_info.split('\n')
|
||||
|
|
@ -485,16 +480,17 @@ class FastMistralModel(FastLlamaModel):
|
|||
bsz = self._train_batch_size
|
||||
total_batches = bsz * ga * args.world_size
|
||||
n_total_devices = total_batches // ga // bsz
|
||||
if n_total_devices > 2:
|
||||
if n_total_devices > 1:
|
||||
logger.warning_once(
|
||||
"Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n"
|
||||
"The OSS currently works on 4 GPUs - we're a 2 person team, so please help fund\\n"
|
||||
"our development costs by supporting us through Ko-fi or buying a license! Thanks!",
|
||||
"* Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||
"* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n"
|
||||
"* We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||
"* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||
)
|
||||
divisor = n_total_devices / 2
|
||||
divisor = n_total_devices / 1
|
||||
bsz = self._train_batch_size = max(int(bsz / divisor), 1)
|
||||
if total_batches // ga // bsz > 2:
|
||||
divisor = n_total_devices / 2
|
||||
if total_batches // ga // bsz > 1:
|
||||
divisor = n_total_devices / 1
|
||||
ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)"""
|
||||
check_batches = check_batches.split('\n')
|
||||
check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]])
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@
|
|||
# limitations under the License.
|
||||
|
||||
from .llama import *
|
||||
from .mistral import FastMistralModel
|
||||
import os
|
||||
from ._utils import __version__
|
||||
|
||||
|
|
@ -60,7 +59,7 @@ class FastQwen2Model(FastLlamaModel):
|
|||
|
||||
@staticmethod
|
||||
def from_pretrained(
|
||||
model_name = "Qwen/Qwen1.5-7B",
|
||||
model_name = "Qwen/Qwen2-7B",
|
||||
max_seq_length = 4096,
|
||||
dtype = None,
|
||||
load_in_4bit = True,
|
||||
|
|
@ -73,7 +72,7 @@ class FastQwen2Model(FastLlamaModel):
|
|||
trust_remote_code = False,
|
||||
**kwargs,
|
||||
):
|
||||
return FastMistralModel.from_pretrained(
|
||||
return FastLlamaModel.from_pretrained(
|
||||
model_name = model_name,
|
||||
max_seq_length = max_seq_length,
|
||||
dtype = dtype,
|
||||
|
|
|
|||
195
unsloth/save.py
195
unsloth/save.py
|
|
@ -59,7 +59,8 @@ ALLOWED_QUANTS = \
|
|||
"fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.",
|
||||
"quantized" : "Recommended. Slow conversion. Fast inference, small files.",
|
||||
"f32" : "Not recommended. Retains 100% accuracy, but super slow and memory hungry.",
|
||||
"f16" : "Fastest conversion + retains 100% accuracy. Slow and memory hungry.",
|
||||
"bf16" : "Bfloat16 - Fastest conversion + retains 100% accuracy. Slow and memory hungry.",
|
||||
"f16" : "Float16 - Fastest conversion + retains 100% accuracy. Slow and memory hungry.",
|
||||
"q8_0" : "Fast conversion. High resource use, but generally acceptable.",
|
||||
"q4_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K",
|
||||
"q5_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K",
|
||||
|
|
@ -102,7 +103,7 @@ def check_if_sentencepiece_model(model, temporary_location = "_unsloth_sentencep
|
|||
if os.path.isfile(f"{file_location}/tokenizer.model"):
|
||||
sentencepiece_model = True
|
||||
pass
|
||||
shutil.rmtree(file_location)
|
||||
shutil.rmtree(file_location, ignore_errors = True)
|
||||
return sentencepiece_model
|
||||
pass
|
||||
|
||||
|
|
@ -700,7 +701,7 @@ def unsloth_save_model(
|
|||
|
||||
# Remove temporary location
|
||||
import shutil
|
||||
shutil.rmtree(temporary_location)
|
||||
shutil.rmtree(temporary_location, ignore_errors = True)
|
||||
|
||||
for _ in range(3):
|
||||
torch.cuda.empty_cache()
|
||||
|
|
@ -763,7 +764,7 @@ def install_llama_cpp_old(version = -10):
|
|||
print(f"**[WARNING]** Deleting llama.cpp directory... {10-i} seconds left.")
|
||||
time.sleep(1)
|
||||
import shutil
|
||||
shutil.rmtree("llama.cpp")
|
||||
shutil.rmtree("llama.cpp", ignore_errors = True)
|
||||
pass
|
||||
|
||||
# Clone a specific commit
|
||||
|
|
@ -866,10 +867,11 @@ pass
|
|||
|
||||
def save_to_gguf(
|
||||
model_type : str,
|
||||
model_dtype : str,
|
||||
is_sentencepiece : bool = False,
|
||||
model_directory : str = "unsloth_finetuned_model",
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
first_conversion : str = None,
|
||||
_run_installer = None, # Non blocking install of llama.cpp
|
||||
):
|
||||
# logger.warning(
|
||||
|
|
@ -877,6 +879,22 @@ def save_to_gguf(
|
|||
# "undergoing some major bug fixes as at 5th of May 2024. This is not an Unsloth issue.\n"\
|
||||
# "Please be patient - GGUF saving should still work, but might not work as well."
|
||||
# )
|
||||
assert(model_dtype == "float16" or model_dtype == "bfloat16")
|
||||
model_dtype = "f16" if model_dtype == "float16" else "bf16"
|
||||
|
||||
# Check if bfloat16 is supported
|
||||
if model_dtype == "bf16" and not torch.cuda.is_bf16_supported():
|
||||
logger.warning(
|
||||
"Unsloth: Cannot convert to bf16 GGUF since your computer doesn't support it.\n"\
|
||||
"We shall switch instead to f16."
|
||||
)
|
||||
model_dtype = "f16"
|
||||
pass
|
||||
|
||||
# Check first_conversion as well
|
||||
if first_conversion is None:
|
||||
first_conversion = model_dtype
|
||||
pass
|
||||
|
||||
if quantization_method.startswith("iq2"):
|
||||
raise RuntimeError("Unsloth: Currently iq2 type quantizations aren't supported yet - sorry!")
|
||||
|
|
@ -889,7 +907,7 @@ def save_to_gguf(
|
|||
pass
|
||||
logger.warning_once(f"Unsloth: Converting {model_type} model. Can use fast conversion = {use_fast_convert}.")
|
||||
|
||||
if quantization_method == "not_quantized": quantization_method = "f16"
|
||||
if quantization_method == "not_quantized": quantization_method = model_dtype
|
||||
elif quantization_method == "fast_quantized": quantization_method = "q8_0"
|
||||
elif quantization_method == "quantized": quantization_method = "q4_k_m"
|
||||
elif quantization_method is None: quantization_method = "q8_0"
|
||||
|
|
@ -911,12 +929,13 @@ def save_to_gguf(
|
|||
print(print_info)
|
||||
|
||||
# Check first_conversion format
|
||||
if first_conversion == "f16" : pass
|
||||
elif first_conversion == "f32" : pass
|
||||
elif first_conversion == "q8_0": pass
|
||||
if first_conversion == "f16" : pass
|
||||
if first_conversion == "bf16" : pass
|
||||
elif first_conversion == "f32" : pass
|
||||
elif first_conversion == "q8_0" : pass
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: `first_conversion` can only be one of ['f16', 'f32', 'q8_0'] and not `{first_conversion}`."
|
||||
f"Unsloth: `first_conversion` can only be one of ['f16', 'bf16', 'f32', 'q8_0'] and not `{first_conversion}`."
|
||||
)
|
||||
pass
|
||||
|
||||
|
|
@ -935,11 +954,13 @@ def save_to_gguf(
|
|||
|
||||
if quantization_method == "f32": first_conversion = "f32"
|
||||
elif quantization_method == "f16": first_conversion = "f16"
|
||||
elif quantization_method == "bf16": first_conversion = "bf16"
|
||||
elif quantization_method == "q8_0": first_conversion = "q8_0"
|
||||
else:
|
||||
# Quantized models must have f16 as the default argument
|
||||
if first_conversion == "f32" : pass
|
||||
elif first_conversion == "f16" : pass
|
||||
if first_conversion == "f32" : pass
|
||||
elif first_conversion == "f16" : pass
|
||||
elif first_conversion == "bf16" : pass
|
||||
elif first_conversion == "q8_0":
|
||||
logger.warning_once(
|
||||
"Unsloth: Using q8_0 for the `first_conversion` will lose a bit of accuracy, "\
|
||||
|
|
@ -950,8 +971,22 @@ def save_to_gguf(
|
|||
pass
|
||||
|
||||
# Non llama/mistral needs can only use f32 or f16
|
||||
if not use_fast_convert and (first_conversion != "f16" or first_conversion != "f32"):
|
||||
logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.")
|
||||
if not use_fast_convert and \
|
||||
(first_conversion != "f16" or first_conversion != "bf16" or first_conversion != "f32"):
|
||||
|
||||
pass
|
||||
# Latest llama.cpp works for all models for q8_0!
|
||||
|
||||
# logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.")
|
||||
# first_conversion = "f16"
|
||||
pass
|
||||
|
||||
# Check if bfloat16 is supported
|
||||
if first_conversion == "bf16" and not torch.cuda.is_bf16_supported():
|
||||
logger.warning(
|
||||
"Unsloth: Cannot convert to bf16 GGUF since your computer doesn't support it.\n"\
|
||||
"We shall switch instead to f16."
|
||||
)
|
||||
first_conversion = "f16"
|
||||
pass
|
||||
|
||||
|
|
@ -975,6 +1010,7 @@ def save_to_gguf(
|
|||
vocab_type = "bpe"
|
||||
pass
|
||||
|
||||
# convert.py is deprecated!
|
||||
use_fast_convert = False
|
||||
if use_fast_convert:
|
||||
command = f"python llama.cpp/convert.py {model_directory} "\
|
||||
|
|
@ -1281,12 +1317,44 @@ def upload_to_huggingface(
|
|||
pass
|
||||
|
||||
|
||||
def fix_tokenizer_bos_token(tokenizer):
|
||||
# Check if BOS added already, then warn
|
||||
fix_bos_token = False
|
||||
chat_template = getattr(tokenizer, "chat_template", None)
|
||||
|
||||
if (tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None)):
|
||||
if chat_template is not None and \
|
||||
(
|
||||
tokenizer.bos_token in chat_template or \
|
||||
"{bos_token}" in chat_template.replace(" ", "") or \
|
||||
"{bos_token+" in chat_template.replace(" ", "")
|
||||
):
|
||||
|
||||
fix_bos_token = True
|
||||
logger.warning(
|
||||
f"Unsloth: ##### The current model auto adds a BOS token.\n"\
|
||||
"Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily."
|
||||
)
|
||||
|
||||
# Remove {{bos_token}}
|
||||
new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}", "", chat_template)
|
||||
# Remove {{bos_token +
|
||||
new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}", "", new_chat_template)
|
||||
|
||||
tokenizer.chat_template = new_chat_template
|
||||
|
||||
pass
|
||||
pass
|
||||
return fix_bos_token, chat_template
|
||||
pass
|
||||
|
||||
|
||||
def unsloth_save_pretrained_gguf(
|
||||
self,
|
||||
save_directory : Union[str, os.PathLike],
|
||||
tokenizer = None,
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
first_conversion : str = None,
|
||||
push_to_hub : bool = False,
|
||||
token : Optional[Union[str, bool]] = None,
|
||||
private : Optional[bool] = None,
|
||||
|
|
@ -1344,6 +1412,9 @@ def unsloth_save_pretrained_gguf(
|
|||
del arguments["quantization_method"]
|
||||
del arguments["first_conversion"]
|
||||
|
||||
# Fix tokenizer adding an extra BOS token at the front
|
||||
fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)
|
||||
|
||||
# Non blocking install GGUF first
|
||||
if not os.path.exists("llama.cpp"):
|
||||
|
||||
|
|
@ -1386,31 +1457,40 @@ def unsloth_save_pretrained_gguf(
|
|||
pass
|
||||
pass
|
||||
|
||||
# Use old chat template if the bos is removed
|
||||
if fix_bos_token:
|
||||
tokenizer.chat_template = old_chat_template
|
||||
pass
|
||||
|
||||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
model_type = self.config.model_type
|
||||
is_sentencepiece_model = check_if_sentencepiece_model(self)
|
||||
|
||||
# Check if BOS added already, then warn
|
||||
print_bos_token_message = False
|
||||
if (tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None)):
|
||||
chat_template = getattr(tokenizer, "chat_template", None)
|
||||
if chat_template is not None and \
|
||||
(tokenizer.bos_token in chat_template or "{bos_token}" in chat_template.replace(" ", "")):
|
||||
print_bos_token_message = True
|
||||
logger.warning(
|
||||
f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\
|
||||
"Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template."
|
||||
)
|
||||
pass
|
||||
model_dtype = self.config.torch_dtype
|
||||
model_type = self.config.model_type
|
||||
if type(model_dtype) is str:
|
||||
assert(model_dtype == "float16" or model_dtype == "bfloat16")
|
||||
elif model_dtype == torch.float16:
|
||||
model_dtype = "float16"
|
||||
elif model_dtype == torch.bfloat16:
|
||||
model_dtype = "bfloat16"
|
||||
else:
|
||||
raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")
|
||||
pass
|
||||
|
||||
is_sentencepiece_model = check_if_sentencepiece_model(self)
|
||||
|
||||
# Save to GGUF
|
||||
file_location = save_to_gguf(model_type, is_sentencepiece_model,
|
||||
file_location = save_to_gguf(model_type, model_dtype, is_sentencepiece_model,
|
||||
new_save_directory, quantization_method, first_conversion, makefile,
|
||||
)
|
||||
|
||||
if fix_bos_token:
|
||||
logger.warning(
|
||||
f"Unsloth: ##### The current model auto adds a BOS token.\n"\
|
||||
"Unsloth: ##### We removed in GGUF's chat template for you."
|
||||
)
|
||||
pass
|
||||
|
||||
if push_to_hub:
|
||||
print("Unsloth: Uploading GGUF to Huggingface Hub...")
|
||||
username = upload_to_huggingface(
|
||||
|
|
@ -1422,13 +1502,6 @@ def unsloth_save_pretrained_gguf(
|
|||
new_save_directory.lstrip('/.')
|
||||
print(f"Saved GGUF to https://huggingface.co/{link}")
|
||||
pass
|
||||
|
||||
if print_bos_token_message:
|
||||
logger.warning(
|
||||
f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\
|
||||
"Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template."
|
||||
)
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
|
|
@ -1437,7 +1510,7 @@ def unsloth_push_to_hub_gguf(
|
|||
repo_id : str,
|
||||
tokenizer = None,
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
first_conversion : str = None,
|
||||
use_temp_dir : Optional[bool] = None,
|
||||
commit_message : Optional[str] = "Trained with Unsloth",
|
||||
private : Optional[bool] = None,
|
||||
|
|
@ -1490,6 +1563,9 @@ def unsloth_push_to_hub_gguf(
|
|||
del arguments["quantization_method"]
|
||||
del arguments["first_conversion"]
|
||||
|
||||
# Fix tokenizer adding an extra BOS token at the front
|
||||
fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)
|
||||
|
||||
# Non blocking install GGUF first
|
||||
if not os.path.exists("llama.cpp"):
|
||||
|
||||
|
|
@ -1532,28 +1608,30 @@ def unsloth_push_to_hub_gguf(
|
|||
pass
|
||||
pass
|
||||
|
||||
# Use old chat template if the bos is removed
|
||||
if fix_bos_token:
|
||||
tokenizer.chat_template = old_chat_template
|
||||
pass
|
||||
|
||||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
model_type = self.config.model_type
|
||||
is_sentencepiece_model = check_if_sentencepiece_model(self)
|
||||
|
||||
# Check if BOS added already, then warn
|
||||
print_bos_token_message = False
|
||||
if (tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None)):
|
||||
chat_template = getattr(tokenizer, "chat_template", None)
|
||||
if chat_template is not None and \
|
||||
(tokenizer.bos_token in chat_template or "{bos_token}" in chat_template.replace(" ", "")):
|
||||
print_bos_token_message = True
|
||||
logger.warning(
|
||||
f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\
|
||||
"Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template."
|
||||
)
|
||||
pass
|
||||
model_dtype = self.config.torch_dtype
|
||||
model_type = self.config.model_type
|
||||
if type(model_dtype) is str:
|
||||
assert(model_dtype == "float16" or model_dtype == "bfloat16")
|
||||
elif model_dtype == torch.float16:
|
||||
model_dtype = "float16"
|
||||
elif model_dtype == torch.bfloat16:
|
||||
model_dtype = "bfloat16"
|
||||
else:
|
||||
raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")
|
||||
pass
|
||||
|
||||
is_sentencepiece_model = check_if_sentencepiece_model(self)
|
||||
|
||||
# Save to GGUF
|
||||
file_location = save_to_gguf(model_type, is_sentencepiece_model,
|
||||
file_location = save_to_gguf(model_type, model_dtype, is_sentencepiece_model,
|
||||
new_save_directory, quantization_method, first_conversion, makefile,
|
||||
)
|
||||
|
||||
|
|
@ -1568,10 +1646,10 @@ def unsloth_push_to_hub_gguf(
|
|||
|
||||
print(f"Saved GGUF to https://huggingface.co/{link}")
|
||||
|
||||
if print_bos_token_message:
|
||||
if fix_bos_token:
|
||||
logger.warning(
|
||||
f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\
|
||||
"Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template."
|
||||
f"Unsloth: ##### The current model auto adds a BOS token.\n"\
|
||||
"Unsloth: ##### We removed in GGUF's chat template for you."
|
||||
)
|
||||
pass
|
||||
pass
|
||||
|
|
@ -1579,7 +1657,6 @@ pass
|
|||
|
||||
def patch_saving_functions(model):
|
||||
import inspect
|
||||
import re
|
||||
import types
|
||||
from typing import Callable, Optional, Union, List
|
||||
|
||||
|
|
|
|||
|
|
@ -185,6 +185,111 @@ def convert_to_fast_tokenizer(
|
|||
pass
|
||||
|
||||
|
||||
# Check Mistral chat template without BOS / EOS
|
||||
mistral_template = \
|
||||
"{% if messages[0]['role'] == 'system' %}"\
|
||||
"{% if messages[1]['role'] == 'user' %}"\
|
||||
"{{ '[INST] ' + messages[0]['content'] + ' ' + messages[1]['content'] + ' [/INST]' }}"\
|
||||
"{% set loop_messages = messages[2:] %}"\
|
||||
"{% else %}"\
|
||||
"{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}"\
|
||||
"{% set loop_messages = messages[1:] %}"\
|
||||
"{% endif %}"\
|
||||
"{% else %}"\
|
||||
"{% set loop_messages = messages %}"\
|
||||
"{% endif %}"\
|
||||
"{% for message in loop_messages %}"\
|
||||
"{% if message['role'] == 'user' %}"\
|
||||
"{{ '[INST] ' + message['content'] + ' [/INST]' }}"\
|
||||
"{% elif message['role'] == 'assistant' %}"\
|
||||
"{{ message['content'] }}"\
|
||||
"{% else %}"\
|
||||
"{{ raise_exception('Only user and assistant roles are supported!') }}"\
|
||||
"{% endif %}"\
|
||||
"{% endfor %}"
|
||||
pass
|
||||
|
||||
# Check Llama chat template without BOS / EOS
|
||||
llama_template = \
|
||||
"{% if messages[0]['role'] == 'system' %}"\
|
||||
"{% if messages[1]['role'] == 'user' %}"\
|
||||
"{{ '[INST] <<SYS>>\n' + messages[0]['content'] + '\n<</SYS>>\n\n' + messages[1]['content'] + ' [/INST]' }}"\
|
||||
"{% set loop_messages = messages[2:] %}"\
|
||||
"{% else %}"\
|
||||
"{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}"\
|
||||
"{% set loop_messages = messages[1:] %}"\
|
||||
"{% endif %}"\
|
||||
"{% else %}"\
|
||||
"{% set loop_messages = messages %}"\
|
||||
"{% endif %}"\
|
||||
"{% for message in loop_messages %}"\
|
||||
"{% if message['role'] == 'user' %}"\
|
||||
"{{ '[INST] ' + message['content'].strip() + ' [/INST]' }}"\
|
||||
"{% elif message['role'] == 'assistant' %}"\
|
||||
"{{ ' ' + message['content'].strip() + ' ' }}"\
|
||||
"{% else %}"\
|
||||
"{{ raise_exception('Only user and assistant roles are supported!') }}"\
|
||||
"{% endif %}"\
|
||||
"{% endfor %}"
|
||||
pass
|
||||
|
||||
|
||||
def select_correct_slow_tokenizer(
|
||||
tokenizer_name,
|
||||
model_max_length = None,
|
||||
padding_side = "right",
|
||||
token = None,
|
||||
trust_remote_code = False,
|
||||
cache_dir = "huggingface_tokenizers_cache",
|
||||
):
|
||||
"""
|
||||
Returns 'correct' tokenizer by checking if the chat templates are
|
||||
actually tokenized correctly.
|
||||
"""
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{"role": "assistant", "content": "It's 4."},
|
||||
]
|
||||
|
||||
settings = (
|
||||
(False, False, True,),
|
||||
(False, True, True,),
|
||||
(True, False, True,),
|
||||
(True, False, False,),
|
||||
)
|
||||
|
||||
for (use_fast, legacy, from_slow,) in settings:
|
||||
# Default as mentioned by Arthur from HF:
|
||||
slow_tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
# Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373
|
||||
use_fast = use_fast,
|
||||
legacy = legacy,
|
||||
from_slow = from_slow,
|
||||
cache_dir = cache_dir,
|
||||
)
|
||||
slow_tokenizer_chat_template = slow_tokenizer.chat_template
|
||||
|
||||
slow_tokenizer.chat_template = llama_template
|
||||
result1 = slow_tokenizer.decode(slow_tokenizer.apply_chat_template(messages))
|
||||
slow_tokenizer.chat_template = mistral_template
|
||||
result2 = slow_tokenizer.decode(slow_tokenizer.apply_chat_template(messages))
|
||||
|
||||
# If 2 spaces seen, normally wrong!
|
||||
if " "*2 not in result1 and " "*2 not in result2:
|
||||
slow_tokenizer.chat_template = slow_tokenizer_chat_template
|
||||
return slow_tokenizer
|
||||
pass
|
||||
pass
|
||||
# Return fast version as default
|
||||
return slow_tokenizer
|
||||
pass
|
||||
|
||||
|
||||
def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
|
||||
# Get eos_token, bos_token etc
|
||||
dir_names = dir(slow_tokenizer)
|
||||
|
|
@ -193,19 +298,64 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
|
|||
if x.endswith("_token") and x.count("_") == 1
|
||||
)))
|
||||
all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens))
|
||||
|
||||
# Check if chat template is enabled!
|
||||
check_chat_template1 = True
|
||||
check_chat_template2 = True
|
||||
check_chat_template3 = True
|
||||
slow_chat_template = getattr(slow_tokenizer, "chat_template", None)
|
||||
fast_chat_template = getattr(fast_tokenizer, "chat_template", None)
|
||||
messages = [
|
||||
{"role": "user", "content": " What is 2+2? "},
|
||||
{"role": "assistant", "content": " It's 4. "},
|
||||
]
|
||||
# Check the tokenizer's own chat template
|
||||
if slow_chat_template is not None and fast_chat_template is not None:
|
||||
check_chat_template1 = \
|
||||
slow_tokenizer.apply_chat_template(messages) == \
|
||||
fast_tokenizer.apply_chat_template(messages)
|
||||
pass
|
||||
|
||||
# Check Mistral chat template without BOS / EOS
|
||||
slow_tokenizer.chat_template = mistral_template
|
||||
fast_tokenizer.chat_template = mistral_template
|
||||
check_chat_template2 = \
|
||||
slow_tokenizer.apply_chat_template(messages) == \
|
||||
fast_tokenizer.apply_chat_template(messages)
|
||||
pass
|
||||
|
||||
# Check Llama chat template without BOS / EOS
|
||||
slow_tokenizer.chat_template = llama_template
|
||||
fast_tokenizer.chat_template = llama_template
|
||||
check_chat_template3 = \
|
||||
slow_tokenizer.apply_chat_template(messages) == \
|
||||
fast_tokenizer.apply_chat_template(messages)
|
||||
pass
|
||||
|
||||
# Combine them all and revert chat templates
|
||||
check_chat_template = check_chat_template1 and check_chat_template2 and check_chat_template3
|
||||
slow_tokenizer.chat_template = slow_chat_template
|
||||
fast_tokenizer.chat_template = fast_chat_template
|
||||
|
||||
# Try special tokens
|
||||
try:
|
||||
string = "\n".join(all_special_tokens) + \
|
||||
"A quick brown fox jumps over the lazy dog!!\n\nHi</s>\n\n" + \
|
||||
"".join(all_special_tokens)
|
||||
return slow_tokenizer(string).input_ids == fast_tokenizer(string).input_ids
|
||||
check_special_tokens = \
|
||||
slow_tokenizer(string).input_ids == \
|
||||
fast_tokenizer(string).input_ids
|
||||
|
||||
return check_chat_template and check_special_tokens
|
||||
except:
|
||||
# For eg see https://github.com/unslothai/unsloth/issues/292
|
||||
# Sometimes tokenizer has weird tokens, causing a combined tokenization to fail.
|
||||
# [TODO] We temporarily disable this for CodeLlama tokenizers
|
||||
if slow_tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING:
|
||||
return True
|
||||
return check_chat_template
|
||||
else:
|
||||
return False
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
|
|
@ -358,17 +508,13 @@ def load_correct_tokenizer(
|
|||
# Mainly to solve Deepseek models with no tokenizer.model file
|
||||
slow_tokenizer = None
|
||||
try:
|
||||
slow_tokenizer = AutoTokenizer.from_pretrained(
|
||||
slow_tokenizer = select_correct_slow_tokenizer(
|
||||
tokenizer_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
# Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373
|
||||
use_fast = False,
|
||||
legacy = False,
|
||||
from_slow = True,
|
||||
cache_dir = cache_dir,
|
||||
cache_dir = cache_dir,
|
||||
)
|
||||
except:
|
||||
pass
|
||||
|
|
@ -397,6 +543,7 @@ def load_correct_tokenizer(
|
|||
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):
|
||||
return fast_tokenizer
|
||||
else:
|
||||
logger.warning(f"Unsloth: Will load {tokenizer_name} as a legacy tokenizer.")
|
||||
return convert_to_fast_tokenizer(slow_tokenizer)
|
||||
pass
|
||||
else:
|
||||
|
|
@ -574,6 +721,8 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16):
|
|||
# Get set and actual tokens
|
||||
where_untrained = where_untrained.tolist()
|
||||
if len(where_untrained) == 0: return
|
||||
|
||||
# Remove untrained indices where it's longer
|
||||
|
||||
where_untrained_set = frozenset(where_untrained)
|
||||
actual_bad_tokens = tokenizer.convert_ids_to_tokens(where_untrained)
|
||||
|
|
@ -854,11 +1003,13 @@ def patch_sft_trainer_tokenizer():
|
|||
" )\n"\
|
||||
"pass\n"\
|
||||
"n_devices = torch.cuda.device_count()\n"\
|
||||
"more_than = 0\n"\
|
||||
"for j in range(n_devices):\n"\
|
||||
" vram = torch.cuda.max_memory_reserved(torch.cuda.device(j)) / 1024 / 1024 / 1024\n"\
|
||||
" more_than += (vram > 4)\n"\
|
||||
"if more_than > 1: raise RuntimeError('Error: More than 1 GPUs have a lot of VRAM usage.')\n"\
|
||||
"import subprocess, re\n"\
|
||||
"output = subprocess.check_output(\n"\
|
||||
" 'nvidia-smi --query-gpu=memory.used --format=csv', shell = True)\n"\
|
||||
"output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output)\n"\
|
||||
"output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output)\n"\
|
||||
"if output > 1: raise RuntimeError(\n"\
|
||||
" 'Error: More than 1 GPUs have a lot of VRAM usage. Please obtain a commercial license.')\n"\
|
||||
"for _ in range(3):\n"\
|
||||
" gc.collect()\n"\
|
||||
" torch.cuda.empty_cache()\n"\
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue