From 8a9e24ed4f092e438bd09436a897f2ca2530e72b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 13 Jun 2024 05:04:54 +1000 Subject: [PATCH] Ollama Chat Templates (#582) * Update llama.py * offload * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * continued pretraining trainer * Update trainer.py * Update trainer.py * Update trainer.py * Update trainer.py * is_bfloat16_supported * Update __init__.py * Update README.md * Update llama.py * is_bfloat16_supported * Update __init__.py * Mistral v3 * Phi 3 medium * Update chat_templates.py * Update chat_templates.py * Phi-3 * Update save.py * Update README.md Mistral v3 to Mistral v0.3 * Untrained tokens * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update save.py * Update save.py * Update save.py * checkpoint * Update _utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * accelerate * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update tokenizer_utils.py * train_dataloader * Update llama.py * Update llama.py * Update llama.py * use_fast_convert * Update save.py * Update save.py * Update save.py * Update save.py * remove_special_tokens * Ollama * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update llama.py * Update chat_templates.py * Support bfloat16 GGUF * Update save.py * Update llama.py * fast_forward_inference * Update mapper.py * Update loader.py * Update llama.py * Update tokenizer_utils.py * info * edits * Create chat template * Fix tokenizer --------- Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> --- unsloth/chat_templates.py | 679 +++++++++++++++++++++++++++++++++++-- unsloth/models/_utils.py | 2 +- unsloth/models/llama.py | 47 +-- unsloth/models/loader.py | 3 + unsloth/models/mapper.py | 8 + unsloth/models/mistral.py | 54 ++- unsloth/models/qwen2.py | 5 +- unsloth/save.py | 195 +++++++---- unsloth/tokenizer_utils.py | 183 +++++++++- 9 files changed, 1015 insertions(+), 161 deletions(-) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 3decdf7ffc..4c782326b1 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -16,6 +16,12 @@ __all__ = [ "get_chat_template", "test_chat_templates", "test_hf_gguf_equivalence", + "remove_special_tokens", + "standardize_dataset", + + "construct_chat_template", + "test_construct_chat_template", + "create_ollama_modelfile", ] from transformers import StoppingCriteria, StoppingCriteriaList @@ -29,6 +35,7 @@ from .models._utils import patch_tokenizer CHAT_TEMPLATES = {} +# =========================================== Unsloth # Unsloth efficient template leverages from Zephyr unsloth_template = \ "{{ bos_token }}"\ @@ -51,10 +58,24 @@ unsloth_template = \ "{% if add_generation_prompt %}"\ "{{ '>>> Assistant: ' }}"\ "{% endif %}" +pass + +unsloth_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} +{{ end }}{{ if .Prompt }}>>> User: {{ .Prompt }} +{{ end }}>>> Assistant: {{ .Response }}{__EOS_TOKEN__} +""" +PARAMETER stop "{__EOS_TOKEN__}" +SYSTEM """You are a helpful assistant to the user""" +''' + unsloth_eos_token = "eos_token" -CHAT_TEMPLATES["unsloth"] = (unsloth_template, unsloth_eos_token, False,) - +CHAT_TEMPLATES["unsloth"] = (unsloth_template, unsloth_eos_token, False, unsloth_ollama,) +pass +# =========================================== Zephyr # Zephyr has no BOS! zephyr_template = \ "{% for message in messages %}"\ @@ -69,10 +90,26 @@ zephyr_template = \ "{% if add_generation_prompt %}"\ "{{ '<|assistant|>\n' }}"\ "{% endif %}" +pass + +zephyr_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|system|> +{{ .System }}{__EOS_TOKEN__} +{{ end }}{{ if .Prompt }}<|user|> +{{ .Prompt }}{__EOS_TOKEN__} +{{ end }}<|assistant|> +{{ .Response }}{__EOS_TOKEN__} +""" +PARAMETER stop "{__EOS_TOKEN__}" +''' + zephyr_eos_token = "eos_token" -CHAT_TEMPLATES["zephyr"] = (zephyr_template, zephyr_eos_token, False,) - +CHAT_TEMPLATES["zephyr"] = (zephyr_template, zephyr_eos_token, False, zephyr_ollama,) +pass +# =========================================== ChatML # ChatML has no BOS and not EOS! Rather <|im_start|> and <|im_end|> acts as BOS / EOS. chatml_template = \ "{% for message in messages %}"\ @@ -87,10 +124,27 @@ chatml_template = \ "{% if add_generation_prompt %}"\ "{{ '<|im_start|>assistant\n' }}"\ "{% endif %}" +pass + +chatml_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ .Response }}<|im_end|> +""" +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +''' + chatml_eos_token = "<|im_end|>" -CHAT_TEMPLATES["chatml"] = (chatml_template, chatml_eos_token, True,) - +CHAT_TEMPLATES["chatml"] = (chatml_template, chatml_eos_token, True, chatml_ollama,) +pass +# =========================================== Mistral-1 # Mistral Instruct doesn't allow system prompts, so we append it to the user message. mistral_template = \ "{{ bos_token }}"\ @@ -114,10 +168,21 @@ mistral_template = \ "{{ raise_exception('Only user and assistant roles are supported!') }}"\ "{% endif %}"\ "{% endfor %}" +pass + +# Ollama from https://www.ollama.com/library/mistral +mistral_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """[INST] {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} [/INST]""" +PARAMETER stop "{__EOS_TOKEN__}" +''' + mistral_eos_token = "eos_token" -CHAT_TEMPLATES["mistral"] = (mistral_template, mistral_eos_token, False,) - +CHAT_TEMPLATES["mistral"] = (mistral_template, mistral_eos_token, False, mistral_ollama,) +pass +# =========================================== Llama-2 # Adds BOS to every convo! And weird <> system messages. llama_template = \ "{% if messages[0]['role'] == 'system' %}"\ @@ -140,10 +205,23 @@ llama_template = \ "{{ raise_exception('Only user and assistant roles are supported!') }}"\ "{% endif %}"\ "{% endfor %}" +pass + +# Ollama from https://www.ollama.com/library/llama3 +llama_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """[INST] <>{{ .System }}<> + +{{ .Prompt }} [/INST]""" +PARAMETER stop "{__EOS_TOKEN__}" +''' + llama_eos_token = "eos_token" -CHAT_TEMPLATES["llama"] = (llama_template, llama_eos_token, False,) - +CHAT_TEMPLATES["llama"] = (llama_template, llama_eos_token, False, llama_ollama,) +pass +# =========================================== Vicuna # https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template vicuna_template = \ "{{ bos_token }}"\ @@ -166,10 +244,21 @@ vicuna_template = \ "{% if add_generation_prompt %}"\ "{{ 'ASSISTANT:' }}"\ "{% endif %}" +pass + +# Ollama from https://www.ollama.com/library/vicuna +vicuna_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}USER: {{ .Prompt }} {{ end }}ASSISTANT: {{ .Response }} {__EOS_TOKEN__}""" +PARAMETER stop "{__EOS_TOKEN__}" +''' + vicuna_eos_token = "eos_token" -CHAT_TEMPLATES["vicuna"] = (vicuna_template, vicuna_eos_token, False,) - +CHAT_TEMPLATES["vicuna"] = (vicuna_template, vicuna_eos_token, False, vicuna_ollama,) +pass +# =========================================== Vicuna Old # https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template vicuna_old_template = \ "{{ bos_token }}"\ @@ -192,10 +281,24 @@ vicuna_old_template = \ "{% if add_generation_prompt %}"\ "{{ '### Assistant:' }}"\ "{% endif %}" +pass + +vicuna_old_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} +{{ end }}{{ if .Prompt }}### Human: {{ .Prompt }} +{{ end }}### Assistant: {{ .Response }}{__EOS_TOKEN__} +""" +PARAMETER stop "{__EOS_TOKEN__}" +SYSTEM """A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.""" +''' + vicuna_old_eos_token = "eos_token" -CHAT_TEMPLATES["vicuna_old"] = (vicuna_old_template, vicuna_old_eos_token, False,) - +CHAT_TEMPLATES["vicuna_old"] = (vicuna_old_template, vicuna_old_eos_token, False, vicuna_old_ollama,) +pass +# =========================================== Alpaca multi turn # https://github.com/tatsu-lab/stanford_alpaca Changed for multi-turn convos alpaca_template = \ "{{ bos_token }}"\ @@ -203,7 +306,7 @@ alpaca_template = \ "{{ messages[0]['content'] + '\n\n' }}"\ "{% set loop_messages = messages[1:] %}"\ "{% else %}"\ - "{{ 'Below are some instructions that describes some tasks. Write responses that appropriately completes each request.\n\n' }}"\ + "{{ 'Below are some instructions that describe some tasks. Write responses that appropriately complete each request.\n\n' }}"\ "{% set loop_messages = messages %}"\ "{% endif %}"\ "{% for message in loop_messages %}"\ @@ -218,42 +321,100 @@ alpaca_template = \ "{% if add_generation_prompt %}"\ "{{ '### Response:\n' }}"\ "{% endif %}" +pass + +alpaca_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} + +{{ end }}{{ if .Prompt }}### Instruction: +{{ .Prompt }}{{ end }} + +### Response: +{{ .Response }}{__EOS_TOKEN__} + +""" +PARAMETER stop "{__EOS_TOKEN__}" +SYSTEM """Below are some instructions that describe some tasks. Write responses that appropriately complete each request.""" +''' + alpaca_eos_token = "eos_token" -CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token, False,) - +CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token, False, alpaca_ollama,) +pass +# =========================================== Gemma # https://huggingface.co/google/gemma-7b-it # Notice we must use |trim for lstrip and rstrip. maps to 106. # maps to 107. user and model are normal 1 word tokens. gemma_template = \ "{{ bos_token }}"\ + "{% if messages[0]['role'] == 'system' %}"\ + "{{'user\n' + messages[0]['content'] | trim + ' ' + messages[1]['content'] | trim + '\n'}}"\ + "{% set loop_messages = messages[2:] %}"\ + "{% endif %}"\ "{% for message in messages %}"\ "{% if message['role'] == 'user' %}"\ "{{'user\n' + message['content'] | trim + '\n'}}"\ "{% elif message['role'] == 'assistant' %}"\ "{{'model\n' + message['content'] | trim + '\n' }}"\ "{% else %}"\ - "{{ 'system\n' + message['content'] | trim + '\n' }}"\ + "{{ raise_exception('Only user and assistant roles are supported!') }}"\ "{% endif %}"\ "{% endfor %}"\ "{% if add_generation_prompt %}"\ "{{ 'model\n' }}"\ "{% endif %}" +pass + +# Ollama from https://www.ollama.com/library/gemma +gemma_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """user +{{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} +model +{{ .Response }} +""" +PARAMETER repeat_penalty 1 +PARAMETER stop "" +PARAMETER stop "" +PARAMETER penalize_newline false +''' + gemma_eos_token = "" -CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token, True,) +CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token, True, gemma_ollama,) +pass - -# Gemma with ChatML instead +# =========================================== Gemma with ChatML instead # We find using is still more appropriate! gemma_chatml_template = "{{ bos_token }}" + chatml_template +pass + +gemma_chatml_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ .Response }}<|im_end|> +""" +PARAMETER repeat_penalty 1 +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER penalize_newline false +''' + gemma_chatml_eos_token = ( {"" : "<|im_start|>", "" : "<|im_end|>"}, "<|im_end|>", ) -CHAT_TEMPLATES["gemma_chatml"] = (gemma_chatml_template, gemma_chatml_eos_token, True,) +CHAT_TEMPLATES["gemma_chatml"] = (gemma_chatml_template, gemma_chatml_eos_token, True, gemma_chatml_ollama,) +pass - -# Llama-3 +# =========================================== Llama-3 # Weirdly \n\n is needed? llama3_template = \ "{{ bos_token }}"\ @@ -269,11 +430,30 @@ llama3_template = \ "{% if add_generation_prompt %}"\ "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"\ "{% endif %}" +pass + +# Ollama from https://www.ollama.com/library/llama3 +llama3_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|> + +{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> + +{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> + +{{ .Response }}<|eot_id|>""" +PARAMETER stop "<|start_header_id|>" +PARAMETER stop "<|end_header_id|>" +PARAMETER stop "<|eot_id|>" +''' + llama3_template_eos_token = "eos_token" -CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token, False,) +CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,) +pass -# Phi-3 +# =========================================== Phi-3 phi3_template = \ "{{ bos_token }}"\ "{% for message in messages %}"\ @@ -288,8 +468,27 @@ phi3_template = \ "{% if add_generation_prompt %}"\ "{{ '<|assistant|>\n' }}"\ "{% endif %}" +pass + +# Ollama from https://www.ollama.com/library/phi3 +phi3_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|system|> +{{ .System }}<|end|> +{{ end }}{{ if .Prompt }}<|user|> +{{ .Prompt }}<|end|> +{{ end }}<|assistant|> +{{ .Response }}<|end|> +""" +PARAMETER stop "<|end|>" +PARAMETER stop "<|user|>" +PARAMETER stop "<|assistant|>" +''' + phi3_template_eos_token = "<|end|>" -CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token, False,) +CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token, False, phi3_ollama,) +pass def get_chat_template( @@ -297,6 +496,7 @@ def get_chat_template( chat_template = "chatml", mapping = {"role" : "role", "content" : "content", "user" : "user", "assistant" : "assistant"}, map_eos_token = True, + system_message = None, ): assert(type(map_eos_token) is bool) old_tokenizer = tokenizer @@ -331,7 +531,7 @@ def get_chat_template( elif type(chat_template) is str: - chat_template, stop_word, yes_map_eos_token = CHAT_TEMPLATES[chat_template] + chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template] # Check mapping to eos_token if not map_eos_token and yes_map_eos_token: map_eos_token = True @@ -496,10 +696,421 @@ def get_chat_template( # Patch saving functions tokenizer = patch_saving_functions(tokenizer) + # Add Ollama + tokenizer._ollama_modelfile = ollama_modelfile + tokenizer._system_message = system_message return tokenizer#, stopping_criteria pass +def remove_special_tokens(tokenizer, prompt): + # Removes double BOS token + if prompt.startswith(tokenizer.bos_token): + prompt = prompt[len(tokenizer.bos_token):] + pass + return prompt +pass + + +def standardize_dataset( + dataset, + conversation_key = "conversations", + system_message = None, + aliases_for_system = ["system",], + aliases_for_user = ["user", "human", "input",], + aliases_for_assistant = ["gpt", "assistant", "output",], +): + """ + Standardizes ShareGPT and other formats to user/assistant Hugging Face format. + """ + import collections + import itertools + + convos = dataset[:10][conversation_key] + uniques = collections.defaultdict(list) + for convo in convos: + for message in convo: + for key, value in message.items(): + uniques[key].append(value) + pass + + # Must be only 2 entries + assert(len(uniques.keys()) == 2) + + keys = list(uniques.keys()) + length_first = len(set(uniques[keys[0]])) + length_second = len(set(uniques[keys[1]])) + + if length_first < length_second: + # Role is assigned to the first element + role_key = keys[0] + content_key = keys[1] + else: + role_key = keys[1] + content_key = keys[0] + pass + + # Check roles are in aliases + all_aliases = set(aliases_for_system + aliases_for_user + aliases_for_assistant) + roles = set(uniques[role_key]) + leftover_aliases = (all_aliases | roles) - all_aliases + if len(leftover_aliases) != 0: + raise TypeError( + f"Unsloth: {list(leftover_aliases)} are not in aliases. Please update aliases." + ) + pass + + # Mapping for aliases + aliases_mapping = {} + for x in aliases_for_system: aliases_mapping[x] = "system" + for x in aliases_for_user: aliases_mapping[x] = "user" + for x in aliases_for_assistant: aliases_mapping[x] = "assistant" + + def _standardize_dataset(examples): + convos = examples[conversation_key] + all_convos = [] + for convo in convos: + new_convo = [] + if len(convo) == 0: continue + has_system = aliases_mapping[convo[0][role_key]] == "system" + if not has_system and system_message is not None: + new_convo.append({ "role" : "system", "content" : system_message, }) + for message in convo: + role = aliases_mapping[message[role_key]] + new_convo.append({ "role" : role, "content" : message[content_key], }) + pass + all_convos.append(new_convo) + pass + return { conversation_key : all_convos, } + pass + + return dataset.map(_standardize_dataset, batched = True,) +pass + + +def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []): + added_tokens_decoder = tokenizer.added_tokens_decoder.values() + added_tokens_decoder = [str(x) for x in added_tokens_decoder] + + # Remove added_tokens_decoder duplicates + added_tokens_decoder = list(set(added_tokens_decoder) - set(extra_eos_tokens)) + + # Remove BOS + if getattr(tokenizer, "bos_token", None) is not None: + added_tokens_decoder = [x for x in added_tokens_decoder if x != tokenizer.bos_token] + pass + + repeatted_tokens = [] + # Join all vocab + joined_text = "\x01\x00".join(added_tokens_decoder) + for token in added_tokens_decoder: + n = len(token) + repeatted_counts = joined_text.count(token[:n//2]) + # Try finding longer than 1/2 of the token in the rest + # For eg <|reserved_special_token_0|>, <|reserved_special_token_1|> + if repeatted_counts > 2: + for j in range(n//2+1, n): + if joined_text.count(token[:j]) < repeatted_counts: + j -= 1 + # Remove repeatted tokens to reduce search space + joined_text = joined_text.replace(token[:j], "") + repeatted_tokens.append(token[:j]) + break + pass + pass + pass + + # Remove duplicates + splitted = joined_text.split("\x01\x00") + final_eos_tokens = [] + for old, new in zip(added_tokens_decoder, splitted): + if old == new: final_eos_tokens.append(old) + pass + final_eos_tokens += extra_eos_tokens + final_eos_tokens += repeatted_tokens + return final_eos_tokens +pass + + +def construct_chat_template( \ + +tokenizer = None, + +template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> + +{SYSTEM}<|eot_id|><|start_header_id|>user<|end_header_id|> + +{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{OUTPUT}<|eot_id|><|start_header_id|>user<|end_header_id|> + +{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{OUTPUT}<|eot_id|>""", + +default_system_message = \ + "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.", + +extra_eos_tokens = None, + +): + """ + Creates a Ollama modelfile and a HF Jinja template from a custom + template. You must provide 2x examples of an input & output. + There is an optional system message as well. + + You must use {INPUT}, {OUTPUT} twice, and {SYSTEM} is optional. + """ + assert(tokenizer is not None) + + if extra_eos_tokens is None: extra_eos_tokens = [] + + vocab = tokenizer.get_vocab() + for extra_eos in extra_eos_tokens: + assert(type(extra_eos) is str) + if extra_eos not in vocab: + raise ValueError(f"Unsloth: `{extra_eos}` is not a singular token in the tokenizer.") + pass + pass + + error_msg = \ + "Unsloth: Your prompt template must have 2 examples showing the user input {INPUT} "\ + "and the assistant output {OUTPUT}\n\n"\ + "For example what is not allowed is just:\n"\ + "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n\n\n"\ + "What is required is 2x of this:\n"\ + "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n"\ + "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n" + + # O(N^2) search finding 2 repeatted pieces of text + j = len(template)-1 + at_least_one = False + while j > 0: + found = template.rfind(template[j:], 0, j) + if found == -1: break + j -= 1 + at_least_one = True + pass + if j > 0: j += 1 + else: raise RuntimeError(error_msg) + + + if not at_least_one: raise RuntimeError(error_msg) + + # Repeatted text + instruction_response = template[j:] + if instruction_response.count("{INPUT}") != 1 or instruction_response.count("{OUTPUT}") != 1: + raise RuntimeError(error_msg) + pass + + # 1st System, Instruction, Output pair + left = template[:j] + # 2nd Instruction, Output pair + right = template[j:] + + # Isolate input + extra_eos_tokens_regex = "|".join(f"(?:{re.escape(x)})" for x in extra_eos_tokens) + if len(extra_eos_tokens_regex) != 0: + find_end = f"(?:{extra_eos_tokens_regex})?" + else: + find_end = "" + find_end = r"\{INPUT\}[\s\n]{0,}" + find_end + input_end = list(re.finditer(find_end, right)) + assert(len(input_end) == 1) + input_end = input_end[0] + input_end = input_end.span(0)[1] + input_part = right[:input_end] + + # Isolate output + output_part = right[input_end:] + + # Isolate system + system_part = left[:left.find(input_part)] + + # Check if the user provided a correct prompt + combined = system_part + input_part + output_part + if combined != left: + combined_changed = combined.replace('\n', '\\n') + left_changed = left .replace('\n', '\\n') + raise RuntimeError( + "Unsloth: The prompt template you provided isn't correct. You gave:\n"\ + f"{combined_changed}\n\n"\ + "But we require the following:\n"\ + f"{left_changed}" + ) + pass + + # Ollama modelfile parts + + # Check bos_token is in system prompt + ollama_system = system_part + has_bos_token = False + if tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None): + if ollama_system.startswith(tokenizer.bos_token): + has_bos_token = True + ollama_system = ollama_system[len(tokenizer.bos_token):] + pass + pass + system_modelfile = "{{ if .System }}" + ollama_system.replace("{SYSTEM}", "{{ .System }}") + "{{ end }}" + input_modelfile = "{{ if .Prompt }}" + input_part .replace("{INPUT}", "{{ .Prompt }}") + "{{ end }}" + output_modelfile = output_part.replace("{OUTPUT}", "{{ .Response }}") + + # Check if EOS token is at the end of the output + if not output_modelfile.endswith(tuple(extra_eos_tokens)): + output_modelfile += "{__EOS_TOKEN__}" + pass + + # Ollama EOS + ollama_eos = get_ollama_eos_tokens(tokenizer, extra_eos_tokens) + ollama_eos = '\n'.join(f'PARAMETER stop "{eos}"' for eos in ollama_eos) + + # Ollama modelfile + modelfile = 'FROM {__FILE_LOCATION__}\n\n'\ + 'TEMPLATE """' + system_modelfile + input_modelfile + output_modelfile + \ + '"""\n\n' + ollama_eos + + # HF Jinja Chat template + def process(part, which, content = "message['content']"): + if part.endswith(which): + part = "'" + part[:part.find(which)] + f"' + {content}" + elif part.startswith(which): + part = f"{content} + '" + part[part.find(which):] + "'" + else: + part = "'" + part.replace(which, f"' + {content} + '") + "'" + if part.startswith("'' + "): part = part[5:] + return part + pass + input_jinja = process(input_part, "{INPUT}") + output_jinja = process(output_part, "{OUTPUT}") + pass + + jinja_template = \ + "{% for message in loop_messages %}"\ + "{% if message['role'] == 'user' %}"\ + "{{ " + input_jinja + " }}"\ + "{% elif message['role'] == 'assistant' %}"\ + "{{ " + output_jinja + " }}"\ + "{% else %}"\ + "{{ raise_exception('Only user and assistant roles are supported!') }}"\ + "{% endif %}"\ + "{% endfor %}"\ + "{% if add_generation_prompt %}"\ + "{{ '" + output_part[:output_part.find("{OUTPUT}")] + "' }}"\ + "{% endif %}" + pass + + # Now add system prompt to jinja + if len(system_part) != 0: + partial_system = process(system_part, "{SYSTEM}", "messages[0]['content']") + partial_system = partial_system.replace("{SYSTEM}", "") + + # Separate the BOS + if has_bos_token: + partial_system = partial_system.replace(tokenizer.bos_token, "", 1) + pass + + partial_system = \ + "{% if messages[0]['role'] == 'system' %}"\ + "{{ " + partial_system + " }}"\ + "{% set loop_messages = messages[1:] %}" + if default_system_message is not None: + partial_system += "{% else %}"\ + "{{ '" + system_part.replace("{SYSTEM}", default_system_message) + "' }}"\ + "{% set loop_messages = messages %}"\ + "{% endif %}" + else: + partial_system += "{% endif %}" + pass + + jinja_template = partial_system + jinja_template + + if has_bos_token: + jinja_template = "{{ bos_token }}" + jinja_template + pass + + return modelfile, jinja_template +pass + + +def test_construct_chat_template(): + token = "hf_" + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct", token = token) + + template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> + +{SYSTEM}<|eot_id|><|start_header_id|>user<|end_header_id|> + +{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{OUTPUT}<|eot_id|><|start_header_id|>user<|end_header_id|> + +{INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{OUTPUT}<|eot_id|>""" + + default_system_message = \ + "Below are some instructions that describe some tasks. Write responses that appropriately complete each request." + + extra_eos_tokens = None + + modelfile, jinja_template = construct_chat_template(template, default_system_message, extra_eos_tokens) + + messages = [ + {"role": "system", "content": "You are an assistant"}, + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "It's 4."}, + {"role": "user", "content": "Ok!"}, + {"role": "assistant", "content": "Anything else?"}, + {"role": "user", "content": "What's 2x2?"}, + ] + correct_output = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + + tokenizer.chat_template = jinja_template + new_output = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + + assert(correct_output == new_output) + pass +pass + + +def create_ollama_modelfile(tokenizer, gguf_location): + """ + Creates an Ollama Modelfile. + Use ollama.create(model = "new_ollama_model", modelfile = modelfile) + """ + modelfile = getattr(tokenizer, "_ollama_modelfile", None) + if modelfile is None: + raise RuntimeError( + "Unsloth: Tokenizer does not have a `ollama_modelfile` attribute.\n"\ + "Please use get_chat_template(...)." + ) + pass + + system_message = getattr(tokenizer, "_system_message", None) + if system_message is None: + __SYSTEM_MESSAGE__ = "" + else: + __SYSTEM_MESSAGE__ = f'SYSTEM """{system_message}"""' + pass + + modelfile = modelfile\ + .replace("{{", "⚫@✅#🦥")\ + .replace("}}", "⚡@🦥#⛵")\ + .format( + __FILE_LOCATION__ = gguf_location, + __SYSTEM_MESSAGE__ = __SYSTEM_MESSAGE__, + __EOS_TOKEN__ = tokenizer.eos_token, + )\ + .replace("⚫@✅#🦥", "{{")\ + .replace("⚡@🦥#⛵", "}}")\ + .rstrip() + pass + + return modelfile +pass + + def create_stopping_criteria(tokenizer, stop_word = "eos_token"): class StoppingCriteriaSub(StoppingCriteria): __slots__ = "stop_token", "single_match", "length", @@ -670,7 +1281,8 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf") if tokenizer.chat_template is not None: prompt = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) prompt = prompt.replace("'", "") # Subprocess does not like '' - prompts.append(prompts) + prompt = remove_special_tokens(tokenizer, prompt) + prompts.append(prompt) pass for prompt in prompts: @@ -688,9 +1300,9 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf") gguf_tokenized = re.findall("([\d]{1,}) \-\> \'([^\']{1,})\'", gguf_tokens, flags = re.MULTILINE) gguf_tokenized = [(int(x[0]), x[1],) for x in gguf_tokenized] input_ids = tokenizer(prompt).input_ids + tokens = tokenizer.batch_decode(input_ids) hf_tokenized = list(zip(input_ids, tokens)) - print(gguf_tokenized[:5]) # Compare to Huggingface for j, (hf_token, gguf_token) in enumerate(zip(hf_tokenized, gguf_tokenized)): @@ -698,9 +1310,10 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf") print("Failed GGUF != HF at", j) print("HF =", hf_token) print("GGUF =", gguf_token) - print(hf_tokenized[:j+1]) - print(gguf_tokenized[:j+1]) - print(gguf_tokens) + print(hf_tokenized) + print() + print(gguf_tokenized) + print() raise RuntimeError("Failed comparing GGUF to HF.") pass pass diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index bcd4a7b30a..a693389355 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -31,7 +31,7 @@ import numpy as np import os import psutil -__version__ = "2024.5" +__version__ = "2024.6" # Get Flash Attention v2 if Ampere (RTX 30xx, A100) major_version, minor_version = torch.cuda.get_device_capability() diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 0e860b9de4..4cbbcf0a82 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -209,8 +209,9 @@ def LlamaAttention_fast_forward_inference( # Attention if bsz == 1: + Qn *= self.scalar # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963 + # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows A = torch.matmul(Qn, Knn.transpose(2, 3), out = self.attention[:,:,:,:cached_len]) - A *= self.scalar # if attention_mask is not None: A += attention_mask # Must add attention_mask for batched A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)#.to(A.dtype) A = torch.matmul(A, Vnn, out = Qn) @@ -791,7 +792,7 @@ def CausalLM_fast_forward(fast_forward_inference): *args, **kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: - if past_key_values is not None and self.config.model_type != "qwen2": + if past_key_values is not None: outputs = fast_forward_inference( self, input_ids, @@ -1195,7 +1196,13 @@ class FastLlamaModel: f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\ f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}' logger.warning(debug_info) - import gc + import subprocess, re, gc + output = subprocess.check_output( + 'nvidia-smi --query-gpu=memory.used --format=csv', shell = True) + output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output) + output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output) + if output > 1: raise RuntimeError( + 'Error: More than 1 GPUs have a lot of VRAM usage. Please obtain a commercial license.') for _ in range(3): gc.collect() torch.cuda.empty_cache()""" @@ -1206,12 +1213,12 @@ class FastLlamaModel: debug_info = """n_total_devices = total_train_batch_size // \\ args.gradient_accumulation_steps // self._train_batch_size - if n_total_devices > 2: + if n_total_devices > 1: logger.warning_once( - "Our OSS was designed for people with few GPU resources to level the playing field.\\n" - "The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n" - "We're a 2 person team, so we still have to fund our development costs - thanks!\\n" - "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + "* Our OSS was designed for people with few GPU resources to level the playing field.\\n" + "* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n" + "* We're a 2 person team, so we still have to fund our development costs - thanks!\\n" + "* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", ) debug_info =""" debug_info = debug_info.split('\n') @@ -1236,17 +1243,17 @@ class FastLlamaModel: bsz = self._train_batch_size total_batches = bsz * ga * args.world_size n_total_devices = total_batches // ga // bsz - if n_total_devices > 2: + if n_total_devices > 1: logger.warning_once( - "Our OSS was designed for people with few GPU resources to level the playing field.\\n" - "The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n" - "We're a 2 person team, so we still have to fund our development costs - thanks!\\n" - "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + "* Our OSS was designed for people with few GPU resources to level the playing field.\\n" + "* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n" + "* We're a 2 person team, so we still have to fund our development costs - thanks!\\n" + "* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", ) - divisor = n_total_devices / 2 + divisor = n_total_devices / 1 bsz = self._train_batch_size = max(int(bsz / divisor), 1) - if total_batches // ga // bsz > 2: - divisor = n_total_devices / 2 + if total_batches // ga // bsz > 1: + divisor = n_total_devices / 1 ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)""" check_batches = check_batches.split('\n') check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]]) @@ -1830,10 +1837,10 @@ class FastLlamaModel: @staticmethod def for_inference(model): - if model.config.model_type == "qwen2": - FastLlamaModel.for_training(model) - return - pass + # if model.config.model_type == "qwen2": + # FastLlamaModel.for_training(model) + # return + # pass internal_model = model internal_model.gradient_checkpointing = False diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index b2f0e4efdc..3bc091b364 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -33,6 +33,9 @@ del major, minor def _get_model_name(model_name, load_in_4bit = True): + # First try replacing lowercase 'b' with uppercase 'B' + model_name = model_name.lower() + if not SUPPORTS_FOURBIT and model_name in INT_TO_FLOAT_MAPPER: model_name = INT_TO_FLOAT_MAPPER[model_name] logger.warning_once( diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 8808b8554d..73aa06ca68 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -197,4 +197,12 @@ for key, values in __INT_TO_FLOAT_MAPPER.items(): for value in values: FLOAT_TO_INT_MAPPER[value] = key pass + + # Get lowercased + lowered_key = key.lower() + INT_TO_FLOAT_MAPPER[lowered_key] = values[0].lower() + + for value in values: + FLOAT_TO_INT_MAPPER[value.lower()] = lowered_key + pass pass diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 365d60a3e4..fc2e1a9fb0 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -393,21 +393,6 @@ class FastMistralModel(FastLlamaModel): layer.self_attn.apply_o = original_apply_o pass - # Patch Trainer - from transformers.trainer import Trainer - if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": - try: - inner_training_loop = inspect.getsource(Trainer._inner_training_loop) - except: - raise RuntimeError( - "Our OSS was designed for people with few GPU resources to level the playing field.\n" - "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" - "We're a 2 person team, so we still have to fund our development costs - thanks!\n" - "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", - ) - pass - pass - # Patch Trainer from transformers.trainer import Trainer try: @@ -419,7 +404,7 @@ class FastMistralModel(FastLlamaModel): except: raise RuntimeError( "Our OSS was designed for people with few GPU resources to level the playing field.\n" - "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\n" "We're a 2 person team, so we still have to fund our development costs - thanks!\n" "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", ) @@ -447,7 +432,17 @@ class FastMistralModel(FastLlamaModel): f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\ f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\ f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}' - logger.warning_once(debug_info)""" + logger.warning(debug_info) + import subprocess, re, gc + output = subprocess.check_output( + 'nvidia-smi --query-gpu=memory.used --format=csv', shell = True) + output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output) + output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output) + if output > 1: raise RuntimeError( + 'Error: More than 1 GPUs have a lot of VRAM usage. Please obtain a commercial license.') + for _ in range(3): + gc.collect() + torch.cuda.empty_cache()""" debug_info = debug_info.split('\n') debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]]) @@ -455,12 +450,12 @@ class FastMistralModel(FastLlamaModel): debug_info = """n_total_devices = total_train_batch_size // \\ args.gradient_accumulation_steps // self._train_batch_size - if n_total_devices > 2: + if n_total_devices > 1: logger.warning_once( - "Our OSS was designed for people with few GPU resources to level the playing field.\\n" - "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n" - "We're a 2 person team, so we still have to fund our development costs - thanks!\\n" - "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + "* Our OSS was designed for people with few GPU resources to level the playing field.\\n" + "* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n" + "* We're a 2 person team, so we still have to fund our development costs - thanks!\\n" + "* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", ) debug_info =""" debug_info = debug_info.split('\n') @@ -485,16 +480,17 @@ class FastMistralModel(FastLlamaModel): bsz = self._train_batch_size total_batches = bsz * ga * args.world_size n_total_devices = total_batches // ga // bsz - if n_total_devices > 2: + if n_total_devices > 1: logger.warning_once( - "Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n" - "The OSS currently works on 4 GPUs - we're a 2 person team, so please help fund\\n" - "our development costs by supporting us through Ko-fi or buying a license! Thanks!", + "* Our OSS was designed for people with few GPU resources to level the playing field.\\n" + "* The OSS Apache 2 license only supports one GPU - please obtain a commercial license.\\n" + "* We're a 2 person team, so we still have to fund our development costs - thanks!\\n" + "* If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", ) - divisor = n_total_devices / 2 + divisor = n_total_devices / 1 bsz = self._train_batch_size = max(int(bsz / divisor), 1) - if total_batches // ga // bsz > 2: - divisor = n_total_devices / 2 + if total_batches // ga // bsz > 1: + divisor = n_total_devices / 1 ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)""" check_batches = check_batches.split('\n') check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]]) diff --git a/unsloth/models/qwen2.py b/unsloth/models/qwen2.py index 76fe31a6d1..115bf3e090 100644 --- a/unsloth/models/qwen2.py +++ b/unsloth/models/qwen2.py @@ -13,7 +13,6 @@ # limitations under the License. from .llama import * -from .mistral import FastMistralModel import os from ._utils import __version__ @@ -60,7 +59,7 @@ class FastQwen2Model(FastLlamaModel): @staticmethod def from_pretrained( - model_name = "Qwen/Qwen1.5-7B", + model_name = "Qwen/Qwen2-7B", max_seq_length = 4096, dtype = None, load_in_4bit = True, @@ -73,7 +72,7 @@ class FastQwen2Model(FastLlamaModel): trust_remote_code = False, **kwargs, ): - return FastMistralModel.from_pretrained( + return FastLlamaModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, dtype = dtype, diff --git a/unsloth/save.py b/unsloth/save.py index 5d6f925d45..3ad2f3465a 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -59,7 +59,8 @@ ALLOWED_QUANTS = \ "fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.", "quantized" : "Recommended. Slow conversion. Fast inference, small files.", "f32" : "Not recommended. Retains 100% accuracy, but super slow and memory hungry.", - "f16" : "Fastest conversion + retains 100% accuracy. Slow and memory hungry.", + "bf16" : "Bfloat16 - Fastest conversion + retains 100% accuracy. Slow and memory hungry.", + "f16" : "Float16 - Fastest conversion + retains 100% accuracy. Slow and memory hungry.", "q8_0" : "Fast conversion. High resource use, but generally acceptable.", "q4_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K", "q5_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K", @@ -102,7 +103,7 @@ def check_if_sentencepiece_model(model, temporary_location = "_unsloth_sentencep if os.path.isfile(f"{file_location}/tokenizer.model"): sentencepiece_model = True pass - shutil.rmtree(file_location) + shutil.rmtree(file_location, ignore_errors = True) return sentencepiece_model pass @@ -700,7 +701,7 @@ def unsloth_save_model( # Remove temporary location import shutil - shutil.rmtree(temporary_location) + shutil.rmtree(temporary_location, ignore_errors = True) for _ in range(3): torch.cuda.empty_cache() @@ -763,7 +764,7 @@ def install_llama_cpp_old(version = -10): print(f"**[WARNING]** Deleting llama.cpp directory... {10-i} seconds left.") time.sleep(1) import shutil - shutil.rmtree("llama.cpp") + shutil.rmtree("llama.cpp", ignore_errors = True) pass # Clone a specific commit @@ -866,10 +867,11 @@ pass def save_to_gguf( model_type : str, + model_dtype : str, is_sentencepiece : bool = False, model_directory : str = "unsloth_finetuned_model", quantization_method : str = "fast_quantized", - first_conversion : str = "f16", + first_conversion : str = None, _run_installer = None, # Non blocking install of llama.cpp ): # logger.warning( @@ -877,6 +879,22 @@ def save_to_gguf( # "undergoing some major bug fixes as at 5th of May 2024. This is not an Unsloth issue.\n"\ # "Please be patient - GGUF saving should still work, but might not work as well." # ) + assert(model_dtype == "float16" or model_dtype == "bfloat16") + model_dtype = "f16" if model_dtype == "float16" else "bf16" + + # Check if bfloat16 is supported + if model_dtype == "bf16" and not torch.cuda.is_bf16_supported(): + logger.warning( + "Unsloth: Cannot convert to bf16 GGUF since your computer doesn't support it.\n"\ + "We shall switch instead to f16." + ) + model_dtype = "f16" + pass + + # Check first_conversion as well + if first_conversion is None: + first_conversion = model_dtype + pass if quantization_method.startswith("iq2"): raise RuntimeError("Unsloth: Currently iq2 type quantizations aren't supported yet - sorry!") @@ -889,7 +907,7 @@ def save_to_gguf( pass logger.warning_once(f"Unsloth: Converting {model_type} model. Can use fast conversion = {use_fast_convert}.") - if quantization_method == "not_quantized": quantization_method = "f16" + if quantization_method == "not_quantized": quantization_method = model_dtype elif quantization_method == "fast_quantized": quantization_method = "q8_0" elif quantization_method == "quantized": quantization_method = "q4_k_m" elif quantization_method is None: quantization_method = "q8_0" @@ -911,12 +929,13 @@ def save_to_gguf( print(print_info) # Check first_conversion format - if first_conversion == "f16" : pass - elif first_conversion == "f32" : pass - elif first_conversion == "q8_0": pass + if first_conversion == "f16" : pass + if first_conversion == "bf16" : pass + elif first_conversion == "f32" : pass + elif first_conversion == "q8_0" : pass else: raise RuntimeError( - f"Unsloth: `first_conversion` can only be one of ['f16', 'f32', 'q8_0'] and not `{first_conversion}`." + f"Unsloth: `first_conversion` can only be one of ['f16', 'bf16', 'f32', 'q8_0'] and not `{first_conversion}`." ) pass @@ -935,11 +954,13 @@ def save_to_gguf( if quantization_method == "f32": first_conversion = "f32" elif quantization_method == "f16": first_conversion = "f16" + elif quantization_method == "bf16": first_conversion = "bf16" elif quantization_method == "q8_0": first_conversion = "q8_0" else: # Quantized models must have f16 as the default argument - if first_conversion == "f32" : pass - elif first_conversion == "f16" : pass + if first_conversion == "f32" : pass + elif first_conversion == "f16" : pass + elif first_conversion == "bf16" : pass elif first_conversion == "q8_0": logger.warning_once( "Unsloth: Using q8_0 for the `first_conversion` will lose a bit of accuracy, "\ @@ -950,8 +971,22 @@ def save_to_gguf( pass # Non llama/mistral needs can only use f32 or f16 - if not use_fast_convert and (first_conversion != "f16" or first_conversion != "f32"): - logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.") + if not use_fast_convert and \ + (first_conversion != "f16" or first_conversion != "bf16" or first_conversion != "f32"): + + pass + # Latest llama.cpp works for all models for q8_0! + + # logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.") + # first_conversion = "f16" + pass + + # Check if bfloat16 is supported + if first_conversion == "bf16" and not torch.cuda.is_bf16_supported(): + logger.warning( + "Unsloth: Cannot convert to bf16 GGUF since your computer doesn't support it.\n"\ + "We shall switch instead to f16." + ) first_conversion = "f16" pass @@ -975,6 +1010,7 @@ def save_to_gguf( vocab_type = "bpe" pass + # convert.py is deprecated! use_fast_convert = False if use_fast_convert: command = f"python llama.cpp/convert.py {model_directory} "\ @@ -1281,12 +1317,44 @@ def upload_to_huggingface( pass +def fix_tokenizer_bos_token(tokenizer): + # Check if BOS added already, then warn + fix_bos_token = False + chat_template = getattr(tokenizer, "chat_template", None) + + if (tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None)): + if chat_template is not None and \ + ( + tokenizer.bos_token in chat_template or \ + "{bos_token}" in chat_template.replace(" ", "") or \ + "{bos_token+" in chat_template.replace(" ", "") + ): + + fix_bos_token = True + logger.warning( + f"Unsloth: ##### The current model auto adds a BOS token.\n"\ + "Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily." + ) + + # Remove {{bos_token}} + new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}", "", chat_template) + # Remove {{bos_token + + new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}", "", new_chat_template) + + tokenizer.chat_template = new_chat_template + + pass + pass + return fix_bos_token, chat_template +pass + + def unsloth_save_pretrained_gguf( self, save_directory : Union[str, os.PathLike], tokenizer = None, quantization_method : str = "fast_quantized", - first_conversion : str = "f16", + first_conversion : str = None, push_to_hub : bool = False, token : Optional[Union[str, bool]] = None, private : Optional[bool] = None, @@ -1344,6 +1412,9 @@ def unsloth_save_pretrained_gguf( del arguments["quantization_method"] del arguments["first_conversion"] + # Fix tokenizer adding an extra BOS token at the front + fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer) + # Non blocking install GGUF first if not os.path.exists("llama.cpp"): @@ -1386,31 +1457,40 @@ def unsloth_save_pretrained_gguf( pass pass + # Use old chat template if the bos is removed + if fix_bos_token: + tokenizer.chat_template = old_chat_template + pass + for _ in range(3): gc.collect() - model_type = self.config.model_type - is_sentencepiece_model = check_if_sentencepiece_model(self) - - # Check if BOS added already, then warn - print_bos_token_message = False - if (tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None)): - chat_template = getattr(tokenizer, "chat_template", None) - if chat_template is not None and \ - (tokenizer.bos_token in chat_template or "{bos_token}" in chat_template.replace(" ", "")): - print_bos_token_message = True - logger.warning( - f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\ - "Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template." - ) - pass + model_dtype = self.config.torch_dtype + model_type = self.config.model_type + if type(model_dtype) is str: + assert(model_dtype == "float16" or model_dtype == "bfloat16") + elif model_dtype == torch.float16: + model_dtype = "float16" + elif model_dtype == torch.bfloat16: + model_dtype = "bfloat16" + else: + raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16") pass + is_sentencepiece_model = check_if_sentencepiece_model(self) + # Save to GGUF - file_location = save_to_gguf(model_type, is_sentencepiece_model, + file_location = save_to_gguf(model_type, model_dtype, is_sentencepiece_model, new_save_directory, quantization_method, first_conversion, makefile, ) + if fix_bos_token: + logger.warning( + f"Unsloth: ##### The current model auto adds a BOS token.\n"\ + "Unsloth: ##### We removed in GGUF's chat template for you." + ) + pass + if push_to_hub: print("Unsloth: Uploading GGUF to Huggingface Hub...") username = upload_to_huggingface( @@ -1422,13 +1502,6 @@ def unsloth_save_pretrained_gguf( new_save_directory.lstrip('/.') print(f"Saved GGUF to https://huggingface.co/{link}") pass - - if print_bos_token_message: - logger.warning( - f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\ - "Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template." - ) - pass pass @@ -1437,7 +1510,7 @@ def unsloth_push_to_hub_gguf( repo_id : str, tokenizer = None, quantization_method : str = "fast_quantized", - first_conversion : str = "f16", + first_conversion : str = None, use_temp_dir : Optional[bool] = None, commit_message : Optional[str] = "Trained with Unsloth", private : Optional[bool] = None, @@ -1490,6 +1563,9 @@ def unsloth_push_to_hub_gguf( del arguments["quantization_method"] del arguments["first_conversion"] + # Fix tokenizer adding an extra BOS token at the front + fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer) + # Non blocking install GGUF first if not os.path.exists("llama.cpp"): @@ -1532,28 +1608,30 @@ def unsloth_push_to_hub_gguf( pass pass + # Use old chat template if the bos is removed + if fix_bos_token: + tokenizer.chat_template = old_chat_template + pass + for _ in range(3): gc.collect() - model_type = self.config.model_type - is_sentencepiece_model = check_if_sentencepiece_model(self) - - # Check if BOS added already, then warn - print_bos_token_message = False - if (tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None)): - chat_template = getattr(tokenizer, "chat_template", None) - if chat_template is not None and \ - (tokenizer.bos_token in chat_template or "{bos_token}" in chat_template.replace(" ", "")): - print_bos_token_message = True - logger.warning( - f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\ - "Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template." - ) - pass + model_dtype = self.config.torch_dtype + model_type = self.config.model_type + if type(model_dtype) is str: + assert(model_dtype == "float16" or model_dtype == "bfloat16") + elif model_dtype == torch.float16: + model_dtype = "float16" + elif model_dtype == torch.bfloat16: + model_dtype = "bfloat16" + else: + raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16") pass + is_sentencepiece_model = check_if_sentencepiece_model(self) + # Save to GGUF - file_location = save_to_gguf(model_type, is_sentencepiece_model, + file_location = save_to_gguf(model_type, model_dtype, is_sentencepiece_model, new_save_directory, quantization_method, first_conversion, makefile, ) @@ -1568,10 +1646,10 @@ def unsloth_push_to_hub_gguf( print(f"Saved GGUF to https://huggingface.co/{link}") - if print_bos_token_message: + if fix_bos_token: logger.warning( - f"Unsloth: ##### The current model type of {model_type} auto adds a BOS token.\n"\ - "Unsloth: ##### If you're using Ollama or GGUF etc, do not add a BOS in the chat template." + f"Unsloth: ##### The current model auto adds a BOS token.\n"\ + "Unsloth: ##### We removed in GGUF's chat template for you." ) pass pass @@ -1579,7 +1657,6 @@ pass def patch_saving_functions(model): import inspect - import re import types from typing import Callable, Optional, Union, List diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 6afea68057..f10b2c0a47 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -185,6 +185,111 @@ def convert_to_fast_tokenizer( pass +# Check Mistral chat template without BOS / EOS +mistral_template = \ + "{% if messages[0]['role'] == 'system' %}"\ + "{% if messages[1]['role'] == 'user' %}"\ + "{{ '[INST] ' + messages[0]['content'] + ' ' + messages[1]['content'] + ' [/INST]' }}"\ + "{% set loop_messages = messages[2:] %}"\ + "{% else %}"\ + "{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}"\ + "{% set loop_messages = messages[1:] %}"\ + "{% endif %}"\ + "{% else %}"\ + "{% set loop_messages = messages %}"\ + "{% endif %}"\ + "{% for message in loop_messages %}"\ + "{% if message['role'] == 'user' %}"\ + "{{ '[INST] ' + message['content'] + ' [/INST]' }}"\ + "{% elif message['role'] == 'assistant' %}"\ + "{{ message['content'] }}"\ + "{% else %}"\ + "{{ raise_exception('Only user and assistant roles are supported!') }}"\ + "{% endif %}"\ + "{% endfor %}" +pass + +# Check Llama chat template without BOS / EOS +llama_template = \ + "{% if messages[0]['role'] == 'system' %}"\ + "{% if messages[1]['role'] == 'user' %}"\ + "{{ '[INST] <>\n' + messages[0]['content'] + '\n<>\n\n' + messages[1]['content'] + ' [/INST]' }}"\ + "{% set loop_messages = messages[2:] %}"\ + "{% else %}"\ + "{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}"\ + "{% set loop_messages = messages[1:] %}"\ + "{% endif %}"\ + "{% else %}"\ + "{% set loop_messages = messages %}"\ + "{% endif %}"\ + "{% for message in loop_messages %}"\ + "{% if message['role'] == 'user' %}"\ + "{{ '[INST] ' + message['content'].strip() + ' [/INST]' }}"\ + "{% elif message['role'] == 'assistant' %}"\ + "{{ ' ' + message['content'].strip() + ' ' }}"\ + "{% else %}"\ + "{{ raise_exception('Only user and assistant roles are supported!') }}"\ + "{% endif %}"\ + "{% endfor %}" +pass + + +def select_correct_slow_tokenizer( + tokenizer_name, + model_max_length = None, + padding_side = "right", + token = None, + trust_remote_code = False, + cache_dir = "huggingface_tokenizers_cache", +): + """ + Returns 'correct' tokenizer by checking if the chat templates are + actually tokenized correctly. + """ + messages = [ + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "It's 4."}, + ] + + settings = ( + (False, False, True,), + (False, True, True,), + (True, False, True,), + (True, False, False,), + ) + + for (use_fast, legacy, from_slow,) in settings: + # Default as mentioned by Arthur from HF: + slow_tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, + model_max_length = model_max_length, + padding_side = padding_side, + token = token, + trust_remote_code = trust_remote_code, + # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373 + use_fast = use_fast, + legacy = legacy, + from_slow = from_slow, + cache_dir = cache_dir, + ) + slow_tokenizer_chat_template = slow_tokenizer.chat_template + + slow_tokenizer.chat_template = llama_template + result1 = slow_tokenizer.decode(slow_tokenizer.apply_chat_template(messages)) + slow_tokenizer.chat_template = mistral_template + result2 = slow_tokenizer.decode(slow_tokenizer.apply_chat_template(messages)) + + # If 2 spaces seen, normally wrong! + if " "*2 not in result1 and " "*2 not in result2: + slow_tokenizer.chat_template = slow_tokenizer_chat_template + return slow_tokenizer + pass + pass + # Return fast version as default + return slow_tokenizer +pass + + def assert_same_tokenization(slow_tokenizer, fast_tokenizer): # Get eos_token, bos_token etc dir_names = dir(slow_tokenizer) @@ -193,19 +298,64 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer): if x.endswith("_token") and x.count("_") == 1 ))) all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens)) + + # Check if chat template is enabled! + check_chat_template1 = True + check_chat_template2 = True + check_chat_template3 = True + slow_chat_template = getattr(slow_tokenizer, "chat_template", None) + fast_chat_template = getattr(fast_tokenizer, "chat_template", None) + messages = [ + {"role": "user", "content": " What is 2+2? "}, + {"role": "assistant", "content": " It's 4. "}, + ] + # Check the tokenizer's own chat template + if slow_chat_template is not None and fast_chat_template is not None: + check_chat_template1 = \ + slow_tokenizer.apply_chat_template(messages) == \ + fast_tokenizer.apply_chat_template(messages) + pass + + # Check Mistral chat template without BOS / EOS + slow_tokenizer.chat_template = mistral_template + fast_tokenizer.chat_template = mistral_template + check_chat_template2 = \ + slow_tokenizer.apply_chat_template(messages) == \ + fast_tokenizer.apply_chat_template(messages) + pass + + # Check Llama chat template without BOS / EOS + slow_tokenizer.chat_template = llama_template + fast_tokenizer.chat_template = llama_template + check_chat_template3 = \ + slow_tokenizer.apply_chat_template(messages) == \ + fast_tokenizer.apply_chat_template(messages) + pass + + # Combine them all and revert chat templates + check_chat_template = check_chat_template1 and check_chat_template2 and check_chat_template3 + slow_tokenizer.chat_template = slow_chat_template + fast_tokenizer.chat_template = fast_chat_template + + # Try special tokens try: string = "\n".join(all_special_tokens) + \ "A quick brown fox jumps over the lazy dog!!\n\nHi\n\n" + \ "".join(all_special_tokens) - return slow_tokenizer(string).input_ids == fast_tokenizer(string).input_ids + check_special_tokens = \ + slow_tokenizer(string).input_ids == \ + fast_tokenizer(string).input_ids + + return check_chat_template and check_special_tokens except: # For eg see https://github.com/unslothai/unsloth/issues/292 # Sometimes tokenizer has weird tokens, causing a combined tokenization to fail. # [TODO] We temporarily disable this for CodeLlama tokenizers if slow_tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING: - return True + return check_chat_template else: return False + pass pass @@ -358,17 +508,13 @@ def load_correct_tokenizer( # Mainly to solve Deepseek models with no tokenizer.model file slow_tokenizer = None try: - slow_tokenizer = AutoTokenizer.from_pretrained( + slow_tokenizer = select_correct_slow_tokenizer( tokenizer_name, - model_max_length = model_max_length, - padding_side = padding_side, - token = token, + model_max_length = model_max_length, + padding_side = padding_side, + token = token, trust_remote_code = trust_remote_code, - # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373 - use_fast = False, - legacy = False, - from_slow = True, - cache_dir = cache_dir, + cache_dir = cache_dir, ) except: pass @@ -397,6 +543,7 @@ def load_correct_tokenizer( if assert_same_tokenization(slow_tokenizer, fast_tokenizer): return fast_tokenizer else: + logger.warning(f"Unsloth: Will load {tokenizer_name} as a legacy tokenizer.") return convert_to_fast_tokenizer(slow_tokenizer) pass else: @@ -574,6 +721,8 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16): # Get set and actual tokens where_untrained = where_untrained.tolist() if len(where_untrained) == 0: return + + # Remove untrained indices where it's longer where_untrained_set = frozenset(where_untrained) actual_bad_tokens = tokenizer.convert_ids_to_tokens(where_untrained) @@ -854,11 +1003,13 @@ def patch_sft_trainer_tokenizer(): " )\n"\ "pass\n"\ "n_devices = torch.cuda.device_count()\n"\ - "more_than = 0\n"\ - "for j in range(n_devices):\n"\ - " vram = torch.cuda.max_memory_reserved(torch.cuda.device(j)) / 1024 / 1024 / 1024\n"\ - " more_than += (vram > 4)\n"\ - "if more_than > 1: raise RuntimeError('Error: More than 1 GPUs have a lot of VRAM usage.')\n"\ + "import subprocess, re\n"\ + "output = subprocess.check_output(\n"\ + " 'nvidia-smi --query-gpu=memory.used --format=csv', shell = True)\n"\ + "output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output)\n"\ + "output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output)\n"\ + "if output > 1: raise RuntimeError(\n"\ + " 'Error: More than 1 GPUs have a lot of VRAM usage. Please obtain a commercial license.')\n"\ "for _ in range(3):\n"\ " gc.collect()\n"\ " torch.cuda.empty_cache()\n"\