diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 7f00c7741c..7962c27f91 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -883,7 +883,7 @@ PARAMETER min_p 0.1 ''' qwen25_template_eos_token = "eos_token" -qwen25_default_system_message = "You are Qwen, created by Alibaba Cloud. You are a helpful assistant." +qwen25_default_system_message = "You are Qwen, created by Alibaba Cloud. You are a helpful assistant." CHAT_TEMPLATES["qwen-2.5"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,) DEFAULT_SYSTEM_MESSAGE["qwen-2.5"] = qwen25_default_system_message # No system message in Qwen 2.5 @@ -1241,7 +1241,8 @@ gemma3n_template = \ # Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802 gemma3n_ollama = \ ''' -{{- range $i, $_ := .Messages }} +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $i, $_ := .Messages }} {{- $last := eq (len (slice $.Messages $i)) 1 }} {{- if or (eq .Role "user") (eq .Role "system") }}user {{ .Content }} @@ -1251,7 +1252,7 @@ gemma3n_ollama = \ {{ .Content }}{{ if not $last }} {{ end }} {{- end }} -{{- end }} +{{- end }}""" ''' gemma3n_template_eos_token = "" @@ -1617,7 +1618,9 @@ gptoss_template = \ # Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802 gptoss_ollama = \ -'''<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +''' +FROM {__FILE_LOCATION__} +TEMPLATE """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. Knowledge cutoff: 2024-06 Current date: {{ currentDate }} {{- if and .IsThinkSet .Think (ne .ThinkLevel "") }} @@ -1789,7 +1792,11 @@ type {{ .Function.Name }} = () => any; {{- end -}} {{- if not (or $prefillingContent $prefillingThinkingOnly) -}} <|start|>assistant -{{- end -}}''' +{{- end -}}""" +PARAMETER temperature 1.0 +PARAMETER top_k 0 +PARAMETER top_p 1.0 +''' gptoss_template_template_eos_token = "<|return|>" CHAT_TEMPLATES["gpt-oss"] = (gptoss_template, gptoss_template_template_eos_token, False, gptoss_ollama,) @@ -1891,7 +1898,8 @@ qwen3_instruct_template = \ # Ollama from https://ollama.com/library/qwen3/blobs/53e4ea15e8f5 qwen3_ollama = \ ''' - +FROM {__FILE_LOCATION__} +TEMPLATE """ {{- $lastUserIdx := -1 -}} {{- range $idx, $msg := .Messages -}} {{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} @@ -1942,6 +1950,7 @@ For each function call, return a json object with function name and arguments wi {{- if and (ne .Role "assistant") $last }}<|im_start|>assistant {{ end }} {{- end }} +""" ''' qwen3_template_eos_token = "<|im_end|>" @@ -2044,9 +2053,82 @@ DEFAULT_SYSTEM_MESSAGE["qwen3-thinking"] = None # No system message in Qwen3 pass +# =========================================== Liquid-LFM2 +liquid_lfm2_template = \ +''' +{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %}''' + +liquid_lfm2_template_eos_token = "<|im_end|>" +CHAT_TEMPLATES["lfm-2"] = (liquid_lfm2_template, liquid_lfm2_template_eos_token, False, None) +DEFAULT_SYSTEM_MESSAGE["lfm-2"] = None # No system message in Phi-3 + +pass + +# =========================================== Starling-LM + +starling_template = \ +"""{{ bos_token }} +{%- for message in messages %} + {{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>' }} +{%- endfor %} +{%- if add_generation_prompt %} + {{ 'GPT4 Correct Assistant:' }} +{%- endif %}""" + +# Ollama from https://ollama.com/library/starling-lm:7b/blobs/4b21bfc435b4 +starling_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}GPT4 Correct System: {{ .System }}<|end_of_turn|> +{{ end }}{{ if .Prompt }}GPT4 Correct User: {{ .Prompt }}<|end_of_turn|> +{{ end }}GPT4 Correct Assistant: {{ .Response }}<|end_of_turn|>""" +PARAMETER stop "<|end_of_turn|>" +PARAMETER stop "GPT4 Correct User:" +PARAMETER stop "GPT4 Correct Assistant:" +PARAMETER stop "GPT4 Correct System:" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +starling_template_eos_token = "<|end_of_turn|>" +CHAT_TEMPLATES["starling"] = (starling_template, starling_template_eos_token, False, starling_ollama) +DEFAULT_SYSTEM_MESSAGE["starling"] = None + +pass + +# =========================================== Yi-chat + +yi_chat_template = \ +""" +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +""" + +# Ollama from https://ollama.com/library/yi:34b-chat/blobs/62fbfd9ed093 +yi_chat_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ .Response }}<|im_end|>""" +''' + +yi_chat_template_eos_token = "<|endoftext|>" +CHAT_TEMPLATES["yi-chat"] = (yi_chat_template, yi_chat_template_eos_token, False, yi_chat_ollama) +DEFAULT_SYSTEM_MESSAGE["yi-chat"] = None +pass + def _change_system_message(template: str, type_chat_template: str, system_message: str = None): system_message_pattern = r"\{system_message\}" - + # For predefined templates, check if default system message exists default_system_message = DEFAULT_SYSTEM_MESSAGE.get(f"{type_chat_template}", None) if default_system_message is None: @@ -2058,24 +2140,24 @@ def _change_system_message(template: str, type_chat_template: str, system_messag ) return template, system_message pass - + # For custom templates if type_chat_template is None: has_placeholder = re.search(system_message_pattern, template) is not None - + if has_placeholder: if system_message is None: raise ValueError("Unsloth: You need to provide a system message for custom templates.") new_template = re.sub(system_message_pattern, system_message, template) return new_template, system_message - + return template, system_message pass - + # For predefined templates with default system message message_to_use = system_message if system_message is not None else default_system_message new_template = re.sub(system_message_pattern, message_to_use, template) - + return new_template, message_to_use pass @@ -2113,7 +2195,7 @@ def get_chat_template( same_padding_token = False type_chat_template = None - + if type(chat_template) in (list, tuple,): # For changing system message later # Since it's not supported yet, we will raise an error first! @@ -2573,7 +2655,7 @@ chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> {INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> {OUTPUT}<|eot_id|>""", - + default_system_message = \ "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.", @@ -2819,7 +2901,7 @@ extra_eos_tokens = None, partial_system = partial_system.replace(tokenizer.bos_token, "", 1) system_part = system_part .replace(tokenizer.bos_token, "", 1) pass - + partial_system = \ "{% if messages[0]['role'] == 'system' %}"\ "{{ " + partial_system + " }}"\ @@ -2891,10 +2973,10 @@ def test_construct_chat_template(): {INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> {OUTPUT}<|eot_id|>""" - + default_system_message = \ "Below are some instructions that describe some tasks. Write responses that appropriately complete each request." - + extra_eos_tokens = None modelfile, jinja_template, _, _ = construct_chat_template( @@ -2936,12 +3018,12 @@ chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> {INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> {OUTPUT}<|eot_id|>""", - + default_system_message = \ "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.", - + extra_eos_tokens = None, - + ): """ Creates a Ollama modelfile and a HF Jinja template from a custom @@ -3149,7 +3231,7 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf") prompt = remove_special_tokens(tokenizer, prompt) prompts.append(prompt) pass - + for prompt in prompts: command = f"./llama.cpp/llama-cli -m {gguf_model} -n 0 --temp 0.0 --verbose-prompt "\ f"--check-tensors -p '{prompt}'" diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 600396ed46..346ec3403a 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -114,7 +114,7 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/gemma-1.1-7b-it", "google/gemma-1.1-7b-it", ), - "unsloth/Starling-LM-7B-beta-bnb-4bit" : ( + "unsloth/Starling-LM-7B-beta" : ( "unsloth/Starling-LM-7B-beta", "Nexusflow/Starling-LM-7B-beta", ), diff --git a/unsloth/ollama_template_mappers.py b/unsloth/ollama_template_mappers.py new file mode 100644 index 0000000000..1ac95f3a36 --- /dev/null +++ b/unsloth/ollama_template_mappers.py @@ -0,0 +1,2266 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = [ + "OLLAMA_TEMPLATES", + "OLLAMA_TEMPLATE_TO_MODEL_MAPPER", + "MODEL_TO_OLLAMA_TEMPLATE_MAPPER", +] + +OLLAMA_TEMPLATES = {} + +# =========================================== Unsloth + +unsloth_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} +{{ end }}{{ if .Prompt }}>>> User: {{ .Prompt }} +{{ end }}>>> Assistant: {{ .Response }}{__EOS_TOKEN__} +""" +PARAMETER stop "{__EOS_TOKEN__}" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +SYSTEM """You are a helpful assistant to the user""" +''' + +OLLAMA_TEMPLATES["unsloth"] = unsloth_ollama +pass + +# =========================================== Zephyr + +zephyr_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|system|> +{{ .System }}{__EOS_TOKEN__} +{{ end }}{{ if .Prompt }}<|user|> +{{ .Prompt }}{__EOS_TOKEN__} +{{ end }}<|assistant|> +{{ .Response }}{__EOS_TOKEN__} +""" +PARAMETER stop "{__EOS_TOKEN__}" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["zephyr"] = zephyr_ollama +pass + +# =========================================== ChatML +chatml_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ .Response }}<|im_end|> +""" +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["chatml"] = chatml_ollama +pass + +# =========================================== Mistral-1 +# Ollama from https://www.ollama.com/library/mistral +# Mistral v0.1 https://ollama.com/library/mistral:v0.1/blobs/22e1b2e8dc2f +# Mistral v0.2 https://ollama.com/library/mistral:v0.2/blobs/e6836092461f +mistral_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """[INST] {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} [/INST]""" +PARAMETER stop "[INST]" +PARAMETER stop "[/INST]" +''' + +# mistral:v0.3 https://ollama.com/library/mistral:v0.3/blobs/1ff5b64b61b9 +# mistral-large https://ollama.com/library/mistral-large:latest/blobs/96adabcf2c08 +mistral_v03_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- if .Messages }} +{{- range $index, $_ := .Messages }} +{{- if eq .Role "user" }} +{{- if and (eq (len (slice $.Messages $index)) 1) $.Tools }}[AVAILABLE_TOOLS] {{ $.Tools }}[/AVAILABLE_TOOLS] +{{- end }}[INST] {{ if and $.System (eq (len (slice $.Messages $index)) 1) }}{{ $.System }} + +{{ end }}{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} +{{- if .Content }}{{ .Content }} +{{- else if .ToolCalls }}[TOOL_CALLS] [ +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{- end }}] +{{- end }} +{{- else if eq .Role "tool" }}[TOOL_RESULTS] {"content": {{ .Content }}} [/TOOL_RESULTS] +{{- end }} +{{- end }} +{{- else }}[INST] {{ if .System }}{{ .System }} + +{{ end }}{{ .Prompt }}[/INST] +{{- end }}{{ .Response }} +{{- if .Response }} +{{- end }}""" +PARAMETER stop "[INST]" +PARAMETER stop "[/INST]" +PARAMETER stop "" +''' + +# Mistral-small https://ollama.com/library/mistral-small:latest/blobs/6db27cd4e277 +mistral_small_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $index, $_ := .Messages }} +{{- if eq .Role "system" }}[SYSTEM_PROMPT]{{ .Content }}[/SYSTEM_PROMPT] +{{- else if eq .Role "user" }} +{{- if and (le (len (slice $.Messages $index)) 2) $.Tools }}[AVAILABLE_TOOLS]{{ $.Tools }}[/AVAILABLE_TOOLS] +{{- end }}[INST]{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} +{{- if .Content }}{{ .Content }} +{{- if not (eq (len (slice $.Messages $index)) 1) }} +{{- end }} +{{- else if .ToolCalls }}[TOOL_CALLS][ +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{- end }}] +{{- end }} +{{- else if eq .Role "tool" }}[TOOL_RESULTS]{"content": {{ .Content }}}[/TOOL_RESULTS] +{{- end }} +{{- end }}""" +PARAMETER temperature 0.15 +SYSTEM """You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris. Your knowledge base was last updated on 2023-10-01. When you're not sure about some information, you say that you don't have the information and don't make up anything. If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. "What are some good restaurants around me?" => "Where are you?" or "When is the next flight to Tokyo" => "Where do you travel from?")""" +''' + +# mistral-small-3.1 https://ollama.com/library/mistral-small3.1:latest/blobs/6db27cd4e277 +mistral_small_31_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $index, $_ := .Messages }} +{{- if eq .Role "system" }}[SYSTEM_PROMPT]{{ .Content }}[/SYSTEM_PROMPT] +{{- else if eq .Role "user" }} +{{- if and (le (len (slice $.Messages $index)) 2) $.Tools }}[AVAILABLE_TOOLS]{{ $.Tools }}[/AVAILABLE_TOOLS] +{{- end }}[INST]{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} +{{- if .Content }}{{ .Content }} +{{- if not (eq (len (slice $.Messages $index)) 1) }} +{{- end }} +{{- else if .ToolCalls }}[TOOL_CALLS][ +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{- end }}] +{{- end }} +{{- else if eq .Role "tool" }}[TOOL_RESULTS]{"content": {{ .Content }}}[/TOOL_RESULTS] +{{- end }} +{{- end }}""" +PARAMETER num_ctx 4096 +SYSTEM """You are Mistral Small 3.1, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris. +You power an AI assistant called Le Chat. +Your knowledge base was last updated on 2023-10-01. + +When you're not sure about some information, you say that you don't have the information and don't make up anything. +If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. "What are some good restaurants around me?" => "Where are you?" or "When is the next flight to Tokyo" => "Where do you travel from?"). +You are always very attentive to dates, in particular you try to resolve dates (e.g. "yesterday" is {yesterday}) and when asked about information at specific dates, you discard information that is at another date. +You follow these instructions in all languages, and always respond to the user in the language they use or request. +Next sections describe the capabilities that you have. + +# WEB BROWSING INSTRUCTIONS + +You cannot perform any web search or access internet to open URLs, links etc. If it seems like the user is expecting you to do so, you clarify the situation and ask the user to copy paste the text directly in the chat. + +# MULTI-MODAL INSTRUCTIONS + +You have the ability to read images, but you cannot generate images. You also cannot transcribe audio files or videos. +You cannot read nor transcribe audio files or videos.""" +''' + +# mistral-small-3.2 https://ollama.com/library/mistral-small3.2:latest/blobs/706c4d1164f7 +mistral_small_32_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $index, $_ := .Messages }} +{{- if eq .Role "system" }}[SYSTEM_PROMPT]{{ .Content }}[/SYSTEM_PROMPT] +{{- else if eq .Role "user" }} +{{- if and (le (len (slice $.Messages $index)) 2) $.Tools }}[AVAILABLE_TOOLS]{{ $.Tools }}[/AVAILABLE_TOOLS] +{{- end }}[INST]{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} +{{- if .Content }}{{ .Content }} +{{- if not (eq (len (slice $.Messages $index)) 1) }} +{{- end }} +{{- else if .ToolCalls }} +{{- range $i, $_ := .ToolCalls }}[TOOL_CALLS]{{ .Function.Name }}[CALL_ID]{{ $i }}[ARGS]{{ .Function.Arguments }} +{{- end }} +{{- end }} +{{- else if eq .Role "tool" }}[TOOL_RESULTS]{"content": {{ .Content }}}[/TOOL_RESULTS] +{{- end }} +{{- end }}""" +PARAMETER temperature 0.15 +SYSTEM """You are Mistral Small 3.2, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris. +You power an AI assistant called Le Chat. +Your knowledge base was last updated on 2023-10-01. + +When you're not sure about some information or when the user's request requires up-to-date or specific data, you must use the available tools to fetch the information. Do not hesitate to use tools whenever they can provide a more accurate or complete response. If no relevant tools are available, then clearly state that you don't have the information and avoid making up anything. +If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. "What are some good restaurants around me?" => "Where are you?" or "When is the next flight to Tokyo" => "Where do you travel from?"). +You are always very attentive to dates, in particular you try to resolve dates and when asked about information at specific dates, you discard information that is at another date. +You follow these instructions in all languages, and always respond to the user in the language they use or request. +Next sections describe the capabilities that you have. + +# WEB BROWSING INSTRUCTIONS + +You cannot perform any web search or access internet to open URLs, links etc. If it seems like the user is expecting you to do so, you clarify the situation and ask the user to copy paste the text directly in the chat. + +# MULTI-MODAL INSTRUCTIONS + +You have the ability to read images, but you cannot generate images. You also cannot transcribe audio files or videos. +You cannot read nor transcribe audio files or videos. + +TOOL CALLING INSTRUCTIONS + +You may have access to tools that you can use to fetch information or perform actions. You must use these tools in the following situations: + +1. When the request requires up-to-date information. +2. When the request requires specific data that you do not have in your knowledge base. +3. When the request involves actions that you cannot perform without tools. + +Always prioritize using tools to provide the most accurate and helpful response. If tools are not available, inform the user that you cannot perform the requested action at the moment.""" +''' + + +# https://ollama.com/library/mixtral:latest/blobs/53d74de0d84c +mixtral_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """[INST] {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} [/INST] {{ .Response }}""" +PARAMETER stop "[INST]" +PARAMETER stop "[/INST]" +''' + +# https://registry.ollama.ai/library/mistral-nemo:latest/blobs/438402ddac75 +mistral_nemo_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """ +{{- range $i, $_ := .Messages }} +{{- if eq .Role "user" }} +{{- if and $.Tools (le (len (slice $.Messages $i)) 2) }}[AVAILABLE_TOOLS]{{ $.Tools }}[/AVAILABLE_TOOLS] +{{- end }}[INST]{{ if and $.System (eq (len (slice $.Messages $i)) 1) }}{{ $.System }} + +{{ end }}{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} +{{- if .Content }} {{ .Content }}{{ if not (eq (len (slice $.Messages $i)) 1) }}{{ end }} +{{- else if .ToolCalls }}[TOOL_CALLS][ +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{- end }}] +{{- end }} +{{- else if eq .Role "tool" }}[TOOL_RESULTS]{"content": {{ .Content }}}[/TOOL_RESULTS] +{{- end }} +{{- end }}""" +PARAMETER stop "[INST]" +PARAMETER stop "[/INST]" +''' + +# https://ollama.com/library/codestral:latest/blobs/51707752a87c +codestral_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """ +{{- if .Suffix }}[SUFFIX]{{ .Suffix }}[PREFIX] {{ .Prompt }} +{{- else if .Messages }} +{{- range $index, $_ := .Messages }} +{{- if eq .Role "user" }}[INST] {{ if and $.System (eq (len (slice $.Messages $index)) 1) }}{{ $.System }} + +{{ end }}{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} {{ .Content }} +{{- end }} +{{- end }} +{{- else }}[INST] {{ if .System }}{{ .System }} + +{{ end }}{{ .Prompt }} [/INST] +{{- end }} {{ .Response }} +{{- if .Response }} +{{- end }} +""" +PARAMETER stop "[INST]" +PARAMETER stop "[/INST]" +PARAMETER stop "[PREFIX]" +PARAMETER stop "[MIDDLE]" +PARAMETER stop "[SUFFIX]" +''' + +# https://ollama.com/library/devstral:latest/blobs/ea9ec42474e0 +devstral_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- $lastUserIndex := -1 }} +{{- range $index, $_ := .Messages }} +{{- if eq .Role "user" }}{{ $lastUserIndex = $index }}{{ end }} +{{- end }} +{{- range $index, $_ := .Messages }} +{{- if eq .Role "system" }}[SYSTEM_PROMPT]{{ .Content }}[/SYSTEM_PROMPT] +{{- else if eq .Role "user" }} +{{- if and (eq $lastUserIndex $index) $.Tools }}[AVAILABLE_TOOLS]{{ $.Tools }}[/AVAILABLE_TOOLS] +{{- end }}[INST]{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} +{{- if .Content }}{{ .Content }} +{{- if not (eq (len (slice $.Messages $index)) 1) }} +{{- end }} +{{- else if .ToolCalls }}[TOOL_CALLS][ +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{- end }}] +{{- end }} +{{- else if eq .Role "tool" }}[TOOL_RESULTS]{"content": {{ .Content }}}[/TOOL_RESULTS] +{{- end }} +{{- end }}""" +SYSTEM """You are Devstral, a helpful agentic model trained by Mistral AI and using the OpenHands scaffold. You can interact with a computer to solve tasks. + + +Your primary role is to assist users by executing commands, modifying code, and solving technical problems effectively. You should be thorough, methodical, and prioritize quality over speed. +* If the user asks a question, like "why is X happening", don't try to fix the problem. Just give an answer to the question. + + + +* Each action you take is somewhat expensive. Wherever possible, combine multiple actions into a single action, e.g. combine multiple bash commands into one, using sed and grep to edit/view multiple files at once. +* When exploring the codebase, use efficient tools like find, grep, and git commands with appropriate filters to minimize unnecessary operations. + + + +* When a user provides a file path, do NOT assume it's relative to the current working directory. First explore the file system to locate the file before working on it. +* If asked to edit a file, edit the file directly, rather than creating a new file with a different filename. +* For global search-and-replace operations, consider using `sed` instead of opening file editors multiple times. + + + +* Write clean, efficient code with minimal comments. Avoid redundancy in comments: Do not repeat information that can be easily inferred from the code itself. +* When implementing solutions, focus on making the minimal changes needed to solve the problem. +* Before implementing any changes, first thoroughly understand the codebase through exploration. +* If you are adding a lot of code to a function or file, consider splitting the function or file into smaller pieces when appropriate. + + + +* When configuring git credentials, use "openhands" as the user.name and "openhands@all-hands.dev" as the user.email by default, unless explicitly instructed otherwise. +* Exercise caution with git operations. Do NOT make potentially dangerous changes (e.g., pushing to main, deleting repositories) unless explicitly asked to do so. +* When committing changes, use `git status` to see all modified files, and stage all files necessary for the commit. Use `git commit -a` whenever possible. +* Do NOT commit files that typically shouldn't go into version control (e.g., node_modules/, .env files, build directories, cache files, large binaries) unless explicitly instructed by the user. +* If unsure about committing certain files, check for the presence of .gitignore files or ask the user for clarification. + + + +* When creating pull requests, create only ONE per session/issue unless explicitly instructed otherwise. +* When working with an existing PR, update it with new commits rather than creating additional PRs for the same issue. +* When updating a PR, preserve the original PR title and purpose, updating description only when necessary. + + + +1. EXPLORATION: Thoroughly explore relevant files and understand the context before proposing solutions +2. ANALYSIS: Consider multiple approaches and select the most promising one +3. TESTING: + * For bug fixes: Create tests to verify issues before implementing fixes + * For new features: Consider test-driven development when appropriate + * If the repository lacks testing infrastructure and implementing tests would require extensive setup, consult with the user before investing time in building testing infrastructure + * If the environment is not set up to run tests, consult with the user first before investing time to install all dependencies +4. IMPLEMENTATION: Make focused, minimal changes to address the problem +5. VERIFICATION: If the environment is set up to run tests, test your implementation thoroughly, including edge cases. If the environment is not set up to run tests, consult with the user first before investing time to run tests. + + + +* Only use GITHUB_TOKEN and other credentials in ways the user has explicitly requested and would expect. +* Use APIs to work with GitHub or other platforms, unless the user asks otherwise or your task requires browsing. + + + +* When user asks you to run an application, don't stop if the application is not installed. Instead, please install the application and run the command again. +* If you encounter missing dependencies: + 1. First, look around in the repository for existing dependency files (requirements.txt, pyproject.toml, package.json, Gemfile, etc.) + 2. If dependency files exist, use them to install all dependencies at once (e.g., `pip install -r requirements.txt`, `npm install`, etc.) + 3. Only install individual packages directly if no dependency files are found or if only specific packages are needed +* Similarly, if you encounter missing dependencies for essential tools requested by the user, install them when possible. + + + +* If you've made repeated attempts to solve a problem but tests still fail or the user reports it's still broken: + 1. Step back and reflect on 5-7 different possible sources of the problem + 2. Assess the likelihood of each possible cause + 3. Methodically address the most likely causes, starting with the highest probability + 4. Document your reasoning process +* When you run into any major issue while executing a plan from the user, please don't try to directly work around it. Instead, propose a new plan and confirm with the user before proceeding. +""" +''' + +# https://ollama.com/library/magistral:latest/blobs/35f7a1efc383 +magistral_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """ +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1}} +{{- if eq .Role "system" }}[SYSTEM_PROMPT]{{ .Content }}[/SYSTEM_PROMPT] +{{- else if eq .Role "user" }} +{{- if and (le (len (slice $.Messages $i)) 2) $.Tools }}[AVAILABLE_TOOLS]{{ $.Tools }}[/AVAILABLE_TOOLS] +{{- end }}[INST]{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} +{{- if and $.IsThinkSet (and $last .Thinking) -}} + +{{ .Thinking }} + +{{ end }} +{{- if .Content }}{{ .Content }} +{{- end }} +{{- if .ToolCalls }}{{ range $i, $_ := .ToolCalls }}[TOOL_CALLS]{{ .Function.Name }}[CALL_ID]{{ $i }}[ARGS]{{ .Function.Arguments }}{{ end }} +{{- end }} +{{- if not (eq (len (slice $.Messages $i)) 1) }} +{{- end }} +{{- else if eq .Role "tool" }}[TOOL_RESULTS]0[TOOL_CONTENT]{{ .Content }}[/TOOL_RESULTS] +{{- end }} +{{- if and $last (ne .Role "assistant") }}{{ if and $.IsThinkSet (not $.Think) -}} + +{{ end }} +{{- end }} +{{- end }}""" +PARAMETER temperature 0.7 +PARAMETER top_p 0.95 +SYSTEM """A user will ask you to solve a task. You should first draft your thinking process (inner monologue) until you have derived the final answer. Afterwards, write a self-contained summary of your thoughts (i.e. your summary should be succinct but contain all the critical steps you needed to reach the conclusion). You should use Markdown and Latex to format your response. Write both your thoughts and summary in the same language as the task posed by the user. + +Your thinking process must follow the template below: + +Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct answer. + + +Here, provide a concise summary that reflects your reasoning and presents a clear final answer to the user. + +Problem:""" +''' + +OLLAMA_TEMPLATES["mistral"] = mistral_ollama +OLLAMA_TEMPLATES["mistral-v03"] = mistral_v03_ollama +OLLAMA_TEMPLATES["mistral-small"] = mistral_small_ollama +OLLAMA_TEMPLATES["mistral-small-31"] = mistral_small_31_ollama +OLLAMA_TEMPLATES["mistral-small-32"] = mistral_small_32_ollama +OLLAMA_TEMPLATES["mixtral"] = mixtral_ollama +OLLAMA_TEMPLATES["mistral-nemo"] = mistral_nemo_ollama +OLLAMA_TEMPLATES["devstral"] = devstral_ollama +OLLAMA_TEMPLATES["magistral"] = magistral_ollama +OLLAMA_TEMPLATES["codestral"] = codestral_ollama + +pass + +# =========================================== Llama-2 +# Ollama from https://www.ollama.com/library/llama3 +llama_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """[INST] <>{{ .System }}<> + +{{ .Prompt }} [/INST]""" +PARAMETER stop "{__EOS_TOKEN__}" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["llama"] =llama_ollama +pass + +# =========================================== Vicuna +# Ollama from https://www.ollama.com/library/vicuna +vicuna_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}USER: {{ .Prompt }} {{ end }}ASSISTANT: {{ .Response }} {__EOS_TOKEN__}""" +PARAMETER stop "{__EOS_TOKEN__}" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["vicuna"] = vicuna_ollama +pass + +# =========================================== Vicuna Old +vicuna_old_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} +{{ end }}{{ if .Prompt }}### Human: {{ .Prompt }} +{{ end }}### Assistant: {{ .Response }}{__EOS_TOKEN__} +""" +PARAMETER stop "{__EOS_TOKEN__}" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +SYSTEM """A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.""" +''' + +OLLAMA_TEMPLATES["vicuna_old"] = vicuna_old_ollama +OLLAMA_TEMPLATES["vicuna old"] = OLLAMA_TEMPLATES["vicuna_old"] +pass + +# =========================================== Alpaca multi turn +alpaca_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}{{ .System }} + +{{ end }}{{ if .Prompt }}### Instruction: +{{ .Prompt }}{{ end }} + +### Response: +{{ .Response }}{__EOS_TOKEN__} + +""" +PARAMETER stop "{__EOS_TOKEN__}" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +SYSTEM """Below are some instructions that describe some tasks. Write responses that appropriately complete each request.""" +''' + +OLLAMA_TEMPLATES["alpaca"] = alpaca_ollama +pass + +# =========================================== Gemma +# Ollama from https://www.ollama.com/library/gemma +gemma_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """user +{{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} +model +{{ .Response }} +""" +PARAMETER repeat_penalty 1 +PARAMETER stop "" +PARAMETER stop "" +PARAMETER penalize_newline false +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["gemma"] = gemma_ollama +pass + +# =========================================== Gemma with ChatML instead +gemma_chatml_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ .Response }}<|im_end|> +""" +PARAMETER repeat_penalty 1 +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER penalize_newline false +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["gemma_chatml"] = gemma_chatml_ollama +pass + +# =========================================== Gemma 2 +# Same as Gemma 1, but with sliding window attention! +# https://ollama.com/library/gemma2/blobs/6522ca797f47 +gemma2_ollama = gemma_ollama + "PARAMETER num_ctx 4096\n" +OLLAMA_TEMPLATES["gemma2"] = gemma2_ollama + +# =========================================== Gemma 2 with ChatML instead +gemma2_chatml_ollama = gemma_chatml_ollama + "PARAMETER num_ctx 4096\n" +OLLAMA_TEMPLATES["gemma2_chatml"] = gemma2_chatml_ollama +pass + +# =========================================== Llama-3 +# Ollama from https://www.ollama.com/library/llama3 +llama3_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|> + +{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> + +{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> + +{{ .Response }}<|eot_id|>""" +PARAMETER num_keep 24 +PARAMETER stop "<|start_header_id|>" +PARAMETER stop "<|end_header_id|>" +PARAMETER stop "<|eot_id|>" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["llama-3"] = llama3_ollama +OLLAMA_TEMPLATES["llama3"] = llama3_ollama +pass + + +# =========================================== Phi-3 +# Ollama from https://www.ollama.com/library/phi3 +phi3_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|system|> +{{ .System }}<|end|> +{{ end }}{{ if .Prompt }}<|user|> +{{ .Prompt }}<|end|> +{{ end }}<|assistant|> +{{ .Response }}<|end|> +""" +PARAMETER stop "<|end|>" +PARAMETER stop "<|user|>" +PARAMETER stop "<|assistant|>" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["phi-3"] = phi3_ollama +OLLAMA_TEMPLATES["phi-35"] = OLLAMA_TEMPLATES["phi-3"] +OLLAMA_TEMPLATES["phi-3.5"] = OLLAMA_TEMPLATES["phi-3"] +pass + +# =========================================== Llama-3.1 +""" +No trimming in Llama 3.1 Instruct! +Also an extra newline for Cutting Knowledge Date +See https://colab.research.google.com/drive/1Xpqq5xpIgO-B00MQ-UccYMwN2J8QFgBM?usp=sharing + +Also should be + +import datetime +tokenizer.apply_chat_template( + messages, + add_generation_prompt = True, + tokenize = False, + date_string = datetime.today().strftime("%d %B %Y")), +) +""" + +# Ollama from https://ollama.com/library/llama3.1 (needs updating!) +llama31_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .Messages }} +{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|> +{{- if .System }} + +{{ .System }} +{{- end }} +{{- if .Tools }} + +You are a helpful assistant with tool calling capabilities. When you receive a tool call response, use the output to format an answer to the original use question. +{{- end }} +{{- end }}<|eot_id|> +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if eq .Role "user" }}<|start_header_id|>user<|end_header_id|> +{{- if and $.Tools $last }} + +Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. + +Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. + +{{ $.Tools }} +{{- end }} + +{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- else if eq .Role "assistant" }}<|start_header_id|>assistant<|end_header_id|> +{{- if .ToolCalls }} + +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }} +{{- else }} + +{{ .Content }}{{ if not $last }}<|eot_id|>{{ end }} +{{- end }} +{{- else if eq .Role "tool" }}<|start_header_id|>ipython<|end_header_id|> + +{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- end }} +{{- end }} +{{- else }} +{{- if .System }}<|start_header_id|>system<|end_header_id|> + +{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> + +{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }}{{ .Response }}{{ if .Response }}<|eot_id|>{{ end }}""" +PARAMETER stop "<|start_header_id|>" +PARAMETER stop "<|end_header_id|>" +PARAMETER stop "<|eot_id|>" +PARAMETER stop "<|eom_id|>" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +# https://ollama.com/ajindal/llama3.1-storm:8b/blobs/1970553b62f4 +llama_31_storm_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """ +{{ if .Messages }} +{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|> +{{- if .System }} + +{{ .System }} +{{- end }} +{{- if .Tools }} + +You are a function calling AI model. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into function. The user may use the terms function calling or tool use interchangeably. + +Here are the available functions: +{{ json .Tools }} + +For each function call return a json object with function name and arguments within XML tags in the format: +{"tool_name": , "tool_arguments": } +{{- end }} +{{- end }}<|eot_id|> +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if eq .Role "user" }}<|start_header_id|>user<|end_header_id|> + +{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|> +{{ end }} +{{- else if eq .Role "assistant" }}<|start_header_id|>assistant<|end_header_id|> +{{- if .ToolCalls }} + +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }} +{{- else }} + +{{ .Content }}{{ if not $last }}<|eot_id|>{{ end }} +{{- end }} +{{- else if eq .Role "tool" }}<|start_header_id|>ipython<|end_header_id|> + +{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|> +{{ end }} +{{- end }} +{{- end }} +{{- else }} +{{- if .System }}<|start_header_id|>system<|end_header_id|> + +{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> + +{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }}{{ .Response }}{{ if .Response }}<|eot_id|>{{ end }} +""" +PARAMETER stop "<|start_header_id|>" +PARAMETER stop "<|end_header_id|>" +PARAMETER stop "<|eot_id|>" +''' + +# https://ollama.com/library/nemotron:latest/blobs/4863fe3335f3 +llama_31_nemotron_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """<|start_header_id|>system<|end_header_id|> + +{{ if .Tools }}You have access to the following functions. To call a function, please respond with JSON for a function call. Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. + +{{ range .Tools }}{{ . }} + +{{ end }} +{{- end }}{{ .System }}<|eot_id|> +{{- range $i, $_ := .Messages }} +{{- $isLastMessage := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "system" }} +{{- else if eq .Role "assistant" }}<|start_header_id|>assistant<|end_header_id|> + +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }} } +{{- end }} +{{- end }} +{{- if not $isLastMessage }}<|eot_id|> +{{- end }} +{{- else if eq .Role "tool" }}<|start_header_id|>ipython<|end_header_id|> + +{{ .Content }}<|eot_id|> +{{- if $isLastMessage }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- else }}<|start_header_id|>{{ .Role }}<|end_header_id|> + +{{ .Content }}<|eot_id|> +{{- if $isLastMessage }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- end }} +{{- end }} +""" +PARAMETER stop "<|start_header_id|>" +PARAMETER stop "<|end_header_id|>" +PARAMETER stop "<|eot_id|>" +''' + +# https://ollama.com/library/llama3.2-vision:latest/blobs/715415638c895a1f8e8c6 +llama_32_vision_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $index, $_ := .Messages }}<|start_header_id|>{{ .Role }}<|end_header_id|> + +{{ .Content }} +{{- if gt (len (slice $.Messages $index)) 1 }}<|eot_id|> +{{- else if ne .Role "assistant" }}<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- end }}""" +PARAMETER temperature 0.6 +PARAMETER top_p 0.9 +''' + +OLLAMA_TEMPLATES["llama-3.1"] = llama31_ollama +OLLAMA_TEMPLATES["llama-31"] = llama31_ollama +OLLAMA_TEMPLATES["llama-31-nemotron"] = llama_31_nemotron_ollama +OLLAMA_TEMPLATES["llama-31-storm"] = llama_31_storm_ollama +OLLAMA_TEMPLATES["llama-32-vision"] = llama_32_vision_ollama + +for version in ("llama-3.2", "llama-3.3", "llama-32", "llama-33"): + OLLAMA_TEMPLATES[version] = OLLAMA_TEMPLATES["llama-3.1"] +pass + +# =========================================== tinyllama +# tinyllama-chat https://ollama.com/library/tinyllama:latest/blobs/af0ddbdaaa26 +tinyllama_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """<|system|> +{{ .System }} +<|user|> +{{ .Prompt }} +<|assistant|>""" +PARAMETER stop "<|system|>" +PARAMETER stop "<|user|>" +PARAMETER stop "<|assistant|>" +PARAMETER "" +SYSTEM """You are a helpful AI assistant.""" +''' + +OLLAMA_TEMPLATES["tinyllama"] = tinyllama_ollama + +pass + +# =========================================== Qwen 2/2.5 +# Qwen2 https://ollama.com/library/qwen2:latest/blobs/77c91b422cc9 +# Qwen2.5 from https://ollama.com/library/qwen2.5/blobs/eb4402837c78 +qwen25_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }}""" +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|endoftext|>" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" +''' + +# https://ollama.com/library/qwen2.5-coder:latest/blobs/1e65450c3067 +qwen_25_coder_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- if .Suffix }}<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> +{{- else if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within : + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within with NO other text. Do not include any backticks or ```json. + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }}""" +SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" +''' + +# https://ollama.com/library/qwen2.5vl:latest/blobs/a242d8dfdc8f +qwen_25_vl_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- if .System -}} +<|im_start|>system +{{ .System }}<|im_end|> +{{- end -}} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }} +<|im_start|>user +{{ .Content }}<|im_end|> +{{- else if eq .Role "assistant" }} +<|im_start|>assistant +{{ if .Content }}{{ .Content }}{{ if not $last }}<|im_end|> +{{- else -}}<|im_end|>{{- end -}} +{{- end -}} +{{- end -}} +{{- if and (ne .Role "assistant") $last }} +<|im_start|>assistant +{{ end -}} +{{- end }}""" +PARAMETER temperature 0.0001 +SYSTEM """You are a helpful assistant.""" +''' + +# https://ollama.com/library/openthinker:latest/blobs/32695b892af8 +openthinker_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +<|im_start|>{{ .Role }}<|im_sep|> +{{ .Content }}{{ if not $last }}<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_end|> +<|im_start|>assistant<|im_sep|> +{{ end }} +{{- end }}""" +''' + + +OLLAMA_TEMPLATES["qwen-25"] = qwen25_ollama +OLLAMA_TEMPLATES["qwen-25-coder"] = qwen_25_coder_ollama +OLLAMA_TEMPLATES["qwen-25-vl"] = qwen_25_vl_ollama +OLLAMA_TEMPLATES["openthinker"] = openthinker_ollama +OLLAMA_TEMPLATES["qwen-2"] = qwen25_ollama +pass + +# =========================================== Phi-4 +_phi4_ollama_template = \ + "{{ if .System }}<|im_start|><|system|><|im_sep|>{{ .System }}<|im_end|>{{ end }}"\ + "{{ if .Prompt }}<|im_start|><|user|><|im_sep|>{{ .Prompt }}<|im_end|>{{ end }}"\ + "<|im_start|><|assistant|><|im_sep|>{{ .Response }}<|im_end|>" + +# Ollama from https://www.ollama.com/library/phi4 is different +phi_4_ollama = \ +f''' +FROM {{__FILE_LOCATION__}} +TEMPLATE """{_phi4_ollama_template}""" +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_sep|>" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +# https://ollama.com/library/phi4-reasoning:latest/blobs/32695b892af8 +phi_4_reasoning_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """ +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +<|im_start|>{{ .Role }}<|im_sep|> +{{ .Content }}{{ if not $last }}<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_end|> +<|im_start|>assistant<|im_sep|> +{{ end }} +{{- end }}""" +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|im_sep|>" +''' + +# https://ollama.com/library/phi4-mini:latest/blobs/813f53fdc6e5 +phi_4_mini_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- if or .System .Tools }}<|system|>{{ if .System }}{{ .System }}{{ end }} +{{- if .Tools }}{{ if not .System }}You are a helpful assistant with some tools.{{ end }}<|tool|>{{ .Tools }}<|/tool|><|end|> +{{- end }} +{{- end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if ne .Role "system" }}<|{{ .Role }}|>{{ .Content }} +{{- if .ToolCalls }}<|tool_call|>[{{ range .ToolCalls }}{"name":"{{ .Function.Name }}","arguments":{{ .Function.Arguments }}{{ end }}]<|/tool_call|> +{{- end }} +{{- if not $last }}<|end|> +{{- end }} +{{- if and (ne .Role "assistant") $last }}<|end|><|assistant|>{{ end }} +{{- end }} +{{- end }}""" +''' + +# https://ollama.com/library/phi4-mini-reasoning:latest/blobs/c895a1f8e8c6 +phi_4_mini_reasoning_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """ +{{- if .System }}<|system|>{{ .System }} +{{- end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if ne .Role "system" }}<|{{ .Role }}|>{{ .Content }} +{{- if not $last }}<|end|> +{{- end }} +{{- if and (ne .Role "assistant") $last }}<|end|><|assistant|>{{ end }} +{{- end }} +{{- end }}""" +SYSTEM """Your name is Phi, an AI math expert developed by Microsoft.""" +''' +OLLAMA_TEMPLATES["phi-4"] = phi_4_ollama +OLLAMA_TEMPLATES["phi-4-reasoning"] = phi_4_reasoning_ollama +OLLAMA_TEMPLATES["phi-4-mini"] = phi_4_mini_ollama +OLLAMA_TEMPLATES["phi-4-mini-reasoning"] = phi_4_mini_reasoning_ollama +pass + + +# =========================================== Gemma-3 +# Ollama from https://ollama.com/library/gemma3/blobs/e0a42594d802 +gemma3_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if or (eq .Role "user") (eq .Role "system") }}user +{{ .Content }} +{{ if $last }}model +{{ end }} +{{- else if eq .Role "assistant" }}model +{{ .Content }}{{ if not $last }} +{{ end }} +{{- end }} +{{- end }}""" +PARAMETER stop "" +PARAMETER stop "" +PARAMETER temperature 1.0 +PARAMETER min_p 0.0 +PARAMETER top_k 64 +PARAMETER top_p 0.95 +PARAMETER num_predict 32768 +''' + +# https://ollama.com/library/gemma3:270m/blobs/4b19ac7dd2fb +gemma3_270m_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- $systemPromptAdded := false }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if eq .Role "user" }}user +{{- if (and (not $systemPromptAdded) $.System) }} +{{- $systemPromptAdded = true }} +{{ $.System }} +{{ end }} +{{ .Content }} +{{ if $last }}model +{{ end }} +{{- else if eq .Role "assistant" }}model +{{ .Content }}{{ if not $last }} +{{ end }} +{{- end }} +{{- end }} +""" +PARAMETER stop "" +PARAMETER top_k 64 +PARAMETER top_p 0.95 +''' + +OLLAMA_TEMPLATES["gemma-3"] = gemma3_ollama +OLLAMA_TEMPLATES["gemma3"] = gemma3_ollama +OLLAMA_TEMPLATES["gemma3-270m"] = gemma3_270m_ollama + +pass + +# =========================================== Qwen-3 +# Ollama template for Qwen-3 (see https://ollama.com/library/qwen3/blobs/eb4402837c78) +qwen3_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }}""" +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|im_start|>" +PARAMETER temperature 0.6 +PARAMETER min_p 0.0 +PARAMETER top_k 20 +PARAMETER top_p 0.95 +PARAMETER repeat_penalty 1 +''' + +qwen3_template_eos_token = "<|im_end|>" +OLLAMA_TEMPLATES["qwen-3"] = qwen3_ollama +OLLAMA_TEMPLATES["qwen3"] = qwen3_ollama + +pass + +# =========================================== Gemma-3n +# Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802 +gemma3n_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if or (eq .Role "user") (eq .Role "system") }}user +{{ .Content }} +{{ if $last }}model +{{ end }} +{{- else if eq .Role "assistant" }}model +{{ .Content }}{{ if not $last }} +{{ end }} +{{- end }} +{{- end }}""" +''' + +OLLAMA_TEMPLATES["gemma-3n"] = gemma3n_ollama +OLLAMA_TEMPLATES["gemma3n"] = gemma3n_ollama +pass + +# =========================================== GPT-OSS + +# Ollama from https://ollama.com/library/gpt-oss:latest/blobs/fa6710a93d78 +gptoss_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: {{ currentDate }} +{{- if and .IsThinkSet .Think (ne .ThinkLevel "") }} + +Reasoning: {{ .ThinkLevel }} +{{- else if or (not .IsThinkSet) (and .IsThinkSet .Think) }} + +Reasoning: medium +{{- end }} + +{{- $hasNonBuiltinTools := false }} +{{- if .Tools -}} +{{- $hasBrowserSearch := false }} +{{- $hasBrowserOpen := false }} +{{- $hasBrowserFind := false }} +{{- $hasPython := false }} + {{- range .Tools }} + {{- if eq .Function.Name "browser.search" -}}{{- $hasBrowserSearch = true -}} + {{- else if eq .Function.Name "browser.open" -}}{{- $hasBrowserOpen = true -}} + {{- else if eq .Function.Name "browser.find" -}}{{- $hasBrowserFind = true -}} + {{- else if eq .Function.Name "python" -}}{{- $hasPython = true -}} + {{- else }}{{ $hasNonBuiltinTools = true -}} + {{- end }} + {{- end }} +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind $hasPython }} + +# Tools +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind }} + +## browser + +// Tool for browsing. +// The `cursor` appears in brackets before each browsing display: `[{cursor}]`. +// Cite information from the tool using the following format: +// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`. +// Do not quote more than 10 words directly from the tool output. +// sources=web (default: web) +namespace browser { +{{- if $hasBrowserSearch }} + +// Searches for information related to `query` and displays `topn` results. +type search = (_: { +query: string, +topn?: number, // default: 10 +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserOpen }} + +// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines. +// Valid link ids are displayed with the formatting: `【{id}†.*】`. +// If `cursor` is not provided, the most recent page is implied. +// If `id` is a string, it is treated as a fully qualified URL associated with `source`. +// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. +// Use this function without `id` to scroll to a new location of an opened page. +type open = (_: { +id?: number | string, // default: -1 +cursor?: number, // default: -1 +loc?: number, // default: -1 +num_lines?: number, // default: -1 +view_source?: boolean, // default: false +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserFind }} + +// Finds exact matches of `pattern` in the current page, or the page given by `cursor`. +type find = (_: { +pattern: string, +cursor?: number, // default: -1 +}) => any; +{{- end }} + +} // namespace browser +{{- end }}{{/* end if has browser tools */}} +{{- if $hasPython }} + +## python + +Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files). + +When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster. +{{- end }}{{/* end if hasPython */}} +{{- end }}{{/* end if has any built-in tools */}} +{{- end }}{{/* end if .Tools */}} + +# Valid channels: analysis, commentary, final. Channel must be included for every message.{{ if $hasNonBuiltinTools }} +Calls to these tools must go to the commentary channel: 'functions'. +{{- end -}}<|end|>{{/* end of system */ -}} +{{- if or $hasNonBuiltinTools .System -}} +<|start|>developer<|message|>{{- if $hasNonBuiltinTools }}# Tools + +## functions + +namespace functions { +{{- range .Tools }} +{{- if not (or (eq .Function.Name "browser.search") (eq .Function.Name "browser.open") (eq .Function.Name "browser.find") (eq .Function.Name "python")) }} +{{if .Function.Description }} +// {{ .Function.Description }} +{{- end }} +{{- if and .Function.Parameters.Properties (gt (len .Function.Parameters.Properties) 0) }} +type {{ .Function.Name }} = (_: { +{{- range $name, $prop := .Function.Parameters.Properties }} +{{- if $prop.Description }} + // {{ $prop.Description }} +{{- end }} + {{ $name }}: {{ if gt (len $prop.Type) 1 }}{{ range $i, $t := $prop.Type }}{{ if $i }} | {{ end }}{{ $t }}{{ end }}{{ else }}{{ index $prop.Type 0 }}{{ end }}, +{{- end }} +}) => any; +{{- else }} +type {{ .Function.Name }} = () => any; +{{- end }} +{{- end }}{{/* end if not browser tool */}} +{{- end }}{{/* end of range .Tools */}} + +} // namespace functions +{{- end }}{{/* end if hasNonBuiltinTools */}} +{{- if .System}} + +# Instructions + +{{ .System }} +{{- end -}} +<|end|> +{{- end -}} +{{- /* Find the index of the last user message */ -}} +{{- $lastUserIdx := -1 }} +{{- $prefillingContent := false }} +{{- $prefillingThinkingOnly := false }} +{{- range $i, $msg := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{- if eq $msg.Role "user" }} + {{- $lastUserIdx = $i }} + {{- end -}} + {{- if and $last (eq $msg.Role "assistant") (gt (len $msg.Content) 0) }} + {{- $prefillingContent = true }} + {{- else if and $last (eq $msg.Role "assistant") (gt (len $msg.Thinking) 0) }} + {{- $prefillingThinkingOnly = true }} + {{- end }} +{{- end -}} +{{- /* Now render messages */ -}} +{{- range $i, $msg := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{- if (ne $msg.Role "system") -}} + {{- if eq $msg.Role "tool" -}} + {{- if or (eq $msg.ToolName "python") (eq $msg.ToolName "browser.search") (eq $msg.ToolName "browser.open") (eq $msg.ToolName "browser.find") -}} + <|start|>{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> + {{- else -}} + <|start|>functions.{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> + {{- end -}} + {{- else if eq $msg.Role "assistant" -}} + {{- if and $msg.Thinking (gt $i $lastUserIdx) -}}{{- /* Show thinking only after last user message */ -}} + <|start|>assistant<|channel|>analysis<|message|>{{ $msg.Thinking }}{{- if not $prefillingThinkingOnly -}}<|end|>{{- end -}} + {{- end -}} + {{- if gt (len $msg.Content) 0 -}} + <|start|>assistant<|channel|>final<|message|>{{ $msg.Content }}{{- if not $prefillingContent -}}<|end|>{{- end -}} + {{- end -}} + {{- if gt (len $msg.ToolCalls) 0 -}} + {{- range $j, $toolCall := $msg.ToolCalls -}} + {{- $isBuiltin := or (eq $toolCall.Function.Name "python") (eq $toolCall.Function.Name "browser.search") (eq $toolCall.Function.Name "browser.open") (eq $toolCall.Function.Name "browser.find") -}} + <|start|>assistant<|channel|>{{ if $isBuiltin }}analysis{{ else }}commentary{{ end }} to={{ if not $isBuiltin}}functions.{{end}}{{ $toolCall.Function.Name }} <|constrain|>json<|message|>{{ $toolCall.Function.Arguments }}<|call|> + {{- end -}} + {{- end -}} + {{- else if eq $msg.Role "user" -}} + <|start|>{{ $msg.Role }}<|message|>{{ $msg.Content }}<|end|> + {{- end }} + {{- else }} + {{- end }} +{{- end -}} +{{- if not (or $prefillingContent $prefillingThinkingOnly) -}} +<|start|>assistant +{{- end -}}""" +PARAMETER temperature 1.0 +PARAMETER top_k 0 +PARAMETER top_p 1.0 +''' + +OLLAMA_TEMPLATES["gpt-oss"] = gptoss_ollama +OLLAMA_TEMPLATES["gptoss"] = gptoss_ollama + +pass + +# =========================================== Qwen3 + +# Ollama from https://ollama.com/library/qwen3/blobs/53e4ea15e8f5 +qwen3_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """ +{{- $lastUserIdx := -1 -}} +{{- range $idx, $msg := .Messages -}} +{{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} +{{- end }} +{{- if or .System .Tools }}<|im_start|>system +{{ if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end -}} +<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if (and $.IsThinkSet (and .Thinking (or $last (gt $i $lastUserIdx)))) -}} +{{ .Thinking }} +{{ end -}} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +""" +''' + +OLLAMA_TEMPLATES["qwen3-instruct"] = qwen3_ollama +OLLAMA_TEMPLATES["qwen3-thinking"] = qwen3_ollama + +pass + + +# =========================================== Starling-LM + + +# Ollama from https://ollama.com/library/starling-lm:7b/blobs/4b21bfc435b4 +starling_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}GPT4 Correct System: {{ .System }}<|end_of_turn|> +{{ end }}{{ if .Prompt }}GPT4 Correct User: {{ .Prompt }}<|end_of_turn|> +{{ end }}GPT4 Correct Assistant: {{ .Response }}<|end_of_turn|>""" +PARAMETER stop "<|end_of_turn|>" +PARAMETER stop "GPT4 Correct User:" +PARAMETER stop "GPT4 Correct Assistant:" +PARAMETER stop "GPT4 Correct System:" +PARAMETER temperature 1.5 +PARAMETER min_p 0.1 +''' + +OLLAMA_TEMPLATES["starling"] = starling_ollama + +pass + +# =========================================== Yi-chat + + +# Ollama from https://ollama.com/library/yi:34b-chat/blobs/62fbfd9ed093 +yi_chat_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{ if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ .Response }}<|im_end|>""" +''' + +OLLAMA_TEMPLATES["yi-chat"] = yi_chat_ollama + +# =========================================== Granite + +# Ollama from https://ollama.com/library/granite3.2:latest/blobs/3e7ca51acd6e +granite_32_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- /* + +------ MESSAGE PARSING ------ + +*/}} +{{- /* +Declare the prompt structure variables to be filled in from messages +*/}} +{{- $system := "" }} +{{- $documents := "" }} +{{- $documentCounter := 0 }} +{{- $thinking := false }} +{{- $citations := false }} +{{- $hallucinations := false }} +{{- $length := "" }} + +{{- /* +Loop over messages and look for a user-provided system message and documents +*/ -}} +{{- range .Messages }} + + {{- /* User defined system prompt(s) */}} + {{- if (eq .Role "system")}} + {{- if (ne $system "") }} + {{- $system = print $system " " }} + {{- end}} + {{- $system = print $system .Content }} + {{- end}} + + {{- /* + NOTE: Since Ollama collates consecutive roles, for control and documents, we + work around this by allowing the role to contain an qualifier after the + role string. + */ -}} + + {{- /* Role specified thinking */ -}} + {{- if (and (ge (len .Role) 7) (eq (slice .Role 0 7) "control")) }} + {{- if (eq .Content "thinking")}}{{- $thinking = true }}{{- end}} + {{- if (eq .Content "citations")}}{{- $citations = true }}{{- end}} + {{- if (eq .Content "hallucinations")}}{{- $hallucinations = true }}{{- end}} + {{- if (and (ge (len .Content) 7) (eq (slice .Content 0 7) "length "))}} + {{- $length = print ` {"length": "` (slice .Content 7) `"}` }} + {{- end}} + {{- end}} + + {{- /* Role specified document */ -}} + {{- if (and (ge (len .Role) 8) (eq (slice .Role 0 8) "document")) }} + {{- if (ne $documentCounter 0)}} + {{- $documents = print $documents " "}} + {{- end}} + {{- $identifier := $documentCounter}} + {{- if (ge (len .Role) 9) }} + {{- $identifier = (slice .Role 8)}} + {{- end}} + {{- $documents = print $documents "Document " $identifier "" .Content}} + {{- $documentCounter = len (printf "a%*s" $documentCounter "")}} + {{- end}} +{{- end}} + +{{- /* +If no user message provided, build the default system message +*/ -}} +{{- if eq $system "" }} + {{- $system = "Knowledge Cutoff Date: April 2024.You are Granite, developed by IBM."}} + + {{- /* Add Tools prompt */}} + {{- if .Tools }} + {{- $system = print $system " You are a helpful AI assistant with access to the following tools. When a tool is required to answer the user's query, respond with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request." }} + {{- end}} + + {{- /* Add documents prompt */}} + {{- if $documents }} + {{- if .Tools }} + {{- $system = print $system " "}} + {{- else }} + {{- $system = print $system " "}} + {{- end}} + {{- $system = print $system "Write the response to the user's input by strictly aligning with the facts in the provided documents. If the information needed to answer the question is not available in the documents, inform the user that the question cannot be answered based on the available data." }} + {{- if $citations}} + {{- $system = print $system " In your response, use the symbols and to indicate when a fact comes from a document in the search result, e.g 0 for a fact from document 0. Afterwards, list all the citations with their corresponding documents in an ordered list."}} + {{- end}} + {{- if $hallucinations}} + {{- $system = print $system "Finally, after the response is written, include a numbered list of sentences from the response that are potentially hallucinated and not based in the documents."}} + {{- end}} + {{- end}} + + {{- /* Prompt without tools or documents */}} + {{- if (and (not .Tools) (not $documents)) }} + {{- $system = print $system " You are a helpful AI assistant."}} + {{- if $thinking}} + {{- $system = print $system "Respond to every user query in a comprehensive and detailed way. You can write down your thought process before responding. Write your thoughts after 'Here is my thought process:' and write your response after 'Here is my response:' for each user query."}} + {{- end}} + {{- end}} + + {{- /* Add thinking prompt if no tools or documents */}} + {{- if (and $thinking (not .Tools) (not $documents)) }} + {{- $system = print $system " You are a helpful AI assistant.Respond to every user query in a comprehensive and detailed way. You can write down your thought process before responding. Write your thoughts after 'Here is my thought process:' and write your response after 'Here is my response:' for each user query."}} + {{- end}} + +{{- end}} +{{- /* + +------ TEMPLATE EXPANSION ------ + +*/}} +{{- /* System Prompt */ -}} +<|start_of_role|>system<|end_of_role|>{{- $system }}<|end_of_text|> + +{{- /* Tools */ -}} +{{- if .Tools }} +<|start_of_role|>tools<|end_of_role|>[ +{{- range $index, $_ := .Tools }} +{{ . }} +{{- if and (ne (len (slice $.Tools $index)) 1) (gt (len $.Tools) 1) }}, +{{- end}} +{{- end }} +] +{{- end}} + +{{- /* Documents */ -}} +{{- if $documents }} +<|start_of_role|>documents<|end_of_role|> +{{ $documents }}<|end_of_text|> +{{- end}} + +{{- /* Standard Messages */}} +{{- range $index, $_ := .Messages }} +{{- if (and + (ne .Role "system") + (or (lt (len .Role) 7) (ne (slice .Role 0 7) "control")) + (or (lt (len .Role) 8) (ne (slice .Role 0 8) "document")) +)}} +<|start_of_role|> +{{- if eq .Role "tool" }}tool_response +{{- else }}{{ .Role }} +{{- end }}<|end_of_role|> +{{- if .Content }}{{ .Content }} +{{- else if .ToolCalls }}<|tool_call|> +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{- end }} +{{- end }} +{{- if eq (len (slice $.Messages $index)) 1 }} +{{- if eq .Role "assistant" }} +{{- else }}<|end_of_text|> +<|start_of_role|>assistant<|end_of_role|> +{{- end -}} +{{- else }}<|end_of_text|> +{{- end }} +{{- end }} +{{- end }} +""" +''' + +# granite-3.2-vision https://ollama.com/library/granite3.2-vision:latest/blobs/579046ba1157 +granite_32_vision_ollama = \ +''' +FROM {__FILE_LOCATION__} +TEMPLATE """{{- /* Tools */ -}} +{{- if .Tools -}} +<|start_of_role|>available_tools<|end_of_role|> +{{- range $index, $_ := .Tools }} +{{- $last := eq (len (slice $.Tools $index)) 1 }} +{{ . }} +{{- if not $last }} +{{ end}} +{{- end -}} +<|end_of_text|> +{{ end }} + +{{- /* System Prompt */ -}} +{{- if and (gt (len .Messages) 0) (eq (index .Messages 0).Role "system") -}} +<|system|> +{{(index .Messages 0).Content}} +{{- else -}} +<|system|> +A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. +{{- end }} + +{{- /*Main message loop*/ -}} +{{- range $index, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $index)) 1 }} +{{- if eq .Role "system" }} + +{{- else if eq .Role "user" }} +<|user|> +{{.Content}} + +{{- else if eq .Role "assistant" }} +<|assistant|> +{{- if .Content }} +{{.Content}} +<|end_of_text|> +{{ end }} + +{{- else if eq .Role "assistant_tool_call" }} +<|start_of_role|>assistant<|end_of_role|><|tool_call|>{{.Content}}<|end_of_text|> + +{{- else if eq .Role "tool_response" }} +<|start_of_role|>tool_response<|end_of_role|>{{.Content}}<|end_of_text|> +{{- end }} + +{{- /* Add generation prompt */ -}} +{{ if $last }} +{{- if eq .Role "assistant" }} +{{- else }} +<|assistant|> +{{- end }} +{{- end }} +{{- end }}""" +PARAMETER num_ctx 16384 +PARAMETER temperature 0 +SYSTEM """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.""" +''' + +OLLAMA_TEMPLATES["granite-32"] = granite_32_ollama +OLLAMA_TEMPLATES["granite-32-vision"] = granite_32_vision_ollama + +pass + + +OLLAMA_TEMPLATE_TO_MODEL_MAPPER = { + "phi-3.5": ( + "unsloth/Phi-3.5-mini-instruct-bnb-4bit", + "unsloth/Phi-3.5-mini-instruct", + "microsoft/Phi-3.5-mini-instruct", + ), + "phi-3": ( + "unsloth/Phi-3-mini-4k-instruct-bnb-4bit", + "unsloth/Phi-3-mini-4k-instruct", + "microsoft/Phi-3-mini-4k-instruct", + "unsloth/Phi-3-medium-4k-instruct-bnb-4bit", + "unsloth/Phi-3-medium-4k-instruct", + "microsoft/Phi-3-medium-4k-instruct", + "unsloth/Phi-3-mini-4k-instruct-v0-bnb-4bit", + "unsloth/Phi-3-mini-4k-instruct-v0", + ), + "phi-4": ( + "unsloth/phi-4-unsloth-bnb-4bit", + "unsloth/phi-4", + "microsoft/phi-4", + "unsloth/phi-4-bnb-4bit", + ), + "phi-4-reasoning": ( + "unsloth/phi-4-reasoning-unsloth-bnb-4bit", + "unsloth/phi-4-reasoning", + "microsoft/Phi-4-reasoning", + "unsloth/phi-4-reasoning-bnb-4bit", + "unsloth/phi-4-reasoning-plus-unsloth-bnb-4bit", + "unsloth/phi-4-reasoning-plus", + "microsoft/Phi-4-reasoning-plus", + "unsloth/phi-4-reasoning-plus-bnb-4bit", + ), + "phi-4-mini": ( + "unsloth/Phi-4-mini-instruct-unsloth-bnb-4bit", + "unsloth/Phi-4-mini-instruct", + "microsoft/Phi-4-mini-instruct", + "unsloth/Phi-4-mini-instruct-bnb-4bit", + ), + "phi-4-mini-reasoning": ( + "unsloth/phi-4-mini-reasoning-unsloth-bnb-4bit", + "unsloth/phi-4-mini-reasoning", + "microsoft/Phi-4-mini-reasoning", + "unsloth/phi-4-mini-reasoning-bnb-4bit", + ), + "mistral": ( + "unsloth/mistral-7b-instruct-v0.1-bnb-4bit", + "unsloth/mistral-7b-instruct-v0.1", + "mistralai/Mistral-7B-Instruct-v0.1", + "unsloth/mistral-7b-instruct-v0.2-bnb-4bit", + "unsloth/mistral-7b-instruct-v0.2", + "mistralai/Mistral-7B-Instruct-v0.2", + ), + "mistral-v03":( + "unsloth/mistral-7b-instruct-v0.3-bnb-4bit", + "unsloth/mistral-7b-instruct-v0.3", + "mistralai/Mistral-7B-Instruct-v0.3", + "unsloth/Mistral-Large-Instruct-2407-bnb-4bit", + "mistralai/Mistral-Large-Instruct-2407", + ), + "mistral-small": ( + "unsloth/Mistral-Small-Instruct-2409-bnb-4bit", + "unsloth/Mistral-Small-Instruct-2409", + "mistralai/Mistral-Small-Instruct-2409", + "unsloth/Mistral-Small-24B-Instruct-2501-unsloth-bnb-4bit", + "unsloth/Mistral-Small-24B-Instruct-2501", + "mistralai/Mistral-Small-24B-Instruct-2501", + "unsloth/Mistral-Small-24B-Instruct-2501-bnb-4bit", + ), + "mistral-small-31": ( + "unsloth/Mistral-Small-3.1-24B-Instruct-2503-unsloth-bnb-4bit", + "unsloth/Mistral-Small-3.1-24B-Instruct-2503", + "mistralai/Mistral-Small-3.1-24B-Instruct-2503", + "unsloth/Mistral-Small-3.1-24B-Instruct-2503-bnb-4bit", + ), + "mistral-small-32": ( + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506", + "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit", + ), + "mixtral":( + "unsloth/Mixtral-8x7B-Instruct-v0.1-unsloth-bnb-4bit", + "unsloth/Mixtral-8x7B-Instruct-v0.1", + "mistralai/Mixtral-8x7B-Instruct-v0.1", + "unsloth/Mixtral-8x7B-Instruct-v0.1-bnb-4bit", + ), + "mistral-nemo": ( + "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", + "unsloth/Mistral-Nemo-Instruct-2407", + "mistralai/Mistral-Nemo-Instruct-2407", + ), + "codestral": ( + "mistralai/Codestral-22B-v0.1", + "mistral-community/Codestral-22B-v0.1", + ), + "devstral": ( + "unsloth/Devstral-Small-2505-unsloth-bnb-4bit", + "unsloth/Devstral-Small-2505", + "mistralai/Devstral-Small-2505", + "unsloth/Devstral-Small-2505-bnb-4bit", + "unsloth/Devstral-Small-2507-unsloth-bnb-4bit", + "unsloth/Devstral-Small-2507", + "mistralai/Devstral-Small-2507", + "unsloth/Devstral-Small-2507-bnb-4bit", + ), + "magistral": ( + "unsloth/Magistral-Small-2506-unsloth-bnb-4bit", + "unsloth/Magistral-Small-2506", + "mistralai/Magistral-Small-2506", + "unsloth/Magistral-Small-2506-bnb-4bit", + "unsloth/Magistral-Small-2507-unsloth-bnb-4bit", + "unsloth/Magistral-Small-2507", + "mistralai/Magistral-Small-2507", + "unsloth/Magistral-Small-2507-bnb-4bit", + "unsloth/Magistral-Small-2509-unsloth-bnb-4bit", + "unsloth/Magistral-Small-2509", + "mistralai/Magistral-Small-2509", + "unsloth/Magistral-Small-2509-bnb-4bit", + ), + "tinyllama": ( + "unsloth/tinyllama-chat-bnb-4bit", + "unsloth/tinyllama-chat", + "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + ), + "llama": ( + "unsloth/llama-2-7b-bnb-4bit", + "unsloth/llama-2-7b", + "meta-llama/Llama-2-7b-hf", + "unsloth/llama-2-13b-bnb-4bit", + "unsloth/llama-2-13b", + "meta-llama/Llama-2-13b-hf", + "unsloth/llama-2-7b-chat-bnb-4bit", + "unsloth/llama-2-7b-chat", + "meta-llama/Llama-2-7b-chat-hf", + ), + "llama3": ( + "unsloth/llama-3-8b-Instruct-bnb-4bit", + "unsloth/llama-3-8b-Instruct", + "meta-llama/Meta-Llama-3-8B-Instruct", + "unsloth/llama-3-70b-Instruct-bnb-4bit", + "meta-llama/Meta-Llama-3-70B-Instruct", + ), + "llama-3.1": ( + "unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit", + "unsloth/Meta-Llama-3.1-8B-Instruct", + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit", + "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit", + "unsloth/Llama-3.1-8B-Instruct", + "meta-llama/Llama-3.1-8B-Instruct", + "unsloth/Llama-3.1-8B-Instruct-bnb-4bit", + "unsloth/Meta-Llama-3.1-405B-Instruct-bnb-4bit", + "meta-llama/Meta-Llama-3.1-405B-Instruct", + "unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit", + "unsloth/Meta-Llama-3.1-70B-Instruct", + "meta-llama/Meta-Llama-3.1-70B-Instruct", + "unsloth/Hermes-3-Llama-3.1-8B-bnb-4bit", + "unsloth/Hermes-3-Llama-3.1-8B", + "NousResearch/Hermes-3-Llama-3.1-8B", + "unsloth/Hermes-3-Llama-3.1-70B-bnb-4bit", + "unsloth/Hermes-3-Llama-3.1-70B", + "NousResearch/Hermes-3-Llama-3.1-70B", + "unsloth/Hermes-3-Llama-3.1-405B-bnb-4bit", + "NousResearch/Hermes-3-Llama-3.1-405B", + "unsloth/Llama-3.1-Tulu-3-8B-bnb-4bit", + "unsloth/Llama-3.1-Tulu-3-8B", + "allenai/Llama-3.1-Tulu-3-8B", + "unsloth/Llama-3.1-Tulu-3-70B-bnb-4bit", + "unsloth/Llama-3.1-Tulu-3-70B", + "allenai/Llama-3.1-Tulu-3-70B", + ), + "llama-31-storm": ( + "unsloth/Llama-3.1-Storm-8B-bnb-4bit", + "unsloth/Llama-3.1-Storm-8B", + "akjindal53244/Llama-3.1-Storm-8B", + ), + "llama-31-nemotron":( + "unsloth/Llama-3.1-Nemotron-70B-Instruct-bnb-4bit", + "unsloth/Llama-3.1-Nemotron-70B-Instruct", + "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", + ), + "llama-3.2": ( + "unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit", + "unsloth/Llama-3.2-1B-Instruct", + "meta-llama/Llama-3.2-1B-Instruct", + "unsloth/Llama-3.2-1B-Instruct-bnb-4bit", + "unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit", + "unsloth/Llama-3.2-3B-Instruct", + "meta-llama/Llama-3.2-3B-Instruct", + "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", + + ), + "llama-32-vision":( + "unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit", + "unsloth/Llama-3.2-11B-Vision-Instruct", + "meta-llama/Llama-3.2-11B-Vision-Instruct", + "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit", + "unsloth/Llama-3.2-90B-Vision-Instruct-bnb-4bit", + "unsloth/Llama-3.2-90B-Vision-Instruct", + "meta-llama/Llama-3.2-90B-Vision-Instruct", + ), + "llama-3.3": ( + "unsloth/Llama-3.3-70B-Instruct-bnb-4bit", + "unsloth/Llama-3.3-70B-Instruct", + "meta-llama/Llama-3.3-70B-Instruct", + ), + "gemma": ( + "unsloth/gemma-7b-it-bnb-4bit", + "unsloth/gemma-7b-it", + "google/gemma-7b-it", + "google/gemma-2b-it", + "unsloth/gemma-1.1-2b-it-bnb-4bit", + "unsloth/gemma-1.1-2b-it", + "google/gemma-1.1-2b-it", + "unsloth/gemma-1.1-7b-it-bnb-4bit", + "unsloth/gemma-1.1-7b-it", + "google/gemma-1.1-7b-it", + ), + "gemma2": ( + "unsloth/gemma-2-9b-it-bnb-4bit", + "unsloth/gemma-2-9b-it", + "google/gemma-2-9b-it", + "unsloth/gemma-2-27b-it-bnb-4bit", + "unsloth/gemma-2-27b-it", + "google/gemma-2-27b-it", + "unsloth/gemma-2-2b-it-bnb-4bit", + "unsloth/gemma-2-2b-it", + "google/gemma-2-2b-it", + ), + "gemma-3": ( + "unsloth/gemma-3-1b-it-unsloth-bnb-4bit", + "unsloth/gemma-3-1b-it", + "google/gemma-3-1b-it", + "unsloth/gemma-3-1b-it-bnb-4bit", + "unsloth/gemma-3-4b-it-unsloth-bnb-4bit", + "unsloth/gemma-3-4b-it", + "google/gemma-3-4b-it", + "unsloth/gemma-3-4b-it-bnb-4bit", + "unsloth/gemma-3-12b-it-unsloth-bnb-4bit", + "unsloth/gemma-3-12b-it", + "google/gemma-3-12b-it", + "unsloth/gemma-3-12b-it-bnb-4bit", + "unsloth/gemma-3-27b-it-unsloth-bnb-4bit", + "unsloth/gemma-3-27b-it", + "google/gemma-3-27b-it", + "unsloth/gemma-3-27b-it-bnb-4bit", + "unsloth/medgemma-4b-it-unsloth-bnb-4bit", + "unsloth/medgemma-4b-it", + "google/medgemma-4b-it", + "unsloth/medgemma-4b-it-bnb-4bit", + "unsloth/medgemma-27b-text-it-unsloth-bnb-4bit", + "unsloth/medgemma-27b-text-it", + "google/medgemma-27b-text-it", + "unsloth/medgemma-27b-text-it-bnb-4bit", + ), + "gemma3n": ( + "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit", + "unsloth/gemma-3n-E4B-it", + "google/gemma-3n-E4B-it", + "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit", + "unsloth/gemma-3n-E2B-it-unsloth-bnb-4bit", + "unsloth/gemma-3n-E2B-it", + "google/gemma-3n-E2B-it", + "unsloth/gemma-3n-E2B-it-unsloth-bnb-4bit", + ), + "gemma3-270m":( + "unsloth/gemma-3-270m-it-unsloth-bnb-4bit", + "unsloth/gemma-3-270m-it", + "google/gemma-3-270m-it", + "unsloth/gemma-3-270m-it-bnb-4bit", + ), + "qwen-25": ( + "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-0.5B-Instruct", + "Qwen/Qwen2.5-0.5B-Instruct", + "unsloth/Qwen2.5-0.5B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-1.5B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-1.5B-Instruct", + "Qwen/Qwen2.5-1.5B-Instruct", + "unsloth/Qwen2.5-1.5B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-3B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-3B-Instruct", + "Qwen/Qwen2.5-3B-Instruct", + "unsloth/Qwen2.5-3B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-7B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-7B-Instruct", + "Qwen/Qwen2.5-7B-Instruct", + "unsloth/Qwen2.5-7B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-14B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-14B-Instruct", + "Qwen/Qwen2.5-14B-Instruct", + "unsloth/Qwen2.5-14B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-32B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-32B-Instruct", + "Qwen/Qwen2.5-32B-Instruct", + "unsloth/Qwen2.5-72B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-72B-Instruct", + "Qwen/Qwen2.5-72B-Instruct", + "unsloth/Qwen2.5-Math-1.5B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Math-1.5B-Instruct", + "Qwen/Qwen2.5-Math-1.5B-Instruct", + "unsloth/Qwen2.5-Math-7B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Math-7B-Instruct", + "Qwen/Qwen2.5-Math-7B-Instruct", + "unsloth/Qwen2.5-Math-72B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Math-72B-Instruct", + "Qwen/Qwen2.5-Math-72B-Instruct", + + ), + "qwen-25-coder":( + "unsloth/Qwen2.5-Coder-0.5B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Coder-0.5B-Instruct", + "Qwen/Qwen2.5-Coder-0.5B-Instruct", + "unsloth/Qwen2.5-Coder-1.5B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Coder-1.5B-Instruct", + "Qwen/Qwen2.5-Coder-1.5B-Instruct", + "unsloth/Qwen2.5-Coder-3B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Coder-3B-Instruct", + "Qwen/Qwen2.5-Coder-3B-Instruct", + "unsloth/Qwen2.5-Coder-7B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Coder-7B-Instruct", + "Qwen/Qwen2.5-Coder-7B-Instruct", + "unsloth/Qwen2.5-Coder-14B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Coder-14B-Instruct", + "Qwen/Qwen2.5-Coder-14B-Instruct", + "unsloth/Qwen2.5-Coder-32B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-Coder-32B-Instruct", + "Qwen/Qwen2.5-Coder-32B-Instruct", + ), + "qwen-25-vl":( + "unsloth/Qwen2.5-VL-3B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-VL-3B-Instruct", + "Qwen/Qwen2.5-VL-3B-Instruct", + "unsloth/Qwen2.5-VL-3B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-VL-7B-Instruct", + "Qwen/Qwen2.5-VL-7B-Instruct", + "unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-VL-32B-Instruct", + "Qwen/Qwen2.5-VL-32B-Instruct", + "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit", + "unsloth/Qwen2.5-VL-72B-Instruct-unsloth-bnb-4bit", + "unsloth/Qwen2.5-VL-72B-Instruct", + "Qwen/Qwen2.5-VL-72B-Instruct", + "unsloth/Qwen2.5-VL-72B-Instruct-bnb-4bit", + ), + "openthinker": ( + "unsloth/OpenThinker-7B-unsloth-bnb-4bit", + "unsloth/OpenThinker-7B", + "open-thoughts/OpenThinker-7B", + "unsloth/OpenThinker-7B-bnb-4bit", + ), + "qwen-2": ( + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit", + "unsloth/Qwen2-0.5B-Instruct", + "Qwen/Qwen2-0.5B-Instruct", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit", + "unsloth/Qwen2-1.5B-Instruct", + "Qwen/Qwen2-1.5B-Instruct", + "unsloth/Qwen2-7B-Instruct-bnb-4bit", + "unsloth/Qwen2-7B-Instruct", + "Qwen/Qwen2-7B-Instruct", + "unsloth/Qwen2-70B-Instruct-bnb-4bit", + "Qwen/Qwen2-70B-Instruct", + ), + "qwen3": ( + "unsloth/Qwen3-0.6B-unsloth-bnb-4bit", + "unsloth/Qwen3-0.6B", + "Qwen/Qwen3-0.6B", + "unsloth/Qwen3-0.6B-bnb-4bit", + "unsloth/Qwen3-1.7B-unsloth-bnb-4bit", + "unsloth/Qwen3-1.7B", + "Qwen/Qwen3-1.7B", + "unsloth/Qwen3-1.7B-bnb-4bit", + "unsloth/Qwen3-4B-unsloth-bnb-4bit", + "unsloth/Qwen3-4B", + "Qwen/Qwen3-4B", + "unsloth/Qwen3-4B-bnb-4bit", + "unsloth/Qwen3-8B-unsloth-bnb-4bit", + "unsloth/Qwen3-8B", + "Qwen/Qwen3-8B", + "unsloth/Qwen3-8B-bnb-4bit", + "unsloth/Qwen3-14B-unsloth-bnb-4bit", + "unsloth/Qwen3-14B", + "Qwen/Qwen3-14B", + "unsloth/Qwen3-14B-bnb-4bit", + "unsloth/Qwen3-32B-unsloth-bnb-4bit", + "unsloth/Qwen3-32B", + "Qwen/Qwen3-32B", + "unsloth/Qwen3-32B-bnb-4bit", + "unsloth/Qwen3-30B-A3B-unsloth-bnb-4bit", + "unsloth/Qwen3-30B-A3B", + "Qwen/Qwen3-30B-A3B", + "unsloth/Qwen3-30B-A3B-bnb-4bit", + ), + "qwen3-instruct": ( + "unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit", + "unsloth/Qwen3-4B-Instruct-2507", + "Qwen/Qwen3-4B-Instruct-2507", + "unsloth/Qwen3-4B-Instruct-2507-bnb-4bit", + "unsloth/Qwen3-30B-A3B-Instruct-2507", + "Qwen/Qwen3-30B-A3B-Instruct-2507", + "unsloth/Qwen3-Coder-30B-A3B-Instruct", + "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit", + "unsloth/Qwen3-4B-Instruct-2507", + "Qwen/Qwen3-4B-Instruct-2507", + "unsloth/Qwen3-4B-Instruct-2507-bnb-4bit", + ), + "qwen3-thinking": ( + "unsloth/QwQ-32B-Preview-bnb-4bit", + "unsloth/QwQ-32B-Preview", + "Qwen/QwQ-32B-Preview", + "unsloth/QwQ-32B-unsloth-bnb-4bit", + "unsloth/QwQ-32B", + "Qwen/QwQ-32B", + "unsloth/QwQ-32B-bnb-4bit", + "unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit", + "unsloth/Qwen3-4B-Thinking-2507", + "Qwen/Qwen3-4B-Thinking-2507", + "unsloth/Qwen3-4B-Thinking-2507-bnb-4bit", + "unsloth/Qwen3-30B-A3B-Thinking-2507", + "Qwen/Qwen3-30B-A3B-Thinking-2507", + ), + "zephyr": ( + "unsloth/zephyr-sft-bnb-4bit", + "unsloth/zephyr-sft", + "HuggingFaceH4/mistral-7b-sft-beta", + ), + "chatml": ( + "unsloth/Hermes-2-Pro-Mistral-7B-bnb-4bit", + "unsloth/Hermes-2-Pro-Mistral-7B", + "NousResearch/Hermes-2-Pro-Mistral-7B", + "unsloth/OpenHermes-2.5-Mistral-7B-bnb-4bit", + "unsloth/OpenHermes-2.5-Mistral-7B", + "teknium/OpenHermes-2.5-Mistral-7B", + ), + "gpt-oss": ( + "unsloth/gpt-oss-20b-unsloth-bnb-4bit", + "unsloth/gpt-oss-20b", + "openai/gpt-oss-20b", + "unsloth/gpt-oss-20b-unsloth-bnb-4bit", + "unsloth/gpt-oss-120b-unsloth-bnb-4bit", + "unsloth/gpt-oss-120b", + "openai/gpt-oss-120b", + "unsloth/gpt-oss-120b-unsloth-bnb-4bit", + ), + "starling": ( + "unsloth/Starling-LM-7B-beta-bnb-4bit", + "unsloth/Starling-LM-7B-beta", + "Nexusflow/Starling-LM-7B-beta", + ), + "yi-chat": ( + "unsloth/yi-34b-chat-bnb-4bit", + "01-ai/Yi-6B-Chat", + "01-ai/Yi-34B-Chat", + ), + "granite-32": ( + "unsloth/granite-3.2-2b-instruct-unsloth-bnb-4bit", + "unsloth/granite-3.2-2b-instruct", + "ibm-granite/granite-3.2-2b-instruct", + "unsloth/granite-3.2-2b-instruct-bnb-4bit", + "unsloth/granite-3.2-8b-instruct-unsloth-bnb-4bit", + "unsloth/granite-3.2-8b-instruct", + "ibm-granite/granite-3.2-8b-instruct", + "unsloth/granite-3.2-8b-instruct-bnb-4bit", + ), + "granite-32-vision": ( + "unsloth/granite-vision-3.2-2b-unsloth-bnb-4bit", + "unsloth/granite-vision-3.2-2b", + "ibm-granite/granite-vision-3.2-2b", + "unsloth/granite-vision-3.2-2b-bnb-4bit", + ), +} + +MODEL_TO_OLLAMA_TEMPLATE_MAPPER = {} + +for key, values in OLLAMA_TEMPLATE_TO_MODEL_MAPPER.items(): + for value in values: + MODEL_TO_OLLAMA_TEMPLATE_MAPPER[value] = key + pass + + # Get lowercased + lowered_key = key.lower() + for value in values: + MODEL_TO_OLLAMA_TEMPLATE_MAPPER[value.lower()] = lowered_key + pass +pass diff --git a/unsloth/save.py b/unsloth/save.py index aeac2bab71..8dd69bb52c 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -14,6 +14,7 @@ from unsloth_zoo.utils import Version from unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE +from unsloth_zoo.llama_cpp import convert_to_gguf, quantize_gguf, use_local_gguf, install_llama_cpp, check_llama_cpp, _download_convert_hf_to_gguf from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit from peft.tuners.lora import Linear4bit as Peft_Linear4bit from peft.tuners.lora import Linear as Peft_Linear @@ -32,6 +33,9 @@ import psutil import re from transformers.models.llama.modeling_llama import logger from .tokenizer_utils import fix_sentencepiece_gguf +from .models.loader_utils import get_model_name +from .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPER +from transformers import ProcessorMixin from huggingface_hub import HfApi try: from huggingface_hub import get_token @@ -951,19 +955,27 @@ pass def save_to_gguf( + model_name : str, model_type : str, model_dtype : str, is_sentencepiece : bool = False, model_directory : str = "unsloth_finetuned_model", quantization_method = "fast_quantized", # Can be a list of options! ["q4_k_m", "q8_0", "q5_k_m"] first_conversion : str = None, - _run_installer = None, # Non blocking install of llama.cpp + is_vlm : bool = False, + is_gpt_oss : bool = False, ): - # logger.warning( - # "NOTICE: llama.cpp GGUF conversion is currently unstable, since llama.cpp is\n"\ - # "undergoing some major bug fixes as at 5th of May 2024. This is not an Unsloth issue.\n"\ - # "Please be patient - GGUF saving should still work, but might not work as well." - # ) + """ + Orchestrates the complete GGUF conversion process. + Handles installation, conversion, and quantization. + """ + # print_output True only if UNSLOTH_ENABLE_LOGGING=1 + if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1": + print_output = True + else: + print_output = False + + # Validate model dtype assert(model_dtype == "float16" or model_dtype == "bfloat16") model_dtype = "f16" if model_dtype == "float16" else "bf16" @@ -995,16 +1007,8 @@ def save_to_gguf( raise RuntimeError("Unsloth: Currently iq2 type quantizations aren't supported yet - sorry!") pass - # Careful convert.py is only for Llama / Mistral based archs - use_fast_convert = False - if not is_sentencepiece: use_fast_convert = False # Llama-3 - elif model_type == "llama": use_fast_convert = True - elif model_type == "mistral": use_fast_convert = True - pass - logger.warning_once(f"Unsloth: Converting {model_type} model. Can use fast conversion = {use_fast_convert}.") - # Map quant methods - new_quantization_method = [] + new_quantization_methods = [] for quant_method in quantization_method: if quant_method == "not_quantized": quant_method = model_dtype elif quant_method == "fast_quantized": quant_method = "q8_0" @@ -1019,303 +1023,186 @@ def save_to_gguf( raise RuntimeError(error) pass - new_quantization_method.append(quant_method) + new_quantization_methods.append(quant_method) pass - quantization_method = new_quantization_method + quantization_method = new_quantization_methods + # Determine optimal first_conversion + if is_gpt_oss: + print("Unsloth: GPT-OSS model detected - using special conversion settings") + first_conversion = "None" # No quantization for GPT-OSS + # Only keep one conversion method since GPT-OSS doesn't quantize + quantization_method = ["None"] + else: + if first_conversion is None: + # Check if q8_0 is the ONLY quantization method requested + if len(quantization_method) == 1 and quantization_method[0] == "q8_0": + first_conversion = "None" # Let llama-quantize do the direct conversion + else: + # For all other cases, choose the highest precision format + # that can be requantized to all requested formats + strength = 0 + for quant_method in quantization_method: + if quant_method == "f32": strength = max(strength, 3) + elif quant_method == "f16": strength = max(strength, 2) + elif quant_method == "bf16": strength = max(strength, 1) + # Note: we don't set strength for q8_0 here since we handle it above + + if strength >= 3: first_conversion = "f32" + elif strength >= 2: first_conversion = "f16" + elif strength >= 1: first_conversion = "bf16" + else: first_conversion = "bf16" # requantizing from q8_0 disallowed in new llama.cpp default to bf16. + + # Check bfloat16 support again for first_conversion + if first_conversion == "bf16" and not torch.cuda.is_bf16_supported(): + logger.warning("Unsloth: Switching bf16 to f16 due to hardware limitations") + first_conversion = "f16" + + first_conversion_dtype = "" if first_conversion == "None" else first_conversion + # Print conversion info print_info = \ - f"==((====))== Unsloth: Conversion from QLoRA to GGUF information\n"\ + f"==((====))== Unsloth: Conversion from HF to GGUF information\n"\ f" {chr(92)}{chr(92)} /| [0] Installing llama.cpp might take 3 minutes.\n"\ - f"O^O/ {chr(92)}_/ {chr(92)} [1] Converting HF to GGUF 16bits might take 3 minutes.\n"\ - f"{chr(92)} / [2] Converting GGUF 16bits to {quantization_method} might take 10 minutes each.\n"\ + f"O^O/ {chr(92)}_/ {chr(92)} [1] Converting HF to GGUF {first_conversion_dtype} might take 3 minutes.\n"\ + f"{chr(92)} / [2] Converting GGUF {first_conversion_dtype} to {quantization_method} might take 10 minutes each.\n"\ f' "-____-" In total, you will have to wait at least 16 minutes.\n' print(print_info) - # Check first_conversion format - if first_conversion == "f16" : pass - elif first_conversion == "bf16" : pass - elif first_conversion == "f32" : pass - elif first_conversion == "q8_0" : pass - else: - raise RuntimeError( - f"Unsloth: `first_conversion` can only be one of ['f16', 'bf16', 'f32', 'q8_0'] and not `{first_conversion}`." - ) - pass - - # Determine whether the system already has llama.cpp installed and the scripts are executable - quantize_location = get_executable(["llama-quantize", "quantize", "llama-quantize.exe", "quantize.exe"]) - convert_location = get_executable(["convert-hf-to-gguf.py", "convert_hf_to_gguf.py"]) - - error = 0 - if quantize_location is not None and convert_location is not None: - print("Unsloth: llama.cpp found in the system. We shall skip installation.") - else: + # Step 1: Ensure llama.cpp is installed + try: + quantizer_location, converter_location = check_llama_cpp() + print("Unsloth: llama.cpp found in the system. Skipping installation.") + except: print("Unsloth: Installing llama.cpp. This might take 3 minutes...") - if _run_installer is not None: - _run_installer, IS_CMAKE = _run_installer - - error = _run_installer.wait() - # Check if successful - if error != 0: - print(f"Unsloth: llama.cpp error code = {error}.") - install_llama_cpp_old(-10) - pass - - if IS_CMAKE: - # CMAKE needs to do some extra steps - print("Unsloth: CMAKE detected. Finalizing some steps for installation.") - - check = os.system("cp llama.cpp/build/bin/llama-* llama.cpp") - if check != 0: raise RuntimeError("Failed compiling llama.cpp. Please report this ASAP!") - check = os.system("rm -rf llama.cpp/build") - if check != 0: raise RuntimeError("Failed compiling llama.cpp. Please report this ASAP!") - pass - else: - error = 0 - install_llama_cpp_blocking() - pass - - # Careful llama.cpp/quantize changed to llama.cpp/llama-quantize - # and llama.cpp/main changed to llama.cpp/llama-cli - # See https://github.com/ggerganov/llama.cpp/pull/7809 - quantize_location = None - if os.path.exists("llama.cpp/quantize.exe"): - quantize_location = "llama.cpp/quantize.exe" - elif os.path.exists("llama.cpp/quantize"): - quantize_location = "llama.cpp/quantize" - elif os.path.exists("llama.cpp/llama-quantize.exe"): - quantize_location = "llama.cpp/llama-quantize.exe" - elif os.path.exists("llama.cpp/llama-quantize"): - quantize_location = "llama.cpp/llama-quantize" - elif os.path.exists("llama.cpp/build/bin/llama-quantize"): - quantize_location = "llama.cpp/build/bin/llama-quantize" - elif os.path.exists("llama.cpp/build/bin/quantize"): - quantize_location = "llama.cpp/build/bin/quantize" - else: - raise RuntimeError( - "Unsloth: The file 'llama.cpp/llama-quantize' or `llama.cpp/quantize` does not exist.\n"\ - "We've also double checked the building directory under 'llama.cpp/build/bin/'.\n"\ - "But we expect this file to exist! Check if the file exists under llama.cpp and investigate the building process of llama.cpp (make/cmake)!" + if IS_KAGGLE_ENVIRONMENT: + # Kaggle: no CUDA support due to environment limitations + quantizer_location, converter_location = install_llama_cpp( + gpu_support=False, + print_output=print_output ) - pass - - # See https://github.com/unslothai/unsloth/pull/730 - # Filenames changed again! - convert_location = None - if os.path.exists("llama.cpp/convert-hf-to-gguf.py"): - convert_location = "llama.cpp/convert-hf-to-gguf.py" - elif os.path.exists("llama.cpp/convert_hf_to_gguf.py"): - convert_location = "llama.cpp/convert_hf_to_gguf.py" else: - raise RuntimeError( - "Unsloth: The file 'llama.cpp/convert-hf-to-gguf.py' or 'llama.cpp/convert_hf_to_gguf.py' does not exist.\n"\ - "But we expect this file to exist! Maybe the llama.cpp developers changed the name?" + quantizer_location, converter_location = install_llama_cpp( + gpu_support=False, # GGUF conversion doesn't need CUDA + print_output=print_output ) - pass - pass - # Determine maximum first_conversion state - if first_conversion == "f32" : strength = 3 - elif first_conversion == "f16" : strength = 2 - elif first_conversion == "bf16" : strength = 1 - elif first_conversion == "q8_0" : strength = 0 + # Step 2: Download and patch converter script + print("Unsloth: Preparing converter script...") + with use_local_gguf(): + converter_path, supported_text_archs, supported_vision_archs = _download_convert_hf_to_gguf() - for quant_method in quantization_method: - if quant_method == "f32": strength = max(strength, 3) - elif quant_method == "f16": strength = max(strength, 2) - elif quant_method == "bf16": strength = max(strength, 1) - elif quant_method == "q8_0": strength = max(strength, 0) - else: - # Quantized models must have f16 as the default argument - if first_conversion == "f32" : pass - elif first_conversion == "f16" : pass - elif first_conversion == "bf16" : pass - elif first_conversion == "q8_0": - logger.warning_once( - "Unsloth: Using q8_0 for the `first_conversion` will lose a bit of accuracy, "\ - "but saves disk space!" - ) - # first_conversion = "f16" - pass - pass - pass + # Step 3: Initial GGUF conversion + print(f"Unsloth: [1] Converting model into {first_conversion_dtype} GGUF format.") + print(f"This might take 3 minutes...") - # If only q8_0: - if len(quantization_method) == 1 and quantization_method[0] == "q8_0": - strength = 0 - pass - - if strength >= 3: first_conversion = "f32" - elif strength >= 2: first_conversion = "f16" - elif strength >= 1: first_conversion = "bf16" - else: first_conversion = "q8_0" - - # Non llama/mistral needs can only use f32 or f16 - if not use_fast_convert and \ - (first_conversion != "f16" or first_conversion != "bf16" or first_conversion != "f32"): - - pass - # Latest llama.cpp works for all models for q8_0! - - # logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.") - # first_conversion = "f16" - pass - - # Check if bfloat16 is supported - if first_conversion == "bf16" and not torch.cuda.is_bf16_supported(): - logger.warning( - "Unsloth: Cannot convert to bf16 GGUF since your computer doesn't support it.\n"\ - "We shall switch instead to f16." + initial_files, is_vlm_update = convert_to_gguf( + model_name=model_name, + input_folder=model_directory, + model_dtype = model_dtype, + quantization_type=first_conversion, + converter_location=converter_path, + supported_text_archs=supported_text_archs, + supported_vision_archs=supported_vision_archs, + is_vlm=is_vlm, + is_gpt_oss=is_gpt_oss, + max_shard_size="50GB", + print_output=print_output, ) - first_conversion = "f16" - pass + # update is_vlm switch + is_vlm = is_vlm_update + # Check conversion success + for file in initial_files: + if not os.path.exists(file): + if IS_KAGGLE_ENVIRONMENT: + raise RuntimeError( + f"Unsloth: Conversion failed for {file}\n" + "You are in a Kaggle environment with limited disk space (20GB).\n" + "Try saving to /tmp for more space or use a smaller model.\n" + "Alternatively, save the 16bit model first, then convert manually." + ) + else: + raise RuntimeError( + f"Unsloth: Conversion failed for {file}\n" + "Please check disk space and try again." + ) + print(f"Unsloth: Initial conversion completed! Files: {initial_files}") + + # Step 4: Additional quantizations using llama-quantize + all_saved_locations = initial_files.copy() + + # Get CPU count for quantization n_cpus = psutil.cpu_count() if n_cpus is None: n_cpus = 1 n_cpus *= 2 - # Concurrency from https://rentry.org/llama-cpp-conversions#merging-loras-into-a-model - final_location = str((Path(model_directory) / f"unsloth.{first_conversion.upper()}.gguf").absolute()) + if not is_gpt_oss: + base_gguf = initial_files[0] + quants_created = False + for quant_method in quantization_method: + if quant_method != first_conversion: + print(f"Unsloth: [2] Converting GGUF {first_conversion_dtype} into {quant_method}. This might take 10 minutes...") + output_location = f"{model_name}.{quant_method.upper()}.gguf" - print(f"Unsloth: [1] Converting model at {model_directory} into {first_conversion} GGUF format.\n"\ - f"The output location will be {final_location}\n"\ - "This might take 3 minutes...") - - # We first check if tokenizer.model exists in the model_directory - if os.path.exists(f"{model_directory}/tokenizer.model"): - vocab_type = "spm,hfft,bpe" - # Fix Sentencepiece model as well! - fix_sentencepiece_gguf(model_directory) - else: - vocab_type = "bpe" - pass - - # convert.py is deprecated! - use_fast_convert = False - if use_fast_convert: - command = f"python llama.cpp/convert.py {model_directory} "\ - f"--outfile {final_location} --vocab-type {vocab_type} "\ - f"--outtype {first_conversion} --concurrency {n_cpus} --pad-vocab" - else: - # Fix up conversion script is possible - with open(convert_location, "rb") as f: converter_latest = f.read() - # Fix metadata - converter_latest = re.sub( - rb"(self\.metadata \= .+?\(.+?\)"\ - rb"[\n]{1,}([\s]{4,}))", - rb"\1"\ - rb"if hasattr(self.metadata, 'quantized_by'): self.metadata.quantized_by = 'Unsloth'\n"\ - rb"\2if hasattr(self.metadata, 'repo_url'): self.metadata.repo_url = 'https://huggingface.co/unsloth'\n"\ - rb"\2if hasattr(self.metadata, 'tags'): self.metadata.tags = ['unsloth', 'llama.cpp']\n"\ - rb"\2", - converter_latest, - ) - - # Make mistral_common optional for now - # from x import y - converter_latest = re.sub( - rb"(from mistral_common[^\n\(]{1,})[\s]{0,}\n", - rb"try:\n \1\nexcept:\n pass\n", - converter_latest, - ) - # from x import (y, z,) - converter_latest = re.sub( - rb"(from mistral_common[^\n\(]{1,}[\s]{0,}\(.+?\))", - rb"try:\n \1\nexcept:\n pass\n", - converter_latest, - flags = re.MULTILINE | re.DOTALL, - ) - - try: - # Write file - with open(convert_location, "wb") as file: - file.write(converter_latest) - except: - pass - command = f"python {convert_location} {model_directory} "\ - f"--outfile {final_location} "\ - f"--outtype {first_conversion}" - pass - - try_execute([command,], force_complete = True) - - # Check if quantization succeeded! - if not os.path.isfile(final_location): - if IS_KAGGLE_ENVIRONMENT: - if not Path(final_location).resolve().is_relative_to(Path('/tmp').resolve()): - raise RuntimeError( - f"Unsloth: Quantization failed for {final_location}\n"\ - "You are in a Kaggle environment, which might be the reason this is failing.\n"\ - "Kaggle only provides 20GB of disk space in the working directory.\n"\ - "Merging to 16bit for 7b models use 16GB of space.\n"\ - "This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"\ - "`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"\ - "You can try saving it to the `/tmp` directory for larger disk space.\n"\ - "I suggest you to save the 16bit model first, then use manual llama.cpp conversion." - ) - else: - raise RuntimeError( - f"Unsloth: Quantization failed for {final_location}\n"\ - "You might have to compile llama.cpp yourself, then run this again.\n"\ - "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ - "You must run this in the same folder as you're saving your model.\n"\ - "git clone --recursive https://github.com/ggerganov/llama.cpp\n"\ - "cd llama.cpp && make clean && make all -j\n"\ - "Once that's done, redo the quantization." - ) - pass - pass - print(f"Unsloth: Conversion completed! Output location: {final_location}") - - full_precision_location = final_location - - all_saved_locations = [full_precision_location,] - # Convert each type! - for quant_method in quantization_method: - if quant_method != first_conversion: - print(f"Unsloth: [2] Converting GGUF 16bit into {quant_method}. This might take 20 minutes...") - final_location = str((Path(model_directory) / f"unsloth.{quant_method.upper()}.gguf").absolute()) - - command = f"./{quantize_location} {full_precision_location} "\ - f"{final_location} {quant_method} {n_cpus}" - - try_execute([command,], force_complete = True) - - # Check if quantization succeeded! - if not os.path.isfile(final_location): - if IS_KAGGLE_ENVIRONMENT: - if not Path(final_location).resolve().is_relative_to(Path('/tmp').resolve()): + try: + # Use the quantize_gguf function we created + quantized_file = quantize_gguf( + input_gguf=base_gguf, + output_gguf=output_location, + quant_type=quant_method, + quantizer_location=quantizer_location, + print_output=print_output + ) + all_saved_locations.append(quantized_file) + quants_created = True + except Exception as e: + if IS_KAGGLE_ENVIRONMENT: raise RuntimeError( - f"Unsloth: Quantization failed for {final_location}\n"\ + f"Unsloth: Quantization failed for {output_location}\n"\ "You are in a Kaggle environment, which might be the reason this is failing.\n"\ "Kaggle only provides 20GB of disk space in the working directory.\n"\ "Merging to 16bit for 7b models use 16GB of space.\n"\ "This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"\ "`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"\ "You can try saving it to the `/tmp` directory for larger disk space.\n"\ - "I suggest you to save the 16bit model first, then use manual llama.cpp conversion." + "I suggest you to save the 16bit model first, then use manual llama.cpp conversion.\n"\ + "Error: {e}" ) - else: - raise RuntimeError( - "Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\ - "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ - "You must run this in the same folder as you're saving your model.\n"\ - "git clone --recursive https://github.com/ggerganov/llama.cpp\n"\ - "cd llama.cpp && make clean && make all -j\n"\ - "Once that's done, redo the quantization." - ) + else: + raise RuntimeError( + f"Unsloth: Quantization failed for {output_location}\n"\ + "You might have to compile llama.cpp yourself, then run this again.\n"\ + "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ + "You must run this in the same folder as you're saving your model.\n"\ + "git clone --recursive https://github.com/ggerganov/llama.cpp\n"\ + "cd llama.cpp && make clean && make all -j\n"\ + "Once that's done, redo the quantization.\n"\ + "Error: {e}" + ) + pass pass pass - - print(f"Unsloth: Conversion completed! Output location: {final_location}") - all_saved_locations.append(final_location) pass + print("Unsloth: Model files cleanup...") + if quants_created: + all_saved_locations.remove(base_gguf) + Path(base_gguf).unlink() + + # flip the list to get [text_model, mmproj] order. for text models stays the same. + all_saved_locations.reverse() + else: + print("Unsloth: GPT-OSS model - skipping additional quantizations") pass - # Finally check if first_conversion (f16, bf16 etc) was in the list of actual quant methods - full_precision_seen = first_conversion in frozenset(quantization_method) + if is_gpt_oss: + want_full_precision = True + else: + want_full_precision = first_conversion in frozenset(quantization_method) - return all_saved_locations, full_precision_seen + print(f"Unsloth: All GGUF conversions completed successfully!") + print(f"Generated files: {all_saved_locations}") + + return all_saved_locations, want_full_precision, is_vlm pass @@ -1616,13 +1503,21 @@ def fix_tokenizer_bos_token(tokenizer): pass -def create_ollama_modelfile(tokenizer, gguf_location): +def create_ollama_modelfile(tokenizer, base_model_name, model_location): """ Creates an Ollama Modelfile. Use ollama.create(model = "new_ollama_model", modelfile = modelfile) """ - modelfile = getattr(tokenizer, "_ollama_modelfile", None) - if modelfile is None: return None + ollama_template_name = MODEL_TO_OLLAMA_TEMPLATE_MAPPER.get(base_model_name) + if not ollama_template_name: + print(f"Unsloth: No Ollama template mapping found for model '{base_model_name}'. Skipping Ollama Modelfile") + return None + ollama_modelfile = OLLAMA_TEMPLATES.get(ollama_template_name) + if not ollama_modelfile: + print(f"Unsloth: No Ollama template mapping found for model '{base_model_name}'. Skipping Ollama Modelfile") + return None + tokenizer._ollama_modelfile = ollama_modelfile # This comes from the unpacking above + modelfile = ollama_modelfile FILE_LOCATION_REPLACER = "⚫@✅#🦥__FILE_LOCATION__⚡@🦥#⛵" EOS_TOKEN_REPLACER = "⚫@✅#🦥__EOS_TOKEN__⚡@🦥#⛵" @@ -1644,12 +1539,12 @@ def create_ollama_modelfile(tokenizer, gguf_location): if "__EOS_TOKEN__" in modelfile: modelfile = modelfile.format( - __FILE_LOCATION__ = gguf_location, + __FILE_LOCATION__ = model_location, __EOS_TOKEN__ = tokenizer.eos_token, ) else: modelfile = modelfile.format( - __FILE_LOCATION__ = gguf_location, + __FILE_LOCATION__ = model_location, ) pass @@ -1770,7 +1665,7 @@ def unsloth_save_pretrained_gguf( self, save_directory : Union[str, os.PathLike], tokenizer = None, - quantization_method : str = "fast_quantized", + quantization_method = "fast_quantized", first_conversion : str = None, push_to_hub : bool = False, token : Optional[Union[str, bool]] = None, @@ -1820,99 +1715,165 @@ def unsloth_save_pretrained_gguf( if tokenizer is None: raise ValueError("Unsloth: Saving to GGUF must have a tokenizer.") + try: + base_model_name = get_model_name(self.config._name_or_path, load_in_4bit=False) + model_name = base_model_name.split("/")[-1] + except: + base_model_name = self.config._name_or_path + model_name = base_model_name.split("/")[-1] + + # Check if push_to_hub is requested + if push_to_hub: + raise ValueError( + "Unsloth: Please use .push_to_hub_gguf() instead of .save_pretrained_gguf() with push_to_hub=True" + ) + + # Step 1: Check if this is a VLM (Vision-Language Model) and check if gpt-oss + is_vlm = False + if hasattr(self, 'config') and hasattr(self.config, 'architectures'): + is_vlm = any( + x.endswith(("ForConditionalGeneration", "ForVisionText2Text")) + for x in self.config.architectures + ) + is_vlm = is_vlm or hasattr(self.config, "vision_config") + + is_processor = is_vlm and isinstance(tokenizer, ProcessorMixin) + + is_gpt_oss = True if (hasattr(self.config, "architectures") and self.config.architectures == "GptOssForCausalLM") or (hasattr(self.config, "model_type") and self.config.model_type in ["gpt-oss", "gpt_oss"]) else False + # Step 2: Prepare arguments for model saving arguments = dict(locals()) arguments["model"] = self arguments["tokenizer"] = tokenizer - arguments["push_to_hub"] = False # We save ourselves - arguments["save_method"] = "merged_16bit" # Must be 16bit + arguments["push_to_hub"] = False # We handle upload ourselves + # GPT-OSS needs mxfp4 save method + if is_gpt_oss: + arguments["save_method"] = "mxfp4" + else: + arguments["save_method"] = "merged_16bit" del arguments["self"] del arguments["quantization_method"] del arguments["first_conversion"] + del arguments["is_vlm"] + del arguments["is_gpt_oss"] + del arguments["model_name"] + del arguments["base_model_name"] + del arguments["is_processor"] - # Fix tokenizer adding an extra BOS token at the front - fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer) - - # Non blocking install GGUF first - if not os.path.exists("llama.cpp"): - - if IS_KAGGLE_ENVIRONMENT: - # Kaggle is weird - no blocking installs, and no CUDA? - python_install = install_python_non_blocking(["gguf", "protobuf"]) - python_install.wait() - install_llama_cpp_blocking(use_cuda = False) - new_save_directory, old_username = unsloth_save_model(**arguments) - makefile = None - else: - git_clone = install_llama_cpp_clone_non_blocking() - python_install = install_python_non_blocking(["gguf", "protobuf"]) - git_clone.wait() - makefile = install_llama_cpp_make_non_blocking() - new_save_directory, old_username = unsloth_save_model(**arguments) - python_install.wait() - pass + # Step 3: Fix tokenizer BOS token if needed + if is_processor: + fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer.tokenizer) else: - try: - new_save_directory, old_username = unsloth_save_model(**arguments) - makefile = None - except: - # Retry by recloning llama.cpp - if IS_KAGGLE_ENVIRONMENT: - # Kaggle is weird - no blocking installs, and no CUDA? - python_install = install_python_non_blocking(["gguf", "protobuf"]) - python_install.wait() - install_llama_cpp_blocking(use_cuda = False) - new_save_directory, old_username = unsloth_save_model(**arguments) - makefile = None - else: - git_clone = install_llama_cpp_clone_non_blocking() - python_install = install_python_non_blocking(["gguf", "protobuf"]) - git_clone.wait() - makefile = install_llama_cpp_make_non_blocking() - new_save_directory, old_username = unsloth_save_model(**arguments) - python_install.wait() - pass - pass - pass + fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer) + + # Step 4: Save/merge model to 16-bit format + print(f'Unsloth: Merging model weights to {"mxfp4" if is_gpt_oss else "16-bit"} format...') + try: + # Call unsloth_generic_save directly (it's in the same file) + unsloth_generic_save(**arguments) + + except Exception as e: + raise RuntimeError(f"Failed to save/merge model: {e}") + + if is_processor: + tokenizer = tokenizer.tokenizer # Use old chat template if the bos is removed if fix_bos_token: tokenizer.chat_template = old_chat_template pass + # Step 6: Clean up memory for _ in range(3): + import gc gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() - model_dtype = dtype_from_config(self.config) - model_type = self.config.model_type - if type(model_dtype) is str: - assert(model_dtype == "float16" or model_dtype == "bfloat16") - elif model_dtype == torch.float16: + # Step 7: Get model dtype and type + try: + model_dtype = dtype_from_config(self.config) + model_type = self.config.model_type + if type(model_dtype) is str: + assert(model_dtype == "float16" or model_dtype == "bfloat16") + elif model_dtype == torch.float16: + model_dtype = "float16" + elif model_dtype == torch.bfloat16: + model_dtype = "bfloat16" + else: + raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16") + except Exception as e: + # Fallback if dtype_from_config fails + print(f"Unsloth: Could not determine dtype ({e}), defaulting to float16") model_dtype = "float16" - elif model_dtype == torch.bfloat16: - model_dtype = "bfloat16" - else: - raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16") - pass - is_sentencepiece_model = check_if_sentencepiece_model(self) + # Step 8: Convert to GGUF format + print("Unsloth: Converting to GGUF format...") - # Save to GGUF - all_file_locations, want_full_precision = save_to_gguf( - model_type, model_dtype, is_sentencepiece_model, - new_save_directory, quantization_method, first_conversion, makefile, - ) - - # Save Ollama modelfile - modelfile = create_ollama_modelfile(tokenizer, all_file_locations[0]) - modelfile_location = None - if modelfile is not None: - modelfile_location = os.path.join(new_save_directory, "Modelfile") - with open(modelfile_location, "w", encoding = "utf-8") as file: - file.write(modelfile) + # Convert quantization_method to list if string + # Use old style quantization_method + quantization_methods = [] + if quantization_method is not None: + # Convert quantization_method to list + if isinstance(quantization_method, list): pass + elif isinstance(quantization_method, str): quantization_method = [ quantization_method, ] + elif isinstance(quantization_method, tuple): quantization_method = list(quantization_method) + else: + raise TypeError("Unsloth: quantization_method can only be a string or a list of strings") + pass + for i, quant_method in enumerate(quantization_method): + quant_method = quant_method.lower() + if quant_method == "not_quantized": quant_method = "f16" + elif quant_method == "fast_quantized": quant_method = "q8_0" + elif quant_method == "quantized": quant_method = "q4_k_m" + elif quant_method is None: quant_method = "q8_0" + quantization_methods.append(quant_method.lower()) pass - print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}") pass + try: + all_file_locations, want_full_precision, is_vlm_update = save_to_gguf( + model_name=model_name, + model_type=model_type, + model_dtype=model_dtype, + is_sentencepiece=False, + model_directory=save_directory, + quantization_method=quantization_methods, + first_conversion=first_conversion, + is_vlm=is_vlm, # Pass VLM flag + is_gpt_oss = is_gpt_oss, # Pass gpt_oss Flag + ) + except Exception as e: + if IS_KAGGLE_ENVIRONMENT: + raise RuntimeError( + f"Unsloth: GGUF conversion failed in Kaggle environment.\n" + f"This is likely due to the 20GB disk space limit.\n" + f"Try saving to /tmp directory or use a smaller model.\n" + f"Error: {e}" + ) + else: + raise RuntimeError(f"Unsloth: GGUF conversion failed: {e}") + + # Step 9: Create Ollama modelfile + modelfile_location = None + ollama_success = False + if all_file_locations: + try: + if is_vlm_update: + modelfile = create_ollama_modelfile(tokenizer, base_model_name, ".") + else: + modelfile = create_ollama_modelfile(tokenizer, base_model_name, all_file_locations[0]) + if modelfile is not None: + if is_vlm_update: + modelfile_location = os.path.join(save_directory, "Modelfile") + else: + modelfile_location = os.path.join(os.getcwd(), "Modelfile") + with open(modelfile_location, "w", encoding = "utf-8") as file: + file.write(modelfile) + ollama_success = True + except Exception as e: + print(f"Warning: Could not create Ollama modelfile: {e}") + + # Step 10: Show BOS token warning if applicable if fix_bos_token: logger.warning( "Unsloth: ##### The current model auto adds a BOS token.\n"\ @@ -1920,32 +1881,29 @@ def unsloth_save_pretrained_gguf( ) pass - if push_to_hub: - print("Unsloth: Uploading GGUF to Huggingface Hub...") + if is_vlm_update: + print("\n") + print(f"Unsloth: example usage for Multimodal LLMs: llama-mtmd-cli -m {all_file_locations[0]} --mmproj {all_file_locations[-1]}") + print("Unsloth: load image inside llama.cpp runner: /image test_image.jpg") + print("Unsloth: Prompt model to describe the image") + else: + print(f'Unsloth: example usage for text only LLMs: llama-cli --model {all_file_locations[0]} -p "why is the sky blue?"') + if ollama_success and is_vlm_update: + print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}") + print("Unsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside save directory.") + if ollama_success and not is_vlm_update: + print("Unsloth: Saved Ollama Modelfile to current directory") + print("Unsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside current directory.") - # If not needing full precision, skip the first - if not want_full_precision: all_file_locations = all_file_locations[1:] - - for file_location in all_file_locations: - username = upload_to_huggingface( - self, save_directory, token, - "GGUF converted", "gguf", file_location, old_username, private, - ) - link = f"{username}/{new_save_directory.lstrip('/.')}" \ - if username not in new_save_directory else \ - new_save_directory.lstrip('/.') - print(f"Saved GGUF to https://huggingface.co/{link}") - pass - - # Save modelfile - if modelfile_location is not None: - username = upload_to_huggingface( - self, save_directory, token, - "GGUF converted", "gguf", modelfile_location, old_username, private, - ) - print(f"Saved Ollama Modelfile to https://huggingface.co/{link}") - pass - pass + #Return a dict with all needed info for push_to_hub + return { + "save_directory": save_directory, + "gguf_files": all_file_locations, + "modelfile_location": modelfile_location, + "want_full_precision": want_full_precision, + "is_vlm": is_vlm_update, + "fix_bos_token": fix_bos_token, + } pass @@ -1953,7 +1911,7 @@ def unsloth_push_to_hub_gguf( self, repo_id : str, tokenizer = None, - quantization_method : str = "fast_quantized", + quantization_method = "fast_quantized", first_conversion : str = None, use_temp_dir : Optional[bool] = None, commit_message : Optional[str] = "Trained with Unsloth", @@ -1996,133 +1954,222 @@ def unsloth_push_to_hub_gguf( if tokenizer is None: raise ValueError("Unsloth: Saving to GGUF must have a tokenizer.") - arguments = dict(locals()) - arguments["model"] = self - arguments["tokenizer"] = tokenizer - arguments["save_directory"] = repo_id - arguments["push_to_hub"] = False # We save ourselves - arguments["save_method"] = "merged_16bit" # Must be 16bit - del arguments["self"] - del arguments["repo_id"] - del arguments["quantization_method"] - del arguments["first_conversion"] + # Step 1: Determine save directory + model_name = repo_id.split("/")[-1] if "/" in repo_id else repo_id - # Fix tokenizer adding an extra BOS token at the front - fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer) + if use_temp_dir or use_temp_dir is None: + import tempfile + temp_dir = tempfile.mkdtemp(prefix="unsloth_gguf_") + save_directory = temp_dir + cleanup_temp = True + else: + save_directory = model_name # Use model name, not repo_id + cleanup_temp = False - # Non blocking install GGUF first - if not os.path.exists("llama.cpp"): + # Step 2: Call save_pretrained_gguf to do the conversion + print(f"Unsloth: Converting model to GGUF format...") - if IS_KAGGLE_ENVIRONMENT: - # Kaggle is weird - no blocking installs, and no CUDA? - python_install = install_python_non_blocking(["gguf", "protobuf"]) - python_install.wait() - install_llama_cpp_blocking(use_cuda = False) - new_save_directory, old_username = unsloth_save_model(**arguments) - makefile = None + try: + # Call save_pretrained_gguf - it returns all the info we need + result = unsloth_save_pretrained_gguf( + self=self, + save_directory=save_directory, + tokenizer=tokenizer, + quantization_method=quantization_method, + first_conversion=first_conversion, + push_to_hub=False, # Never push from here + token=None, # Don't need token for local save + max_shard_size=max_shard_size, + safe_serialization=safe_serialization, + temporary_location=temporary_location, + maximum_memory_usage=maximum_memory_usage, + ) + + # Extract results + all_file_locations = result["gguf_files"] + modelfile_location = result["modelfile_location"] + want_full_precision = result["want_full_precision"] + is_vlm = result["is_vlm"] + fix_bos_token = result["fix_bos_token"] + actual_save_directory = result["save_directory"] + + except Exception as e: + if cleanup_temp: + import shutil + try: + shutil.rmtree(save_directory) + except: + pass + raise RuntimeError(f"Failed to convert model to GGUF: {e}") + pass + + # Step 3: Upload to HuggingFace Hub + print("Unsloth: Uploading GGUF to Huggingface Hub...") + + try: + from huggingface_hub import HfApi + api = HfApi(token=token) + + # Get full repo id + if "/" not in repo_id: + username = api.whoami()["name"] + full_repo_id = f"{username}/{repo_id}" else: - git_clone = install_llama_cpp_clone_non_blocking() - python_install = install_python_non_blocking(["gguf", "protobuf"]) - git_clone.wait() - makefile = install_llama_cpp_make_non_blocking() - new_save_directory, old_username = unsloth_save_model(**arguments) - python_install.wait() - pass - else: - try: - new_save_directory, old_username = unsloth_save_model(**arguments) - makefile = None - except: - # Retry by recloning llama.cpp - if IS_KAGGLE_ENVIRONMENT: - # Kaggle is weird - no blocking installs, and no CUDA? - python_install = install_python_non_blocking(["gguf", "protobuf"]) - python_install.wait() - install_llama_cpp_blocking(use_cuda = False) - new_save_directory, old_username = unsloth_save_model(**arguments) - makefile = None + full_repo_id = repo_id + + # Create repo + api.create_repo( + repo_id=full_repo_id, + repo_type="model", + private=private, + exist_ok=True, + ) + + # Upload GGUF files + for file_location in all_file_locations: + original_name = os.path.basename(file_location) + # Replace temp directory name with proper model name + if cleanup_temp and "unsloth_gguf_" in original_name: + # Extract the quantization part (e.g., ".Q8_0.gguf" or ".Q8_0-mmproj.gguf") + quant_suffix = original_name.split(".", 1)[1] if "." in original_name else original_name + proper_name = f"{model_name}.{quant_suffix}" else: - git_clone = install_llama_cpp_clone_non_blocking() - python_install = install_python_non_blocking(["gguf", "protobuf"]) - git_clone.wait() - makefile = install_llama_cpp_make_non_blocking() - new_save_directory, old_username = unsloth_save_model(**arguments) - python_install.wait() + proper_name = original_name.replace(os.path.basename(save_directory), model_name) + + print(f"Uploading {proper_name}...") + + api.upload_file( + path_or_fileobj=file_location, + path_in_repo=proper_name, + repo_id=full_repo_id, + repo_type="model", + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + revision=revision, + ) + pass + + # Upload config.json if exists + config_path = os.path.join(actual_save_directory, "config.json") + if os.path.exists(config_path): + print("Uploading config.json...") + api.upload_file( + path_or_fileobj=config_path, + path_in_repo="config.json", + repo_id=full_repo_id, + repo_type="model", + commit_message=f"{commit_message} - config", + create_pr=create_pr, + revision=revision, + ) + + # Upload Modelfile if exists + if modelfile_location and os.path.exists(modelfile_location): + print("Uploading Ollama Modelfile...") + api.upload_file( + path_or_fileobj=modelfile_location, + path_in_repo="Modelfile", + repo_id=full_repo_id, + repo_type="model", + commit_message=f"{commit_message} - Ollama Modelfile", + create_pr=create_pr, + revision=revision, + ) + + # Create and upload README + readme_content = f"""--- +tags: +- gguf +- llama.cpp +- unsloth +{"- vision-language-model" if is_vlm else ""} +--- + +# {repo_id.split("/")[-1]} - GGUF + +This model was finetuned and converted to GGUF format using [Unsloth](https://github.com/unslothai/unsloth). + +**Example usage**: +- For text only LLMs: **llama-cli** **--hf** repo_id/model_name **-p** "why is the sky blue?" +- For multimodal models: **llama-mtmd-cli** **-m** model_name.gguf **--mmproj** mmproj_file.gguf + +## Available Model files: +""" + for file in all_file_locations: + # Fix filename in README too + original_name = os.path.basename(file) + if cleanup_temp and "unsloth_gguf_" in original_name: + quant_suffix = original_name.split(".", 1)[1] if "." in original_name else original_name + proper_name = f"{model_name}.{quant_suffix}" + else: + proper_name = original_name.replace(os.path.basename(save_directory), model_name) + readme_content += f"- `{proper_name}`\n" + + # Special note for VLM with Modelfile + if is_vlm and modelfile_location: + readme_content += "\n## ⚠️ Ollama Note for Vision Models\n" + readme_content += "**Important:** Ollama currently does not support separate mmproj files for vision models.\n\n" + readme_content += "To create an Ollama model from this vision model:\n" + readme_content += "1. Place the `Modelfile` in the same directory as the finetuned bf16 merged model\n" + readme_content += "3. Run: `ollama create model_name -f ./Modelfile`\n" + readme_content += " (Replace `model_name` with your desired name)\n\n" + readme_content += "This will create a unified bf16 model that Ollama can use.\n" + elif modelfile_location: + readme_content += "\n## Ollama\n" + readme_content += "An Ollama Modelfile is included for easy deployment.\n" + + + if fix_bos_token: + readme_content += "\n## Note\n" + readme_content += "The model's BOS token behavior was adjusted for GGUF compatibility.\n" + + readme_path = os.path.join(actual_save_directory, "README.md") + with open(readme_path, "w") as f: + f.write(readme_content) + + api.upload_file( + path_or_fileobj=readme_path, + path_in_repo="README.md", + repo_id=full_repo_id, + repo_type="model", + commit_message="Add README", + create_pr=create_pr, + revision=revision, + ) + + print(f"Unsloth: Successfully uploaded GGUF to https://huggingface.co/{full_repo_id}") + + # Add tags + if tags is None: + tags = [] + tags.extend(["gguf", "llama-cpp", "unsloth"]) + if is_vlm: + tags.append("vision-language-model") + + try: + api.add_tags( + repo_id=full_repo_id, + tags=tags, + repo_type="model", + ) + except: pass - pass - pass - # Use old chat template if the bos is removed - if fix_bos_token: - tokenizer.chat_template = old_chat_template - pass + except Exception as e: + raise RuntimeError(f"Failed to upload to Hugging Face Hub: {e}") - for _ in range(3): - gc.collect() + finally: + # Clean up temporary directory + if cleanup_temp and os.path.exists(save_directory): + print("Unsloth: Cleaning up temporary files...") + import shutil + try: + shutil.rmtree(save_directory) + except: + pass - model_dtype = dtype_from_config(self.config) - model_type = self.config.model_type - if type(model_dtype) is str: - assert(model_dtype == "float16" or model_dtype == "bfloat16") - elif model_dtype == torch.float16: - model_dtype = "float16" - elif model_dtype == torch.bfloat16: - model_dtype = "bfloat16" - else: - raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16") - pass - - is_sentencepiece_model = check_if_sentencepiece_model(self) - - # Save to GGUF - all_file_locations, want_full_precision = save_to_gguf( - model_type, model_dtype, is_sentencepiece_model, - new_save_directory, quantization_method, first_conversion, makefile, - ) - - # Save Ollama modelfile - modelfile = create_ollama_modelfile(tokenizer, all_file_locations[0]) - modelfile_location = None - if modelfile is not None: - modelfile_location = os.path.join(new_save_directory, "Modelfile") - with open(modelfile_location, "w", encoding = "utf-8") as file: - file.write(modelfile) - pass - print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}") - pass - - # If not needing full precision, skip the first - if not want_full_precision: all_file_locations = all_file_locations[1:] - - for file_location in all_file_locations: - print("Unsloth: Uploading GGUF to Huggingface Hub...") - username = upload_to_huggingface( - self, repo_id, token, - "GGUF converted", "gguf", file_location, old_username, private, - ) - link = f"{username}/{new_save_directory.lstrip('/.')}" \ - if username not in new_save_directory else \ - new_save_directory.lstrip('/.') - - print(f"Saved GGUF to https://huggingface.co/{link}") - pass - - # Save modelfile - if modelfile_location is not None: - username = upload_to_huggingface( - self, repo_id, token, - "GGUF converted", "gguf", modelfile_location, old_username, private, - ) - print(f"Saved Ollama Modelfile to https://huggingface.co/{link}") - pass - - if fix_bos_token: - - logger.warning( - "Unsloth: ##### The current model auto adds a BOS token.\n"\ - "Unsloth: ##### We removed it in GGUF's chat template for you." - ) - pass + return full_repo_id pass @@ -2676,8 +2723,8 @@ def patch_saving_functions(model, vision = False): # Vision only 1 option model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model) model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, model) - model.push_to_hub_gguf = types.MethodType(save_to_gguf_generic, model) - model.save_pretrained_gguf = types.MethodType(save_to_gguf_generic, model) + model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) + model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model) model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model) pass return model