From 62a1ac9d33788e6ff74b5a2da2fc91b1d0034a8e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:12:16 +0000 Subject: [PATCH] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../test_merge_4bit_validation.py | 2 +- .../test_save_merged_grpo_model.py | 1 - .../saving/non_peft/test_mistral_non_peft.py | 2 - .../saving/non_peft/test_whisper_non_peft.py | 5 +- .../test_index_file_sharded_model.py | 18 +- .../vision_models/test_push_to_hub_merged.py | 18 +- ..._merge_qwen2.5vl32B_model_ocr_benchmark.py | 10 - ...t_save_merge_vision_model_ocr_benchmark.py | 10 - tests/test_raw_text.py | 1 - tests/utils/aime_eval.py | 12 +- tests/utils/hf_utils.py | 5 +- tests/utils/ocr_eval.py | 2 +- tests/utils/os_utils.py | 8 +- unsloth-cli.py | 2 - unsloth/__init__.py | 12 +- unsloth/_auto_install.py | 74 +- unsloth/chat_templates.py | 1597 +++++++++++------ unsloth/dataprep/raw_text.py | 2 - unsloth/dataprep/synthetic.py | 1 - unsloth/import_fixes.py | 2 +- unsloth/kernels/cross_entropy_loss.py | 2 - unsloth/kernels/fast_lora.py | 1 - unsloth/kernels/flex_attention.py | 5 +- unsloth/kernels/fp8.py | 5 +- unsloth/kernels/geglu.py | 1 - unsloth/kernels/layernorm.py | 3 - unsloth/kernels/moe/autotune_cache.py | 3 +- unsloth/kernels/moe/benchmark/utils.py | 2 +- unsloth/kernels/moe/grouped_gemm/interface.py | 1 - .../moe/grouped_gemm/kernels/autotuning.py | 1 - .../moe/grouped_gemm/kernels/backward.py | 1 - .../moe/grouped_gemm/kernels/forward.py | 1 - .../kernels/moe/tests/test_grouped_gemm.py | 2 - unsloth/kernels/moe/tests/test_llama4_moe.py | 1 - unsloth/kernels/swiglu.py | 2 +- unsloth/kernels/utils.py | 5 - unsloth/models/_utils.py | 49 +- unsloth/models/cohere.py | 4 - unsloth/models/falcon_h1.py | 4 - unsloth/models/gemma.py | 9 - unsloth/models/gemma2.py | 7 +- unsloth/models/glm4_moe.py | 9 - unsloth/models/granite.py | 5 +- unsloth/models/llama.py | 14 +- unsloth/models/loader.py | 21 +- unsloth/models/loader_utils.py | 1 - unsloth/models/mapper.py | 601 +++---- unsloth/models/mistral.py | 2 - unsloth/models/qwen3.py | 7 +- unsloth/models/qwen3_moe.py | 3 - unsloth/models/rl.py | 3 - unsloth/models/rl_replacements.py | 5 - unsloth/models/sentence_transformer.py | 4 +- unsloth/models/vision.py | 30 +- unsloth/save.py | 17 +- unsloth/tokenizer_utils.py | 16 +- unsloth/trainer.py | 5 - 57 files changed, 1447 insertions(+), 1189 deletions(-) diff --git a/tests/saving/language_models/test_merge_4bit_validation.py b/tests/saving/language_models/test_merge_4bit_validation.py index 343e737710..9669b92a3b 100644 --- a/tests/saving/language_models/test_merge_4bit_validation.py +++ b/tests/saving/language_models/test_merge_4bit_validation.py @@ -1,6 +1,6 @@ from unsloth import FastLanguageModel from unsloth.chat_templates import get_chat_template -from trl import SFTTrainer, SFTConfig +from trl import SFTTrainer from transformers import DataCollatorForSeq2Seq, TrainingArguments from datasets import load_dataset import torch diff --git a/tests/saving/language_models/test_save_merged_grpo_model.py b/tests/saving/language_models/test_save_merged_grpo_model.py index 67b649305a..9625bbaec6 100644 --- a/tests/saving/language_models/test_save_merged_grpo_model.py +++ b/tests/saving/language_models/test_save_merged_grpo_model.py @@ -11,7 +11,6 @@ import sys from pathlib import Path import multiprocessing as mp import gc -from multiprocessing import Queue REPO_ROOT = Path(__file__).parents[3] sys.path.insert(0, str(REPO_ROOT)) diff --git a/tests/saving/non_peft/test_mistral_non_peft.py b/tests/saving/non_peft/test_mistral_non_peft.py index e03813367d..7f28334f5d 100644 --- a/tests/saving/non_peft/test_mistral_non_peft.py +++ b/tests/saving/non_peft/test_mistral_non_peft.py @@ -1,6 +1,4 @@ from unsloth import FastLanguageModel -from transformers import AutoModelForCausalLM -from peft import PeftModel from pathlib import Path import sys import warnings diff --git a/tests/saving/non_peft/test_whisper_non_peft.py b/tests/saving/non_peft/test_whisper_non_peft.py index 303d596c85..98af05f176 100644 --- a/tests/saving/non_peft/test_whisper_non_peft.py +++ b/tests/saving/non_peft/test_whisper_non_peft.py @@ -1,6 +1,5 @@ -from unsloth import FastLanguageModel, FastModel -from transformers import AutoModelForCausalLM, WhisperForConditionalGeneration -from peft import PeftModel +from unsloth import FastModel +from transformers import WhisperForConditionalGeneration from pathlib import Path import sys import warnings diff --git a/tests/saving/vision_models/test_index_file_sharded_model.py b/tests/saving/vision_models/test_index_file_sharded_model.py index f737169841..420125c441 100644 --- a/tests/saving/vision_models/test_index_file_sharded_model.py +++ b/tests/saving/vision_models/test_index_file_sharded_model.py @@ -28,7 +28,7 @@ train_dataset = dataset.select(range(2000)) # To select the next 200 examples for evaluation eval_dataset = dataset.select(range(2000, 2200)) -print(f"āœ… Dataset loaded successfully!") +print("āœ… Dataset loaded successfully!") print(f" šŸ“ˆ Training samples: {len(train_dataset)}") print(f" šŸ“Š Evaluation samples: {len(eval_dataset)}") @@ -110,10 +110,10 @@ try: loftq_config = None, # And LoftQ ) print("āœ… LoRA configuration applied successfully!") - print(f" šŸŽÆ LoRA rank (r): 16") - print(f" šŸ“Š LoRA alpha: 32") - print(f" šŸ” Vision layers: Enabled") - print(f" šŸ’¬ Language layers: Enabled") + print(" šŸŽÆ LoRA rank (r): 16") + print(" šŸ“Š LoRA alpha: 32") + print(" šŸ” Vision layers: Enabled") + print(" šŸ’¬ Language layers: Enabled") except Exception as e: print(f"āŒ Failed to apply LoRA configuration: {e}") raise @@ -165,10 +165,10 @@ try: ), ) print("āœ… Trainer setup completed!") - print(f" šŸ“¦ Batch size: 2") - print(f" šŸ”„ Gradient accumulation steps: 4") - print(f" šŸ“ˆ Max training steps: 10") - print(f" šŸŽÆ Learning rate: 2e-4") + print(" šŸ“¦ Batch size: 2") + print(" šŸ”„ Gradient accumulation steps: 4") + print(" šŸ“ˆ Max training steps: 10") + print(" šŸŽÆ Learning rate: 2e-4") print(f" šŸ’¾ Precision: {'BF16' if is_bf16_supported() else 'FP16'}") except Exception as e: print(f"āŒ Failed to setup trainer: {e}") diff --git a/tests/saving/vision_models/test_push_to_hub_merged.py b/tests/saving/vision_models/test_push_to_hub_merged.py index 74fa058988..1e2b53d8ff 100644 --- a/tests/saving/vision_models/test_push_to_hub_merged.py +++ b/tests/saving/vision_models/test_push_to_hub_merged.py @@ -29,7 +29,7 @@ train_dataset = dataset.select(range(2000)) # To select the next 200 examples for evaluation eval_dataset = dataset.select(range(2000, 2200)) -print(f"āœ… Dataset loaded successfully!") +print("āœ… Dataset loaded successfully!") print(f" šŸ“ˆ Training samples: {len(train_dataset)}") print(f" šŸ“Š Evaluation samples: {len(eval_dataset)}") @@ -111,10 +111,10 @@ try: loftq_config = None, # And LoftQ ) print("āœ… LoRA configuration applied successfully!") - print(f" šŸŽÆ LoRA rank (r): 16") - print(f" šŸ“Š LoRA alpha: 32") - print(f" šŸ” Vision layers: Enabled") - print(f" šŸ’¬ Language layers: Enabled") + print(" šŸŽÆ LoRA rank (r): 16") + print(" šŸ“Š LoRA alpha: 32") + print(" šŸ” Vision layers: Enabled") + print(" šŸ’¬ Language layers: Enabled") except Exception as e: print(f"āŒ Failed to apply LoRA configuration: {e}") raise @@ -166,10 +166,10 @@ try: ), ) print("āœ… Trainer setup completed!") - print(f" šŸ“¦ Batch size: 2") - print(f" šŸ”„ Gradient accumulation steps: 4") - print(f" šŸ“ˆ Max training steps: 10") - print(f" šŸŽÆ Learning rate: 2e-4") + print(" šŸ“¦ Batch size: 2") + print(" šŸ”„ Gradient accumulation steps: 4") + print(" šŸ“ˆ Max training steps: 10") + print(" šŸŽÆ Learning rate: 2e-4") print(f" šŸ’¾ Precision: {'BF16' if is_bf16_supported() else 'FP16'}") except Exception as e: print(f"āŒ Failed to setup trainer: {e}") diff --git a/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py b/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py index ebe078c73b..33f76c55c3 100644 --- a/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py +++ b/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py @@ -2,9 +2,6 @@ from unsloth import FastVisionModel -import torch -from qwen_vl_utils import process_vision_info -import os from datasets import load_dataset from trl import SFTTrainer, SFTConfig @@ -20,7 +17,6 @@ from tests.utils.ocr_eval import OCRModelEvaluator ## Dataset Preparation -from datasets import load_dataset dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train") # To select the first 2000 examples @@ -66,12 +62,6 @@ train_dataset = [format_data(sample) for sample in train_dataset] eval_dataset = [format_data(sample) for sample in eval_dataset] ## Setup OCR main evaluation function and helpers -import os -import torch -from tqdm import tqdm -import pandas as pd -from jiwer import wer, cer -from qwen_vl_utils import process_vision_info # ocr_evaluator = OCRModelEvaluator() diff --git a/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py b/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py index b99785bcb1..5722b7697a 100644 --- a/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py +++ b/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py @@ -2,9 +2,6 @@ from unsloth import FastVisionModel -import torch -from qwen_vl_utils import process_vision_info -import os from datasets import load_dataset from trl import SFTTrainer, SFTConfig @@ -20,7 +17,6 @@ from tests.utils.ocr_eval import OCRModelEvaluator ## Dataset Preparation -from datasets import load_dataset dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train") # To select the first 2000 examples @@ -66,12 +62,6 @@ train_dataset = [format_data(sample) for sample in train_dataset] eval_dataset = [format_data(sample) for sample in eval_dataset] ## Setup OCR main evaluation function and helpers -import os -import torch -from tqdm import tqdm -import pandas as pd -from jiwer import wer, cer -from qwen_vl_utils import process_vision_info # ocr_evaluator = OCRModelEvaluator() diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 9f2e8cda4e..e580e97b69 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -7,7 +7,6 @@ Tests basic functionality without heavy dependencies. import sys import os import tempfile -from pathlib import Path import importlib.util diff --git a/tests/utils/aime_eval.py b/tests/utils/aime_eval.py index 131da3e50b..6a80b16ce8 100644 --- a/tests/utils/aime_eval.py +++ b/tests/utils/aime_eval.py @@ -207,7 +207,7 @@ def evaluate_model_aime( print(f"\n{'='*70}") print(f"šŸ§® AIME EVALUATION - {model_type.upper()} MODEL") - print(f"Combined Dataset: test2024 + test2025-I + test2025-II") + print("Combined Dataset: test2024 + test2025-I + test2025-II") print(f"{'='*70}") # Load combined AIME dataset @@ -244,7 +244,7 @@ def evaluate_model_aime( seed = seed, ) - print(f"\nšŸ”§ Configuration:") + print("\nšŸ”§ Configuration:") print(f" Temperature: {temperature}") print(f" Samples per question: {n_sampling}") print(f" Max tokens: {max_tokens}") @@ -421,28 +421,28 @@ def evaluate_model_aime( print(f"šŸ“Š AIME EVALUATION RESULTS - {model_type.upper()}") print(f"{'='*70}") - print(f"\nšŸŽÆ Overall Performance:") + print("\nšŸŽÆ Overall Performance:") print(f" Total problems: {total_problems:>6}") print( f" Correct answers: {correct_answers:>6}/{total_problems} ({accuracy:>5.1f}%)" ) print(f" Pass@{n_sampling}: {pass_at_k:>10.1f}%") - print(f"\nšŸ“ˆ Performance by Dataset:") + print("\nšŸ“ˆ Performance by Dataset:") for source, stats in source_stats.items(): source_acc = source_accuracies[source] print( f" {source:>12}: {stats['correct']:>3}/{stats['total']:>3} ({source_acc:>5.1f}%)" ) - print(f"\nšŸ”§ Configuration:") + print("\nšŸ”§ Configuration:") print(f" Temperature: {temperature}") print(f" Samples per problem: {n_sampling}") print(f" Max tokens: {max_tokens}") print(f" Top-p: {top_p}") print(f" Seed: {seed}") - print(f"\nšŸ“ Token Statistics:") + print("\nšŸ“ Token Statistics:") print(f" Avg input tokens: {results['avg_input_tokens']:>10.1f}") print(f" Avg output tokens: {results['avg_output_tokens']:>10.1f}") print(f" Max input tokens: {results['max_input_tokens']:>10}") diff --git a/tests/utils/hf_utils.py b/tests/utils/hf_utils.py index 8ad6d5ad08..02b0192748 100644 --- a/tests/utils/hf_utils.py +++ b/tests/utils/hf_utils.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -from contextlib import contextmanager, nullcontext -from typing import Callable, Optional +from contextlib import nullcontext +from typing import Callable import bitsandbytes as bnb import torch diff --git a/tests/utils/ocr_eval.py b/tests/utils/ocr_eval.py index 3c5cd74a22..eff316a094 100644 --- a/tests/utils/ocr_eval.py +++ b/tests/utils/ocr_eval.py @@ -348,7 +348,7 @@ class OCRModelEvaluator: plt.savefig("ocr_model_comparison.png") plt.show() - print(f"\nVisualization saved to ocr_model_comparison.png") + print("\nVisualization saved to ocr_model_comparison.png") def get_comparison_results(self) -> Dict[str, Dict[str, float]]: """Get the current comparison results.""" diff --git a/tests/utils/os_utils.py b/tests/utils/os_utils.py index 448f13b8a0..a76627946f 100644 --- a/tests/utils/os_utils.py +++ b/tests/utils/os_utils.py @@ -98,10 +98,10 @@ def require_package(package_name, executable_name = None): for pm_name, cmd in install_commands.items(): print(f" {pm_name}: {cmd}") - print(f"\nAlternatively, install with conda:") + print("\nAlternatively, install with conda:") print(f" conda install -c conda-forge {package_name}") - print(f"\nPlease install the required package and run the script again.") + print("\nPlease install the required package and run the script again.") sys.exit(1) @@ -120,9 +120,9 @@ def require_python_package(package_name, import_name = None, pip_name = None): print(f"āŒ Error: Python package '{package_name}' is not installed") print(f"\nPlease install {package_name} using pip:") print(f" pip install {pip_name}") - print(f" # or with conda:") + print(" # or with conda:") print(f" conda install {pip_name}") - print(f"\nAfter installation, run this script again.") + print("\nAfter installation, run this script again.") sys.exit(1) else: print(f"āœ“ Python package '{package_name}' is installed") diff --git a/unsloth-cli.py b/unsloth-cli.py index 612da11eb2..3675d2ed02 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -101,8 +101,6 @@ def run(args): return {"text": texts} def load_dataset_smart(args): - from transformers.utils import strtobool - if args.raw_text_file: # Use raw text loader loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index fad37a786d..b2fe0124ee 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -12,9 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import warnings, importlib, sys +import warnings +import importlib +import sys from packaging.version import Version -import os, re, subprocess, inspect, functools +import os +import re +import subprocess +import inspect +import functools import numpy as np # Log Unsloth is being used @@ -95,7 +101,7 @@ try: import unsloth_zoo except PackageNotFoundError: raise ImportError( - f"Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo` then retry!" + "Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo` then retry!" ) except: raise diff --git a/unsloth/_auto_install.py b/unsloth/_auto_install.py index f6deefeb33..73440595cd 100644 --- a/unsloth/_auto_install.py +++ b/unsloth/_auto_install.py @@ -12,32 +12,60 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: import torch -except: raise ImportError('Install torch via `pip install torch`') +try: + import torch +except: + raise ImportError("Install torch via `pip install torch`") from packaging.version import Version as V import re + v = V(re.match(r"[0-9\.]{3,}", torch.__version__).group(0)) cuda = str(torch.version.cuda) is_ampere = torch.cuda.get_device_capability()[0] >= 8 USE_ABI = torch._C._GLIBCXX_USE_CXX11_ABI -if cuda not in ("11.8", "12.1", "12.4", "12.6", "12.8", "13.0"): raise RuntimeError(f"CUDA = {cuda} not supported!") -if v <= V('2.1.0'): raise RuntimeError(f"Torch = {v} too old!") -elif v <= V('2.1.1'): x = 'cu{}{}-torch211' -elif v <= V('2.1.2'): x = 'cu{}{}-torch212' -elif v < V('2.3.0'): x = 'cu{}{}-torch220' -elif v < V('2.4.0'): x = 'cu{}{}-torch230' -elif v < V('2.5.0'): x = 'cu{}{}-torch240' -elif v < V('2.5.1'): x = 'cu{}{}-torch250' -elif v <= V('2.5.1'): x = 'cu{}{}-torch251' -elif v < V('2.7.0'): x = 'cu{}{}-torch260' -elif v < V('2.7.9'): x = 'cu{}{}-torch270' -elif v < V('2.8.0'): x = 'cu{}{}-torch271' -elif v < V('2.8.9'): x = 'cu{}{}-torch280' -elif v < V('2.9.1'): x = 'cu{}{}-torch290' -elif v < V('2.9.2'): x = 'cu{}{}-torch291' -elif v < V('2.10.1'): x = 'cu{}{}-torch2100' -else: raise RuntimeError(f"Torch = {v} too new!") -if v > V('2.6.9') and cuda not in ("11.8", "12.6", "12.8", "13.0"): raise RuntimeError(f"CUDA = {cuda} not supported!") -if v >= V('2.10.0') and cuda not in ("12.6", "12.8", "13.0"): raise RuntimeError(f"Torch 2.10 requires CUDA 12.6, 12.8, or 13.0! Got CUDA = {cuda}") -x = x.format(cuda.replace(".", ""), "-ampere" if False else "") # is_ampere is broken due to flash-attn -print(f'pip install --upgrade pip && pip install --no-deps git+https://github.com/unslothai/unsloth-zoo.git && pip install "unsloth[{x}] @ git+https://github.com/unslothai/unsloth.git" --no-build-isolation') \ No newline at end of file +if cuda not in ("11.8", "12.1", "12.4", "12.6", "12.8", "13.0"): + raise RuntimeError(f"CUDA = {cuda} not supported!") +if v <= V("2.1.0"): + raise RuntimeError(f"Torch = {v} too old!") +elif v <= V("2.1.1"): + x = "cu{}{}-torch211" +elif v <= V("2.1.2"): + x = "cu{}{}-torch212" +elif v < V("2.3.0"): + x = "cu{}{}-torch220" +elif v < V("2.4.0"): + x = "cu{}{}-torch230" +elif v < V("2.5.0"): + x = "cu{}{}-torch240" +elif v < V("2.5.1"): + x = "cu{}{}-torch250" +elif v <= V("2.5.1"): + x = "cu{}{}-torch251" +elif v < V("2.7.0"): + x = "cu{}{}-torch260" +elif v < V("2.7.9"): + x = "cu{}{}-torch270" +elif v < V("2.8.0"): + x = "cu{}{}-torch271" +elif v < V("2.8.9"): + x = "cu{}{}-torch280" +elif v < V("2.9.1"): + x = "cu{}{}-torch290" +elif v < V("2.9.2"): + x = "cu{}{}-torch291" +elif v < V("2.10.1"): + x = "cu{}{}-torch2100" +else: + raise RuntimeError(f"Torch = {v} too new!") +if v > V("2.6.9") and cuda not in ("11.8", "12.6", "12.8", "13.0"): + raise RuntimeError(f"CUDA = {cuda} not supported!") +if v >= V("2.10.0") and cuda not in ("12.6", "12.8", "13.0"): + raise RuntimeError( + f"Torch 2.10 requires CUDA 12.6, 12.8, or 13.0! Got CUDA = {cuda}" + ) +x = x.format( + cuda.replace(".", ""), "-ampere" if False else "" +) # is_ampere is broken due to flash-attn +print( + f'pip install --upgrade pip && pip install --no-deps git+https://github.com/unslothai/unsloth-zoo.git && pip install "unsloth[{x}] @ git+https://github.com/unslothai/unsloth.git" --no-build-isolation' +) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 63d310af8e..d6240ab7f1 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -17,13 +17,11 @@ __all__ = [ "test_chat_templates", "test_hf_gguf_equivalence", "remove_special_tokens", - "to_sharegpt", "standardize_sharegpt", "standardize_data_formats", "apply_chat_template", "train_on_responses_only", - "test_construct_chat_template", ] @@ -32,7 +30,6 @@ from torch import LongTensor, FloatTensor from transformers.models.llama.modeling_llama import logger from .save import patch_saving_functions import os -import shutil from .tokenizer_utils import * from .models._utils import patch_tokenizer import re @@ -40,36 +37,37 @@ from unsloth_zoo.dataset_utils import ( train_on_responses_only, standardize_data_formats, ) + standardize_sharegpt = standardize_data_formats CHAT_TEMPLATES = {} DEFAULT_SYSTEM_MESSAGE = {} # =========================================== Unsloth # Unsloth efficient template leverages from Zephyr -unsloth_template = \ - "{{ bos_token }}"\ - "{% if messages[0]['role'] == 'system' %}"\ - "{{ messages[0]['content'] + '\n' }}"\ - "{% set loop_messages = messages[1:] %}"\ - "{% else %}"\ - "{{ '{system_message}' + '\n' }}"\ - "{% set loop_messages = messages %}"\ - "{% endif %}"\ - "{% for message in loop_messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ '>>> User: ' + message['content'] + '\n' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ '>>> Assistant: ' + message['content'] + eos_token + '\n' }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '>>> Assistant: ' }}"\ +unsloth_template = ( + "{{ bos_token }}" + "{% if messages[0]['role'] == 'system' %}" + "{{ messages[0]['content'] + '\n' }}" + "{% set loop_messages = messages[1:] %}" + "{% else %}" + "{{ '{system_message}' + '\n' }}" + "{% set loop_messages = messages %}" "{% endif %}" + "{% for message in loop_messages %}" + "{% if message['role'] == 'user' %}" + "{{ '>>> User: ' + message['content'] + '\n' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ '>>> Assistant: ' + message['content'] + eos_token + '\n' }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" + "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '>>> Assistant: ' }}" + "{% endif %}" +) -unsloth_ollama = \ -''' +unsloth_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}>>> User: {{ .Prompt }} @@ -82,27 +80,32 @@ SYSTEM """You are a helpful assistant to the user""" ''' unsloth_eos_token = "eos_token" -CHAT_TEMPLATES["unsloth"] = (unsloth_template, unsloth_eos_token, False, unsloth_ollama,) +CHAT_TEMPLATES["unsloth"] = ( + unsloth_template, + unsloth_eos_token, + False, + unsloth_ollama, +) DEFAULT_SYSTEM_MESSAGE["unsloth"] = "You are a helpful assistant to the user" # =========================================== Zephyr # Zephyr has no BOS! -zephyr_template = \ - "{% for message in messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ '<|user|>\n' + message['content'] + eos_token + '\n' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ '<|assistant|>\n' + message['content'] + eos_token + '\n' }}"\ - "{% else %}"\ - "{{ '<|system|>\n' + message['content'] + eos_token + '\n' }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '<|assistant|>\n' }}"\ +zephyr_template = ( + "{% for message in messages %}" + "{% if message['role'] == 'user' %}" + "{{ '<|user|>\n' + message['content'] + eos_token + '\n' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ '<|assistant|>\n' + message['content'] + eos_token + '\n' }}" + "{% else %}" + "{{ '<|system|>\n' + message['content'] + eos_token + '\n' }}" "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '<|assistant|>\n' }}" + "{% endif %}" +) -zephyr_ollama = \ -''' +zephyr_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}<|system|> {{ .System }}{__EOS_TOKEN__} @@ -117,27 +120,32 @@ PARAMETER min_p 0.1 ''' zephyr_eos_token = "eos_token" -CHAT_TEMPLATES["zephyr"] = (zephyr_template, zephyr_eos_token, False, zephyr_ollama,) -DEFAULT_SYSTEM_MESSAGE["zephyr"] = None # No system message in Zephyr +CHAT_TEMPLATES["zephyr"] = ( + zephyr_template, + zephyr_eos_token, + False, + zephyr_ollama, +) +DEFAULT_SYSTEM_MESSAGE["zephyr"] = None # No system message in Zephyr # =========================================== ChatML # ChatML has no BOS and not EOS! Rather <|im_start|> and <|im_end|> acts as BOS / EOS. -chatml_template = \ - "{% for message in messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{'<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n' }}"\ - "{% else %}"\ - "{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '<|im_start|>assistant\n' }}"\ +chatml_template = ( + "{% for message in messages %}" + "{% if message['role'] == 'user' %}" + "{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}" + "{% elif message['role'] == 'assistant' %}" + "{{'<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n' }}" + "{% else %}" + "{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}" "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '<|im_start|>assistant\n' }}" + "{% endif %}" +) -chatml_ollama = \ -''' +chatml_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}<|im_start|>system {{ .System }}<|im_end|> @@ -153,37 +161,42 @@ PARAMETER min_p 0.1 ''' chatml_eos_token = "<|im_end|>" -CHAT_TEMPLATES["chatml"] = (chatml_template, chatml_eos_token, True, chatml_ollama,) -DEFAULT_SYSTEM_MESSAGE["chatml"] = None # No system message in ChatML +CHAT_TEMPLATES["chatml"] = ( + chatml_template, + chatml_eos_token, + True, + chatml_ollama, +) +DEFAULT_SYSTEM_MESSAGE["chatml"] = None # No system message in ChatML # =========================================== Mistral-1 # Mistral Instruct doesn't allow system prompts, so we append it to the user message. -mistral_template = \ - "{{ bos_token }}"\ - "{% if messages[0]['role'] == 'system' %}"\ - "{% if messages[1]['role'] == 'user' %}"\ - "{{ '[INST] ' + messages[0]['content'] + ' ' + messages[1]['content'] + ' [/INST]' }}"\ - "{% set loop_messages = messages[2:] %}"\ - "{% else %}"\ - "{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}"\ - "{% set loop_messages = messages[1:] %}"\ - "{% endif %}"\ - "{% else %}"\ - "{% set loop_messages = messages %}"\ - "{% endif %}"\ - "{% for message in loop_messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ '[INST] ' + message['content'] + ' [/INST]' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ message['content'] + eos_token }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ +mistral_template = ( + "{{ bos_token }}" + "{% if messages[0]['role'] == 'system' %}" + "{% if messages[1]['role'] == 'user' %}" + "{{ '[INST] ' + messages[0]['content'] + ' ' + messages[1]['content'] + ' [/INST]' }}" + "{% set loop_messages = messages[2:] %}" + "{% else %}" + "{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}" + "{% set loop_messages = messages[1:] %}" + "{% endif %}" + "{% else %}" + "{% set loop_messages = messages %}" + "{% endif %}" + "{% for message in loop_messages %}" + "{% if message['role'] == 'user' %}" + "{{ '[INST] ' + message['content'] + ' [/INST]' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ message['content'] + eos_token }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" + "{% endif %}" "{% endfor %}" +) # Ollama from https://www.ollama.com/library/mistral -mistral_ollama = \ -''' +mistral_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """[INST] {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} [/INST]""" PARAMETER stop "{__EOS_TOKEN__}" @@ -192,36 +205,41 @@ PARAMETER min_p 0.1 ''' mistral_eos_token = "eos_token" -CHAT_TEMPLATES["mistral"] = (mistral_template, mistral_eos_token, False, mistral_ollama,) -DEFAULT_SYSTEM_MESSAGE["mistral"] = None # No system message in Mistral +CHAT_TEMPLATES["mistral"] = ( + mistral_template, + mistral_eos_token, + False, + mistral_ollama, +) +DEFAULT_SYSTEM_MESSAGE["mistral"] = None # No system message in Mistral # =========================================== Llama-2 # Adds BOS to every convo! And weird <> system messages. -llama_template = \ - "{% if messages[0]['role'] == 'system' %}"\ - "{% if messages[1]['role'] == 'user' %}"\ - "{{ bos_token + '[INST] <>\n' + messages[0]['content'] + '\n<>\n\n' + messages[1]['content'] + ' [/INST]' }}"\ - "{% set loop_messages = messages[2:] %}"\ - "{% else %}"\ - "{{ bos_token + '[INST] ' + messages[0]['content'] + ' [/INST]' }}"\ - "{% set loop_messages = messages[1:] %}"\ - "{% endif %}"\ - "{% else %}"\ - "{% set loop_messages = messages %}"\ - "{% endif %}"\ - "{% for message in loop_messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ bos_token + '[INST] ' + message['content'].strip() + ' [/INST]' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ ' ' + message['content'].strip() + ' ' + eos_token }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ +llama_template = ( + "{% if messages[0]['role'] == 'system' %}" + "{% if messages[1]['role'] == 'user' %}" + "{{ bos_token + '[INST] <>\n' + messages[0]['content'] + '\n<>\n\n' + messages[1]['content'] + ' [/INST]' }}" + "{% set loop_messages = messages[2:] %}" + "{% else %}" + "{{ bos_token + '[INST] ' + messages[0]['content'] + ' [/INST]' }}" + "{% set loop_messages = messages[1:] %}" + "{% endif %}" + "{% else %}" + "{% set loop_messages = messages %}" + "{% endif %}" + "{% for message in loop_messages %}" + "{% if message['role'] == 'user' %}" + "{{ bos_token + '[INST] ' + message['content'].strip() + ' [/INST]' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ ' ' + message['content'].strip() + ' ' + eos_token }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" + "{% endif %}" "{% endfor %}" +) # Ollama from https://www.ollama.com/library/llama3 -llama_ollama = \ -''' +llama_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """[INST] <>{{ .System }}<> @@ -232,36 +250,41 @@ PARAMETER min_p 0.1 ''' llama_eos_token = "eos_token" -CHAT_TEMPLATES["llama"] = (llama_template, llama_eos_token, False, llama_ollama,) -DEFAULT_SYSTEM_MESSAGE["llama"] = None # No system message in Llama +CHAT_TEMPLATES["llama"] = ( + llama_template, + llama_eos_token, + False, + llama_ollama, +) +DEFAULT_SYSTEM_MESSAGE["llama"] = None # No system message in Llama # =========================================== Vicuna # https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template -vicuna_template = \ - "{{ bos_token }}"\ - "{% if messages[0]['role'] == 'system' %}"\ - "{{ messages[0]['content'] + ' ' }}"\ - "{% set loop_messages = messages[1:] %}"\ - "{% else %}"\ - "{{ '{system_message}' + ' ' }}"\ - "{% set loop_messages = messages %}"\ - "{% endif %}"\ - "{% for message in loop_messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ 'USER: ' + message['content'] + ' ' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ 'ASSISTANT: ' + message['content'] + eos_token }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ 'ASSISTANT:' }}"\ +vicuna_template = ( + "{{ bos_token }}" + "{% if messages[0]['role'] == 'system' %}" + "{{ messages[0]['content'] + ' ' }}" + "{% set loop_messages = messages[1:] %}" + "{% else %}" + "{{ '{system_message}' + ' ' }}" + "{% set loop_messages = messages %}" "{% endif %}" + "{% for message in loop_messages %}" + "{% if message['role'] == 'user' %}" + "{{ 'USER: ' + message['content'] + ' ' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ 'ASSISTANT: ' + message['content'] + eos_token }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" + "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ 'ASSISTANT:' }}" + "{% endif %}" +) # Ollama from https://www.ollama.com/library/vicuna -vicuna_ollama = \ -''' +vicuna_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}USER: {{ .Prompt }} {{ end }}ASSISTANT: {{ .Response }} {__EOS_TOKEN__}""" PARAMETER stop "{__EOS_TOKEN__}" @@ -270,35 +293,42 @@ PARAMETER min_p 0.1 ''' vicuna_eos_token = "eos_token" -CHAT_TEMPLATES["vicuna"] = (vicuna_template, vicuna_eos_token, False, vicuna_ollama,) -DEFAULT_SYSTEM_MESSAGE["vicuna"] = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions." +CHAT_TEMPLATES["vicuna"] = ( + vicuna_template, + vicuna_eos_token, + False, + vicuna_ollama, +) +DEFAULT_SYSTEM_MESSAGE["vicuna"] = ( + "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions." +) # =========================================== Vicuna Old # https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template -vicuna_old_template = \ - "{{ bos_token }}"\ - "{% if messages[0]['role'] == 'system' %}"\ - "{{ messages[0]['content'] + '\n' }}"\ - "{% set loop_messages = messages[1:] %}"\ - "{% else %}"\ - "{{ '{system_message}' + '\n' }}"\ - "{% set loop_messages = messages %}"\ - "{% endif %}"\ - "{% for message in loop_messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ '### Human: ' + message['content'] + '\n' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ '### Assistant: ' + message['content'] + eos_token + '\n' }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '### Assistant:' }}"\ +vicuna_old_template = ( + "{{ bos_token }}" + "{% if messages[0]['role'] == 'system' %}" + "{{ messages[0]['content'] + '\n' }}" + "{% set loop_messages = messages[1:] %}" + "{% else %}" + "{{ '{system_message}' + '\n' }}" + "{% set loop_messages = messages %}" "{% endif %}" + "{% for message in loop_messages %}" + "{% if message['role'] == 'user' %}" + "{{ '### Human: ' + message['content'] + '\n' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ '### Assistant: ' + message['content'] + eos_token + '\n' }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" + "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '### Assistant:' }}" + "{% endif %}" +) -vicuna_old_ollama = \ -''' +vicuna_old_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}### Human: {{ .Prompt }} @@ -311,38 +341,45 @@ SYSTEM """A chat between a curious human and an artificial intelligence assistan ''' vicuna_old_eos_token = "eos_token" -CHAT_TEMPLATES["vicuna_old"] = (vicuna_old_template, vicuna_old_eos_token, False, vicuna_old_ollama,) -DEFAULT_SYSTEM_MESSAGE["vicuna_old"] = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human\\'s questions." +CHAT_TEMPLATES["vicuna_old"] = ( + vicuna_old_template, + vicuna_old_eos_token, + False, + vicuna_old_ollama, +) +DEFAULT_SYSTEM_MESSAGE["vicuna_old"] = ( + "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human\\'s questions." +) CHAT_TEMPLATES["vicuna old"] = CHAT_TEMPLATES["vicuna_old"] DEFAULT_SYSTEM_MESSAGE["vicuna old"] = DEFAULT_SYSTEM_MESSAGE["vicuna_old"] # =========================================== Alpaca multi turn # https://github.com/tatsu-lab/stanford_alpaca Changed for multi-turn convos -alpaca_template = \ - "{{ bos_token }}"\ - "{% if messages[0]['role'] == 'system' %}"\ - "{{ messages[0]['content'] + '\n\n' }}"\ - "{% set loop_messages = messages[1:] %}"\ - "{% else %}"\ - "{{ '{system_message}' + '\n\n' }}"\ - "{% set loop_messages = messages %}"\ - "{% endif %}"\ - "{% for message in loop_messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ '### Instruction:\n' + message['content'] + '\n\n' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ '### Response:\n' + message['content'] + eos_token + '\n\n' }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '### Response:\n' }}"\ +alpaca_template = ( + "{{ bos_token }}" + "{% if messages[0]['role'] == 'system' %}" + "{{ messages[0]['content'] + '\n\n' }}" + "{% set loop_messages = messages[1:] %}" + "{% else %}" + "{{ '{system_message}' + '\n\n' }}" + "{% set loop_messages = messages %}" "{% endif %}" + "{% for message in loop_messages %}" + "{% if message['role'] == 'user' %}" + "{{ '### Instruction:\n' + message['content'] + '\n\n' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ '### Response:\n' + message['content'] + eos_token + '\n\n' }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" + "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '### Response:\n' }}" + "{% endif %}" +) -alpaca_ollama = \ -''' +alpaca_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}{{ .System }} @@ -360,35 +397,42 @@ SYSTEM """Below are some instructions that describe some tasks. Write responses ''' alpaca_eos_token = "eos_token" -CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token, False, alpaca_ollama,) -DEFAULT_SYSTEM_MESSAGE["alpaca"] = "Below are some instructions that describe some tasks. Write responses that appropriately complete each request." +CHAT_TEMPLATES["alpaca"] = ( + alpaca_template, + alpaca_eos_token, + False, + alpaca_ollama, +) +DEFAULT_SYSTEM_MESSAGE["alpaca"] = ( + "Below are some instructions that describe some tasks. Write responses that appropriately complete each request." +) # =========================================== Gemma # https://huggingface.co/google/gemma-7b-it # Notice we must use |trim for lstrip and rstrip. maps to 106. # maps to 107. user and model are normal 1 word tokens. -gemma_template = \ - "{{ bos_token }}"\ - "{% if messages[0]['role'] == 'system' %}"\ - "{{'user\n' + messages[0]['content'] | trim + ' ' + messages[1]['content'] | trim + '\n'}}"\ - "{% set messages = messages[2:] %}"\ - "{% endif %}"\ - "{% for message in messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{'user\n' + message['content'] | trim + '\n'}}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{'model\n' + message['content'] | trim + '\n' }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ 'model\n' }}"\ +gemma_template = ( + "{{ bos_token }}" + "{% if messages[0]['role'] == 'system' %}" + "{{'user\n' + messages[0]['content'] | trim + ' ' + messages[1]['content'] | trim + '\n'}}" + "{% set messages = messages[2:] %}" "{% endif %}" + "{% for message in messages %}" + "{% if message['role'] == 'user' %}" + "{{'user\n' + message['content'] | trim + '\n'}}" + "{% elif message['role'] == 'assistant' %}" + "{{'model\n' + message['content'] | trim + '\n' }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" + "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ 'model\n' }}" + "{% endif %}" +) # Ollama from https://www.ollama.com/library/gemma -gemma_ollama = \ -''' +gemma_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """user {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} @@ -404,15 +448,19 @@ PARAMETER min_p 0.1 ''' gemma_eos_token = "" -CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token, True, gemma_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma"] = None # No system message in Gemma +CHAT_TEMPLATES["gemma"] = ( + gemma_template, + gemma_eos_token, + True, + gemma_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma"] = None # No system message in Gemma # =========================================== Gemma with ChatML instead # We find using is still more appropriate! gemma_chatml_template = "{{ bos_token }}" + chatml_template -gemma_chatml_ollama = \ -''' +gemma_chatml_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}<|im_start|>system {{ .System }}<|im_end|> @@ -430,11 +478,16 @@ PARAMETER min_p 0.1 ''' gemma_chatml_eos_token = ( - {"" : "<|im_start|>", "" : "<|im_end|>"}, + {"": "<|im_start|>", "": "<|im_end|>"}, "<|im_end|>", ) -CHAT_TEMPLATES["gemma_chatml"] = (gemma_chatml_template, gemma_chatml_eos_token, True, gemma_chatml_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma_chatml"] = None # No system message in Gemma +CHAT_TEMPLATES["gemma_chatml"] = ( + gemma_chatml_template, + gemma_chatml_eos_token, + True, + gemma_chatml_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma_chatml"] = None # No system message in Gemma # =========================================== Gemma 2 # Same as Gemma 1, but with sliding window attention! @@ -442,36 +495,46 @@ DEFAULT_SYSTEM_MESSAGE["gemma_chatml"] = None # No system message in Gemma gemma2_template = gemma_template gemma2_ollama = gemma_ollama + "PARAMETER num_ctx 4096\n" gemma2_eos_token = "" -CHAT_TEMPLATES["gemma2"] = (gemma2_template, gemma2_eos_token, True, gemma2_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma2"] = None # No system message in Gemma 2 +CHAT_TEMPLATES["gemma2"] = ( + gemma2_template, + gemma2_eos_token, + True, + gemma2_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma2"] = None # No system message in Gemma 2 # =========================================== Gemma 2 with ChatML instead gemma2_chatml_template = gemma_chatml_template gemma2_chatml_ollama = gemma_chatml_ollama + "PARAMETER num_ctx 4096\n" gemma2_chatml_eos_token = gemma_chatml_eos_token -CHAT_TEMPLATES["gemma2_chatml"] = (gemma2_chatml_template, gemma2_chatml_eos_token, True, gemma2_chatml_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma2_chatml"] = None # No system message in Gemma 2 +CHAT_TEMPLATES["gemma2_chatml"] = ( + gemma2_chatml_template, + gemma2_chatml_eos_token, + True, + gemma2_chatml_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma2_chatml"] = None # No system message in Gemma 2 # =========================================== Llama-3 # Weirdly \n\n is needed? -llama3_template = \ - "{{ bos_token }}"\ - "{% for message in messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ '<|start_header_id|>user<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}"\ - "{% else %}"\ - "{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"\ +llama3_template = ( + "{{ bos_token }}" + "{% for message in messages %}" + "{% if message['role'] == 'user' %}" + "{{ '<|start_header_id|>user<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}" + "{% elif message['role'] == 'assistant' %}" + "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}" + "{% else %}" + "{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}" "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}" + "{% endif %}" +) # Ollama from https://www.ollama.com/library/llama3 -llama3_ollama = \ -''' +llama3_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|> @@ -489,32 +552,42 @@ PARAMETER min_p 0.1 llama3_template_eos_token = "eos_token" -CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,) -DEFAULT_SYSTEM_MESSAGE["llama-3"] = None # No system message in Llama-3 +CHAT_TEMPLATES["llama-3"] = ( + llama3_template, + llama3_template_eos_token, + False, + llama3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["llama-3"] = None # No system message in Llama-3 -CHAT_TEMPLATES["llama3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,) -DEFAULT_SYSTEM_MESSAGE["llama3"] = None # No system message in Llama-3 +CHAT_TEMPLATES["llama3"] = ( + llama3_template, + llama3_template_eos_token, + False, + llama3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["llama3"] = None # No system message in Llama-3 # =========================================== Phi-3 # "{{ bos_token }}"\ # Phi-3.5 removes BOS? -phi3_template = \ - "{% for message in messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{'<|user|>\n' + message['content'] + '<|end|>\n'}}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}"\ - "{% else %}"\ - "{{'<|' + message['role'] + '|>\n' + message['content'] + '<|end|>\n'}}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '<|assistant|>\n' }}"\ +phi3_template = ( + "{% for message in messages %}" + "{% if message['role'] == 'user' %}" + "{{'<|user|>\n' + message['content'] + '<|end|>\n'}}" + "{% elif message['role'] == 'assistant' %}" + "{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}" + "{% else %}" + "{{'<|' + message['role'] + '|>\n' + message['content'] + '<|end|>\n'}}" "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '<|assistant|>\n' }}" + "{% endif %}" +) # Ollama from https://www.ollama.com/library/phi3 -phi3_ollama = \ -''' +phi3_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}<|system|> {{ .System }}<|end|> @@ -531,14 +604,19 @@ PARAMETER min_p 0.1 ''' phi3_template_eos_token = "<|end|>" -CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token, False, phi3_ollama,) -DEFAULT_SYSTEM_MESSAGE["phi-3"] = None # No system message in Phi-3 +CHAT_TEMPLATES["phi-3"] = ( + phi3_template, + phi3_template_eos_token, + False, + phi3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["phi-3"] = None # No system message in Phi-3 -CHAT_TEMPLATES["phi-35"] = CHAT_TEMPLATES["phi-3"] -DEFAULT_SYSTEM_MESSAGE["phi-35"] = None # No system message in Phi-3.5 +CHAT_TEMPLATES["phi-35"] = CHAT_TEMPLATES["phi-3"] +DEFAULT_SYSTEM_MESSAGE["phi-35"] = None # No system message in Phi-3.5 CHAT_TEMPLATES["phi-3.5"] = CHAT_TEMPLATES["phi-3"] -DEFAULT_SYSTEM_MESSAGE["phi-3.5"] = None # No system message in Phi-3.5 +DEFAULT_SYSTEM_MESSAGE["phi-3.5"] = None # No system message in Phi-3.5 # =========================================== Llama-3.1 """ @@ -557,8 +635,7 @@ tokenizer.apply_chat_template( ) """ -llama31_template = \ -"""{{- bos_token }} +llama31_template = """{{- bos_token }} {%- if custom_tools is defined %} {%- set tools = custom_tools %} {%- endif %} @@ -670,8 +747,7 @@ llama31_template = \ """ # Ollama from https://ollama.com/library/llama3.1 (needs updating!) -llama31_ollama = \ -''' +llama31_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .Messages }} {{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|> @@ -731,11 +807,25 @@ PARAMETER min_p 0.1 ''' llama31_template_eos_token = "eos_token" -CHAT_TEMPLATES["llama-3.1"] = (llama31_template, llama31_template_eos_token, False, llama31_ollama,) -DEFAULT_SYSTEM_MESSAGE["llama-3.1"] = "" # Llama3.1 default system message is empty + the dates +CHAT_TEMPLATES["llama-3.1"] = ( + llama31_template, + llama31_template_eos_token, + False, + llama31_ollama, +) +DEFAULT_SYSTEM_MESSAGE["llama-3.1"] = ( + "" # Llama3.1 default system message is empty + the dates +) -CHAT_TEMPLATES["llama-31"] = (llama31_template, llama31_template_eos_token, False, llama31_ollama,) -DEFAULT_SYSTEM_MESSAGE["llama-31"] = "" # Llama3.1 default system message is empty + the dates +CHAT_TEMPLATES["llama-31"] = ( + llama31_template, + llama31_template_eos_token, + False, + llama31_ollama, +) +DEFAULT_SYSTEM_MESSAGE["llama-31"] = ( + "" # Llama3.1 default system message is empty + the dates +) for version in ("llama-3.2", "llama-3.3", "llama-32", "llama-33"): CHAT_TEMPLATES[version] = CHAT_TEMPLATES["llama-3.1"] @@ -743,8 +833,7 @@ for version in ("llama-3.2", "llama-3.3", "llama-32", "llama-33"): # =========================================== Qwen 2.5 -qwen25_template = \ -"""{%- if tools %} +qwen25_template = """{%- if tools %} {{- \'<|im_start|>system\\n\' }} {%- if messages[0][\'role\'] == \'system\' %} {{- messages[0][\'content\'] }} @@ -796,8 +885,7 @@ qwen25_template = \ # Ollama from https://ollama.com/library/qwen2.5/blobs/eb4402837c78 -qwen25_ollama = \ -''' +qwen25_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{- if .Messages }} {{- if or .System .Tools }}<|im_start|>system @@ -856,43 +944,74 @@ PARAMETER min_p 0.1 ''' qwen25_template_eos_token = "eos_token" -qwen25_default_system_message = "You are Qwen, created by Alibaba Cloud. You are a helpful assistant." -CHAT_TEMPLATES["qwen-2.5"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen-2.5"] = qwen25_default_system_message # No system message in Qwen 2.5 +qwen25_default_system_message = ( + "You are Qwen, created by Alibaba Cloud. You are a helpful assistant." +) +CHAT_TEMPLATES["qwen-2.5"] = ( + qwen25_template, + qwen25_template_eos_token, + False, + qwen25_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen-2.5"] = ( + qwen25_default_system_message # No system message in Qwen 2.5 +) -CHAT_TEMPLATES["qwen-25"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen-25"] = qwen25_default_system_message # No system message in Qwen 2.5 +CHAT_TEMPLATES["qwen-25"] = ( + qwen25_template, + qwen25_template_eos_token, + False, + qwen25_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen-25"] = ( + qwen25_default_system_message # No system message in Qwen 2.5 +) -CHAT_TEMPLATES["qwen25"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen25"] = qwen25_default_system_message # No system message in Qwen 2.5 +CHAT_TEMPLATES["qwen25"] = ( + qwen25_template, + qwen25_template_eos_token, + False, + qwen25_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen25"] = ( + qwen25_default_system_message # No system message in Qwen 2.5 +) -CHAT_TEMPLATES["qwen2.5"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen2.5"] = qwen25_default_system_message # No system message in Qwen 2.5 +CHAT_TEMPLATES["qwen2.5"] = ( + qwen25_template, + qwen25_template_eos_token, + False, + qwen25_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen2.5"] = ( + qwen25_default_system_message # No system message in Qwen 2.5 +) # =========================================== Phi-4 # "{{ bos_token }}"\ # Phi-4 removes BOS? -phi4_template = \ - "{% for message in messages %}"\ - "{% if (message['role'] == 'system') %}"\ - "{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}"\ - "{% elif (message['role'] == 'user') %}"\ - "{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|>'}}"\ - "{% elif (message['role'] == 'assistant') %}"\ - "{{'<|im_start|>assistant<|im_sep|>' + message['content'] + '<|im_end|>'}}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '<|im_start|>assistant<|im_sep|>' }}"\ +phi4_template = ( + "{% for message in messages %}" + "{% if (message['role'] == 'system') %}" + "{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}" + "{% elif (message['role'] == 'user') %}" + "{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|>'}}" + "{% elif (message['role'] == 'assistant') %}" + "{{'<|im_start|>assistant<|im_sep|>' + message['content'] + '<|im_end|>'}}" "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '<|im_start|>assistant<|im_sep|>' }}" + "{% endif %}" +) -_phi4_ollama_template = \ - "{{ if .System }}<|im_start|><|system|><|im_sep|>{{ .System }}<|im_end|>{{ end }}"\ - "{{ if .Prompt }}<|im_start|><|user|><|im_sep|>{{ .Prompt }}<|im_end|>{{ end }}"\ +_phi4_ollama_template = ( + "{{ if .System }}<|im_start|><|system|><|im_sep|>{{ .System }}<|im_end|>{{ end }}" + "{{ if .Prompt }}<|im_start|><|user|><|im_sep|>{{ .Prompt }}<|im_end|>{{ end }}" "<|im_start|><|assistant|><|im_sep|>{{ .Response }}<|im_end|>" +) # Ollama from https://www.ollama.com/library/phi4 is different -phi4_ollama = \ -f''' +phi4_ollama = f''' FROM {{__FILE_LOCATION__}} TEMPLATE """{_phi4_ollama_template}""" PARAMETER stop "<|im_end|>" @@ -903,15 +1022,19 @@ PARAMETER min_p 0.1 ''' phi4_template_eos_token = "<|im_end|>" -CHAT_TEMPLATES["phi-4"] = (phi4_template, phi4_template_eos_token, False, phi4_ollama,) -DEFAULT_SYSTEM_MESSAGE["phi-4"] = None # No system message in Phi-4 +CHAT_TEMPLATES["phi-4"] = ( + phi4_template, + phi4_template_eos_token, + False, + phi4_ollama, +) +DEFAULT_SYSTEM_MESSAGE["phi-4"] = None # No system message in Phi-4 # =========================================== Gemma-3 # Obtained via # print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n")) -gemma3_template = \ -"""{{ bos_token }} +gemma3_template = """{{ bos_token }} {%- if messages[0]['role'] == 'system' -%} {%- if messages[0]['content'] is string -%} {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%} @@ -954,8 +1077,7 @@ gemma3_template = \ """ # Ollama from https://ollama.com/library/gemma3/blobs/e0a42594d802 -gemma3_ollama = \ -''' +gemma3_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{- range $i, $_ := .Messages }} {{- $last := eq (len (slice $.Messages $i)) 1 }} @@ -978,16 +1100,25 @@ PARAMETER num_predict 32768 ''' gemma3_template_eos_token = "" -CHAT_TEMPLATES["gemma-3"] = (gemma3_template, gemma3_template_eos_token, False, gemma3_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma-3"] = None # No system message in Gemma-3 +CHAT_TEMPLATES["gemma-3"] = ( + gemma3_template, + gemma3_template_eos_token, + False, + gemma3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma-3"] = None # No system message in Gemma-3 -CHAT_TEMPLATES["gemma3"] = (gemma3_template, gemma3_template_eos_token, False, gemma3_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma3"] = None # No system message in Gemma-3 +CHAT_TEMPLATES["gemma3"] = ( + gemma3_template, + gemma3_template_eos_token, + False, + gemma3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma3"] = None # No system message in Gemma-3 # =========================================== Qwen-3 # Official Qwen-3 chat template (see https://ollama.com/library/qwen3/blobs/eb4402837c78) -qwen3_template = \ -""" +qwen3_template = """ {%- if tools %} {{- '<|im_start|>system\n' }} {%- if messages[0].role == 'system' %} @@ -1089,8 +1220,7 @@ qwen3_template = \ """ # Ollama template for Qwen-3 (see https://ollama.com/library/qwen3/blobs/eb4402837c78) -qwen3_ollama = \ -''' +qwen3_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{- if .Messages }} {{- if or .System .Tools }}<|im_start|>system @@ -1152,17 +1282,26 @@ PARAMETER repeat_penalty 1 ''' qwen3_template_eos_token = "<|im_end|>" -CHAT_TEMPLATES["qwen-3"] = (qwen3_template, qwen3_template_eos_token, False, qwen3_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen-3"] = None # No default system message for Qwen-3 +CHAT_TEMPLATES["qwen-3"] = ( + qwen3_template, + qwen3_template_eos_token, + False, + qwen3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen-3"] = None # No default system message for Qwen-3 -CHAT_TEMPLATES["qwen3"] = (qwen3_template, qwen3_template_eos_token, False, qwen3_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen3"] = None # No default system message for Qwen-3 +CHAT_TEMPLATES["qwen3"] = ( + qwen3_template, + qwen3_template_eos_token, + False, + qwen3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen3"] = None # No default system message for Qwen-3 # =========================================== Gemma-3n # Obtained via # print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n")) -gemma3n_template = \ -"""{{ bos_token }} +gemma3n_template = """{{ bos_token }} {%- if messages[0]['role'] == 'system' -%} {%- if messages[0]['content'] is string -%} {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%} @@ -1207,8 +1346,7 @@ gemma3n_template = \ """ # Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802 -gemma3n_ollama = \ -''' +gemma3n_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{- range $i, $_ := .Messages }} {{- $last := eq (len (slice $.Messages $i)) 1 }} @@ -1224,17 +1362,26 @@ TEMPLATE """{{- range $i, $_ := .Messages }} ''' gemma3n_template_eos_token = "" -CHAT_TEMPLATES["gemma-3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None # No system message in Gemma-3n +CHAT_TEMPLATES["gemma-3n"] = ( + gemma3n_template, + gemma3n_template_eos_token, + False, + gemma3n_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None # No system message in Gemma-3n -CHAT_TEMPLATES["gemma3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,) -DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None # No system message in Gemma-3n +CHAT_TEMPLATES["gemma3n"] = ( + gemma3n_template, + gemma3n_template_eos_token, + False, + gemma3n_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None # No system message in Gemma-3n # =========================================== GPT-OSS # Obtained via # print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n")) -gptoss_template = \ -"""{#- +gptoss_template = """{#- In addition to the normal inputs of `messages` and `tools`, this template also accepts the following kwargs: - "builtin_tools": A list, can contain "browser" and/or "python". @@ -1584,8 +1731,7 @@ gptoss_template = \ {%- endif -%}""" # Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802 -gptoss_ollama = \ -''' +gptoss_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. Knowledge cutoff: 2024-06 @@ -1766,15 +1912,24 @@ PARAMETER top_p 1.0 ''' gptoss_template_template_eos_token = "<|return|>" -CHAT_TEMPLATES["gpt-oss"] = (gptoss_template, gptoss_template_template_eos_token, False, gptoss_ollama,) -DEFAULT_SYSTEM_MESSAGE["gpt-oss"] = None # No system message in GPT-oss +CHAT_TEMPLATES["gpt-oss"] = ( + gptoss_template, + gptoss_template_template_eos_token, + False, + gptoss_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gpt-oss"] = None # No system message in GPT-oss -CHAT_TEMPLATES["gptoss"] = (gptoss_template, gptoss_template_template_eos_token, False, gptoss_ollama,) -DEFAULT_SYSTEM_MESSAGE["gptoss"] = None # No system message in GPT-oss +CHAT_TEMPLATES["gptoss"] = ( + gptoss_template, + gptoss_template_template_eos_token, + False, + gptoss_ollama, +) +DEFAULT_SYSTEM_MESSAGE["gptoss"] = None # No system message in GPT-oss # =========================================== Qwen3-Instruct -qwen3_instruct_template = \ -'''{%- if tools %} +qwen3_instruct_template = """{%- if tools %} {{- '<|im_start|>system\\n' }} {%- if messages[0].role == 'system' %} {{- messages[0].content + '\\n\\n' }} @@ -1859,11 +2014,10 @@ qwen3_instruct_template = \ {%- endfor %} {%- if add_generation_prompt %} {{- '<|im_start|>assistant\\n' }} -{%- endif %}''' +{%- endif %}""" # Ollama from https://ollama.com/library/qwen3/blobs/53e4ea15e8f5 -qwen3_ollama = \ -''' +qwen3_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """ {{- $lastUserIdx := -1 -}} @@ -1920,13 +2074,17 @@ For each function call, return a json object with function name and arguments wi ''' qwen3_template_eos_token = "<|im_end|>" -CHAT_TEMPLATES["qwen3-instruct"] = (qwen3_instruct_template, qwen3_template_eos_token, False, qwen3_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen3-instruct"] = None # No system message in Qwen3 +CHAT_TEMPLATES["qwen3-instruct"] = ( + qwen3_instruct_template, + qwen3_template_eos_token, + False, + qwen3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen3-instruct"] = None # No system message in Qwen3 # =========================================== Qwen3-Thinking -qwen3_thinking_template = \ -'''{%- if tools %} +qwen3_thinking_template = """{%- if tools %} {{- '<|im_start|>system\\n' }} {%- if messages[0].role == 'system' %} {{- messages[0].content + '\\n\\n' }} @@ -2011,29 +2169,37 @@ qwen3_thinking_template = \ {%- endfor %} {%- if add_generation_prompt %} {{- '<|im_start|>assistant\n\n' }} -{%- endif %}''' +{%- endif %}""" -CHAT_TEMPLATES["qwen3-thinking"] = (qwen3_thinking_template, qwen3_template_eos_token, False, qwen3_ollama,) -DEFAULT_SYSTEM_MESSAGE["qwen3-thinking"] = None # No system message in Qwen3 +CHAT_TEMPLATES["qwen3-thinking"] = ( + qwen3_thinking_template, + qwen3_template_eos_token, + False, + qwen3_ollama, +) +DEFAULT_SYSTEM_MESSAGE["qwen3-thinking"] = None # No system message in Qwen3 # =========================================== Liquid-LFM2 -liquid_lfm2_template = \ -''' +liquid_lfm2_template = """ {{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + ' ' + message['content'] + '<|im_end|>' + ' '}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant -' }}{% endif %}''' +' }}{% endif %}""" liquid_lfm2_template_eos_token = "<|im_end|>" -CHAT_TEMPLATES["lfm-2"] = (liquid_lfm2_template, liquid_lfm2_template_eos_token, False, None) -DEFAULT_SYSTEM_MESSAGE["lfm-2"] = None # No system message in Phi-3 +CHAT_TEMPLATES["lfm-2"] = ( + liquid_lfm2_template, + liquid_lfm2_template_eos_token, + False, + None, +) +DEFAULT_SYSTEM_MESSAGE["lfm-2"] = None # No system message in Phi-3 # =========================================== Starling-LM -starling_template = \ -"""{{ bos_token }} +starling_template = """{{ bos_token }} {%- for message in messages %} {{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>' }} {%- endfor %} @@ -2042,8 +2208,7 @@ starling_template = \ {%- endif %}""" # Ollama from https://ollama.com/library/starling-lm:7b/blobs/4b21bfc435b4 -starling_ollama = \ -''' +starling_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}GPT4 Correct System: {{ .System }}<|end_of_turn|> {{ end }}{{ if .Prompt }}GPT4 Correct User: {{ .Prompt }}<|end_of_turn|> @@ -2057,14 +2222,18 @@ PARAMETER min_p 0.1 ''' starling_template_eos_token = "<|end_of_turn|>" -CHAT_TEMPLATES["starling"] = (starling_template, starling_template_eos_token, False, starling_ollama) +CHAT_TEMPLATES["starling"] = ( + starling_template, + starling_template_eos_token, + False, + starling_ollama, +) DEFAULT_SYSTEM_MESSAGE["starling"] = None # =========================================== Yi-chat -yi_chat_template = \ -""" +yi_chat_template = """ {% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + ' ' + message['content'] + '<|im_end|>' + ' '}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant @@ -2072,8 +2241,7 @@ yi_chat_template = \ """ # Ollama from https://ollama.com/library/yi:34b-chat/blobs/62fbfd9ed093 -yi_chat_ollama = \ -''' +yi_chat_ollama = ''' FROM {__FILE_LOCATION__} TEMPLATE """{{ if .System }}<|im_start|>system {{ .System }}<|im_end|> @@ -2084,10 +2252,18 @@ TEMPLATE """{{ if .System }}<|im_start|>system ''' yi_chat_template_eos_token = "<|endoftext|>" -CHAT_TEMPLATES["yi-chat"] = (yi_chat_template, yi_chat_template_eos_token, False, yi_chat_ollama) +CHAT_TEMPLATES["yi-chat"] = ( + yi_chat_template, + yi_chat_template_eos_token, + False, + yi_chat_ollama, +) DEFAULT_SYSTEM_MESSAGE["yi-chat"] = None -def _change_system_message(template: str, type_chat_template: str, system_message: str = None): + +def _change_system_message( + template: str, type_chat_template: str, system_message: str = None +): system_message_pattern = r"\{system_message\}" # For predefined templates, check if default system message exists @@ -2107,14 +2283,18 @@ def _change_system_message(template: str, type_chat_template: str, system_messag if has_placeholder: if system_message is None: - raise ValueError("Unsloth: You need to provide a system message for custom templates.") + raise ValueError( + "Unsloth: You need to provide a system message for custom templates." + ) new_template = re.sub(system_message_pattern, system_message, template) return new_template, system_message return template, system_message # For predefined templates with default system message - message_to_use = system_message if system_message is not None else default_system_message + message_to_use = ( + system_message if system_message is not None else default_system_message + ) new_template = re.sub(system_message_pattern, message_to_use, template) return new_template, message_to_use @@ -2123,16 +2303,22 @@ def _change_system_message(template: str, type_chat_template: str, system_messag def get_chat_template( tokenizer, chat_template = "chatml", - mapping = {"role" : "role", "content" : "content", "user" : "user", "assistant" : "assistant"}, + mapping = { + "role": "role", + "content": "content", + "user": "user", + "assistant": "assistant", + }, map_eos_token = True, system_message = None, ): - assert(type(map_eos_token) is bool) + assert type(map_eos_token) is bool old_tokenizer = tokenizer IS_GEMMA = False if tokenizer.__class__.__name__.startswith("Gemma"): - if chat_template == "chatml": chat_template = "gemma_chatml" + if chat_template == "chatml": + chat_template = "gemma_chatml" IS_GEMMA = True # We add a check for Llama-3 @@ -2153,32 +2339,42 @@ def get_chat_template( same_padding_token = False type_chat_template = None - if type(chat_template) in (list, tuple,): + if type(chat_template) in ( + list, + tuple, + ): # For changing system message later # Since it's not supported yet, we will raise an error first! type_chat_template = chat_template[0].lower() chat_template, stop_word = chat_template - assert(type(chat_template) is str) - assert(type(stop_word) is str) + assert type(chat_template) is str + assert type(stop_word) is str ollama_modelfile = None elif type(chat_template) is str: # For changing system message later type_chat_template = chat_template.lower() - chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template] + chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[ + chat_template + ] # Check mapping to eos_token - if not map_eos_token and yes_map_eos_token: map_eos_token = True - if not yes_map_eos_token and map_eos_token: map_eos_token = False + if not map_eos_token and yes_map_eos_token: + map_eos_token = True + if not yes_map_eos_token and map_eos_token: + map_eos_token = False - if type(stop_word) in (list, tuple,): + if type(stop_word) in ( + list, + tuple, + ): token_mapping, stop_word = stop_word - assert(type(token_mapping) is dict) + assert type(token_mapping) is dict else: token_mapping = None - assert(type(stop_word) is str) + assert type(stop_word) is str # Check fast tokenizer if not is_fast_tokenizer: @@ -2205,13 +2401,17 @@ def get_chat_template( elif old_count == 0: raise RuntimeError(f"{old_token} was not part of the tokenizer!") else: - string_vocab = string_vocab.replace(f'"{old_token}"', f'"{new_token}"') + string_vocab = string_vocab.replace( + f'"{old_token}"', f'"{new_token}"' + ) pass pass - if map_eos_token and (not stop_word in token_mapping.values()): + if map_eos_token and (stop_word not in token_mapping.values()): # Do not map 107 = <|im_end|> and 1 = <|im_end|>. This will reduce the vocab size by 1 - logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.") + logger.warning_once( + f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}." + ) string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word) pass @@ -2239,12 +2439,18 @@ def get_chat_template( pass # Must fix the sentence piece tokenizer since there's no tokenizer.model file! - tokenizer = fix_sentencepiece_tokenizer(tokenizer, new_tokenizer, token_mapping,) + tokenizer = fix_sentencepiece_tokenizer( + tokenizer, + new_tokenizer, + token_mapping, + ) else: pass elif map_eos_token and (stop_word != "eos_token"): - logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.") + logger.warning_once( + f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}." + ) # Replaces the old EOS token with a new one. # Useful for ChatML <|im_end|> for example. @@ -2286,43 +2492,60 @@ def get_chat_template( ) # Must fix the sentence piece tokenizer since there's no tokenizer.model file! - token_mapping = { old_eos_token : stop_word, } - tokenizer = fix_sentencepiece_tokenizer(tokenizer, new_tokenizer, token_mapping,) + token_mapping = { + old_eos_token: stop_word, + } + tokenizer = fix_sentencepiece_tokenizer( + tokenizer, + new_tokenizer, + token_mapping, + ) pass else: raise TypeError( - f"Unsloth: `chat_template` must be a tuple of (your_template, eos_token,) or one of\n"\ + f"Unsloth: `chat_template` must be a tuple of (your_template, eos_token,) or one of\n" f"{CHAT_TEMPLATES.keys()}" ) # Careful on Gemma # bos_token is a must or else losses become too high - if IS_GEMMA and not chat_template.startswith(("{{ bos_token }}", "{{- bos_token }}")): + if IS_GEMMA and not chat_template.startswith( + ("{{ bos_token }}", "{{- bos_token }}") + ): chat_template = "{{ bos_token }}" + chat_template # For ShareGPT role -> from and content -> value - new_chat_template = chat_template\ - .replace("'role'", "'" + mapping["role"] + "'")\ - .replace("'content'", "'" + mapping["content"] + "'")\ - .replace("'user'", "'" + mapping["user"] + "'")\ + new_chat_template = ( + chat_template.replace("'role'", "'" + mapping["role"] + "'") + .replace("'content'", "'" + mapping["content"] + "'") + .replace("'user'", "'" + mapping["user"] + "'") .replace("'assistant'", "'" + mapping["assistant"] + "'") + ) _, tokenizer = patch_tokenizer(model = None, tokenizer = tokenizer) tokenizer.padding_side = old_padding_side # If not normal HF, we add a check to make old templates work - if mapping != {"role" : "role", "content" : "content", "user" : "user", "assistant" : "assistant"}: - chat_template = \ - "{% if 'role' in messages[0] %}" + \ - chat_template + \ - "{% else %}" + \ - new_chat_template + \ - "{% endif %}" + if mapping != { + "role": "role", + "content": "content", + "user": "user", + "assistant": "assistant", + }: + chat_template = ( + "{% if 'role' in messages[0] %}" + + chat_template + + "{% else %}" + + new_chat_template + + "{% endif %}" + ) else: chat_template = new_chat_template - chat_template, system_message = _change_system_message(chat_template, type_chat_template, system_message) + chat_template, system_message = _change_system_message( + chat_template, type_chat_template, system_message + ) tokenizer.chat_template = chat_template @@ -2330,13 +2553,16 @@ def get_chat_template( old_pad_token = getattr(old_tokenizer, "pad_token", None) old_bos_token = getattr(old_tokenizer, "bos_token", None) old_unk_token = getattr(old_tokenizer, "unk_token", None) - new_pad_token = getattr(tokenizer, "pad_token", None) - new_bos_token = getattr(tokenizer, "bos_token", None) - new_unk_token = getattr(tokenizer, "unk_token", None) - if old_bos_token != new_bos_token: tokenizer.bos_token = old_bos_token - if old_unk_token != new_unk_token: tokenizer.unk_token = old_unk_token + new_pad_token = getattr(tokenizer, "pad_token", None) + new_bos_token = getattr(tokenizer, "bos_token", None) + new_unk_token = getattr(tokenizer, "unk_token", None) + if old_bos_token != new_bos_token: + tokenizer.bos_token = old_bos_token + if old_unk_token != new_unk_token: + tokenizer.unk_token = old_unk_token if not same_padding_token: - if old_pad_token != new_pad_token: tokenizer.pad_token = old_pad_token + if old_pad_token != new_pad_token: + tokenizer.pad_token = old_pad_token # stopping_criteria = create_stopping_criteria(tokenizer, stop_word) @@ -2345,14 +2571,14 @@ def get_chat_template( # Add Ollama tokenizer._ollama_modelfile = ollama_modelfile - tokenizer._system_message = system_message - return tokenizer#, stopping_criteria + tokenizer._system_message = system_message + return tokenizer # , stopping_criteria def remove_special_tokens(tokenizer, prompt): # Removes double BOS token if prompt.startswith(tokenizer.bos_token): - prompt = prompt[len(tokenizer.bos_token):] + prompt = prompt[len(tokenizer.bos_token) :] return prompt @@ -2363,12 +2589,14 @@ def _parse_combined_prompt(combined_prompt, dataset): for column in possible_columns: if column not in dataset_columns: raise KeyError( - f"Unsloth: Your prompt includes '{column}' but this does not exist in the dataset. "\ + f"Unsloth: Your prompt includes '{column}' but this does not exist in the dataset. " f"Only allowed columns are {list(dataset_columns)}" ) # Find [[...]] - optional_prompts = list(re.finditer(r"\[\[.+?\]\]", combined_prompt, flags = re.DOTALL | re.MULTILINE)) + optional_prompts = list( + re.finditer(r"\[\[.+?\]\]", combined_prompt, flags = re.DOTALL | re.MULTILINE) + ) optional_prompts = [(x.span(), x.group(0)) for x in optional_prompts] final_optional_prompts = [] @@ -2376,25 +2604,30 @@ def _parse_combined_prompt(combined_prompt, dataset): # Add left left = optional_prompts[0] l = left[0][0] - if l != 0: final_optional_prompts.append(combined_prompt[:l]) + if l != 0: + final_optional_prompts.append(combined_prompt[:l]) # Add in between for left, right in zip(optional_prompts[:-1], optional_prompts[1:]): l, r = left[0][-1], right[0][0] final_optional_prompts.append(left) - if l != r: final_optional_prompts.append(combined_prompt[l : r]) + if l != r: + final_optional_prompts.append(combined_prompt[l:r]) final_optional_prompts.append(optional_prompts[-1]) # Add right right = optional_prompts[-1] r = right[0][1] - if r != len(combined_prompt): final_optional_prompts.append(combined_prompt[r:]) + if r != len(combined_prompt): + final_optional_prompts.append(combined_prompt[r:]) else: # Just add in the entire string final_optional_prompts.append(combined_prompt) - check_combined = "".join(x if type(x) is str else x[1] for x in final_optional_prompts) - assert(combined_prompt == check_combined) + check_combined = "".join( + x if type(x) is str else x[1] for x in final_optional_prompts + ) + assert combined_prompt == check_combined return possible_columns, final_optional_prompts @@ -2406,7 +2639,9 @@ def _create_formatter(possible_columns, final_optional_prompts, user_column_name for column in columns: function.append(f"{' '*4}{column}__ = examples['{column}']") function.append(f"{' '*4}texts = []") - function.append(f"{' '*4}for ({', '.join(columns)}) in zip({', '.join(f'{x}__' for x in columns)}):") + function.append( + f"{' '*4}for ({', '.join(columns)}) in zip({', '.join(f'{x}__' for x in columns)}):" + ) # Add optional tags as well! final_prompt = "" @@ -2417,12 +2652,23 @@ def _create_formatter(possible_columns, final_optional_prompts, user_column_name columns = re.findall(r"\{(.+?)\}", optional_prompt) formatter += columns # Must escape \n \r - final_prompt += optional_prompt.encode("unicode-escape").decode("utf-8").replace("'", "\\'").replace('"', '\\"') + final_prompt += ( + optional_prompt.encode("unicode-escape") + .decode("utf-8") + .replace("'", "\\'") + .replace('"', '\\"') + ) else: where, prompt = optional_prompt # Strip [[...]] # Must escape \n \r - prompt = prompt[2:-2].encode("unicode-escape").decode("utf-8").replace("'", "\\'").replace('"', '\\"') + prompt = ( + prompt[2:-2] + .encode("unicode-escape") + .decode("utf-8") + .replace("'", "\\'") + .replace('"', '\\"') + ) columns = re.findall(r"\{(.+?)\}", prompt) x = f"__optional_{j}__" prompt = f"{' '*8}{x} = '{prompt}'.format({', '.join(f'{x} = {x}' for x in columns)}) if {columns[0]} else ''" @@ -2431,8 +2677,10 @@ def _create_formatter(possible_columns, final_optional_prompts, user_column_name final_prompt += "{" + x + "}" function.insert(1, f"{' '*4}__combined_prompt__ = '{final_prompt}'") - function.append(f"{' '*8}texts.append("\ - f"__combined_prompt__.format({', '.join(f'{x} = {x}' for x in formatter)}))") + function.append( + f"{' '*8}texts.append(" + f"__combined_prompt__.format({', '.join(f'{x} = {x}' for x in formatter)}))" + ) function.append(f"{' '*4}return " + "{ " + f"'{user_column_name}' : texts" + " }") return "\n".join(function) @@ -2463,24 +2711,34 @@ def to_sharegpt( if "conversations" in dataset.column_names: convo = dataset[0]["conversations"] if type(convo) is list: - raise TypeError("Unsloth: Your dataset is probably already in ShareGPT format!") + raise TypeError( + "Unsloth: Your dataset is probably already in ShareGPT format!" + ) - possible_columns, final_optional_prompts = _parse_combined_prompt(merged_prompt, dataset) - function = _create_formatter(possible_columns, final_optional_prompts, merged_column_name) + possible_columns, final_optional_prompts = _parse_combined_prompt( + merged_prompt, dataset + ) + function = _create_formatter( + possible_columns, final_optional_prompts, merged_column_name + ) exec(function, globals()) - dataset = dataset.map(__combined_prompt_processor__, batched = True, desc = "Merging columns") + dataset = dataset.map( + __combined_prompt_processor__, batched = True, desc = "Merging columns" + ) def __convert_to_sharegpt__(examples): - users = examples[merged_column_name] + users = examples[merged_column_name] assistants = examples[output_column_name] texts = [ [ - {"from" : "human", "value" : str(user) }, - {"from" : "gpt", "value" : str(assistant)}, - ] \ + {"from": "human", "value": str(user)}, + {"from": "gpt", "value": str(assistant)}, + ] for user, assistant in zip(users, assistants) ] - return { "conversations" : texts, } + return { + "conversations": texts, + } dataset = dataset.map( __convert_to_sharegpt__, @@ -2492,13 +2750,17 @@ def to_sharegpt( # Randomnly concat conversations to create a long stream! from datasets import concatenate_datasets - n_extensions = max(conversation_extension-1, 0) - if n_extensions == 0: return dataset - dataset = dataset.rename_columns({"conversations" : "conversations0"}) + n_extensions = max(conversation_extension - 1, 0) + if n_extensions == 0: + return dataset + + dataset = dataset.rename_columns({"conversations": "conversations0"}) all_shuffled = [dataset] - for j in range(1, n_extensions+1): - shuffled = dataset.shuffle(seed = random_state+j).rename_columns({"conversations0" : f"conversations{j}"}) + for j in range(1, n_extensions + 1): + shuffled = dataset.shuffle(seed = random_state + j).rename_columns( + {"conversations0": f"conversations{j}"} + ) all_shuffled.append(shuffled) dataset = concatenate_datasets(all_shuffled, axis = 1) @@ -2508,10 +2770,14 @@ def to_sharegpt( for j in range(n_extensions): function += f"{' '*4}conversations{j}__ = examples['conversations{j}']\n" function += f"{' '*4}convos = []\n" - function += f"{' '*4}for ({', '.join(f'conversations{j}' for j in range(n_extensions))}) "\ - f"in zip({', '.join(f'conversations{j}__' for j in range(n_extensions))}):\n" - function += f"{' '*8}convos.append("\ - f"{'+'.join(f'conversations{j}' for j in range(n_extensions))})\n" + function += ( + f"{' '*4}for ({', '.join(f'conversations{j}' for j in range(n_extensions))}) " + f"in zip({', '.join(f'conversations{j}__' for j in range(n_extensions))}):\n" + ) + function += ( + f"{' '*8}convos.append(" + f"{'+'.join(f'conversations{j}' for j in range(n_extensions))})\n" + ) function += f"{' '*4}return " + "{ " + "'conversations' : convos" + " }" # Map function @@ -2535,18 +2801,20 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []): # Remove BOS if getattr(tokenizer, "bos_token", None) is not None: - added_tokens_decoder = [x for x in added_tokens_decoder if x != tokenizer.bos_token] + added_tokens_decoder = [ + x for x in added_tokens_decoder if x != tokenizer.bos_token + ] repeatted_tokens = [] # Join all vocab joined_text = "\x01\x00".join(added_tokens_decoder) for token in added_tokens_decoder: n = len(token) - repeatted_counts = joined_text.count(token[:n//2]) + repeatted_counts = joined_text.count(token[: n // 2]) # Try finding longer than 1/2 of the token in the rest # For eg <|reserved_special_token_0|>, <|reserved_special_token_1|> if repeatted_counts > 2: - for j in range(n//2+1, n): + for j in range(n // 2 + 1, n): if joined_text.count(token[:j]) < repeatted_counts: j -= 1 # Remove repeatted tokens to reduce search space @@ -2556,26 +2824,30 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []): # Remove duplicates splitted = joined_text.split("\x01\x00") - final_eos_tokens = [old for old, new in zip(added_tokens_decoder, splitted) if old == new] + final_eos_tokens = [ + old for old, new in zip(added_tokens_decoder, splitted) if old == new + ] final_eos_tokens += extra_eos_tokens final_eos_tokens += repeatted_tokens # Remove new lines, spaces and HTML tags filtered_eos_tokens = [] for token in final_eos_tokens: - if token.count("\n") == len(token): continue - elif token.count("ā–") == len(token): continue - elif token.startswith("<") and len(token) <= 2: continue - elif token.startswith("<|start_header_id|>system<|end_header_id|> +def construct_chat_template( + tokenizer = None, + chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> {SYSTEM}<|eot_id|><|start_header_id|>user<|end_header_id|> @@ -2586,11 +2858,8 @@ chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> {INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> {OUTPUT}<|eot_id|>""", - -default_system_message = \ - "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.", - -extra_eos_tokens = None, + default_system_message = "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.", + extra_eos_tokens = None, ): """ Creates an Ollama modelfile and a HF Jinja template from a custom @@ -2602,25 +2871,32 @@ extra_eos_tokens = None, # Strip only the left chat_template = chat_template.lstrip() - assert(tokenizer is not None) + assert tokenizer is not None - if extra_eos_tokens is None: extra_eos_tokens = [] - elif type(extra_eos_tokens) is str: extra_eos_tokens = [extra_eos_tokens,] + if extra_eos_tokens is None: + extra_eos_tokens = [] + elif type(extra_eos_tokens) is str: + extra_eos_tokens = [ + extra_eos_tokens, + ] vocab = tokenizer.get_vocab() for extra_eos in extra_eos_tokens: - assert(type(extra_eos) is str) + assert type(extra_eos) is str if extra_eos not in vocab: - raise ValueError(f"Unsloth: `{extra_eos}` is not a singular token in the tokenizer.") + raise ValueError( + f"Unsloth: `{extra_eos}` is not a singular token in the tokenizer." + ) - error_msg = \ - "Unsloth: Your prompt template must have 2 examples showing the user input {INPUT} "\ - "and the assistant output {OUTPUT}\n\n"\ - "For example what is not allowed is just:\n"\ - "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n\n\n"\ - "What is required is 2x of this:\n"\ - "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n"\ + error_msg = ( + "Unsloth: Your prompt template must have 2 examples showing the user input {INPUT} " + "and the assistant output {OUTPUT}\n\n" + "For example what is not allowed is just:\n" + "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n\n\n" + "What is required is 2x of this:\n" "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n" + "### Input:\\n{INPUT}\\n\\n### Response:\\n{OUTPUT}\\n" + ) # Check for EOS after {OUTPUT} if tokenizer.eos_token is not None: @@ -2632,14 +2908,17 @@ extra_eos_tokens = None, # Check tokenizer types tokenizer_name = tokenizer.name_or_path.lower() - if tokenizer_name.startswith(("unsloth/llama-3-8b-instruct", "unsloth/llama-3-70b-instruct")): + if tokenizer_name.startswith( + ("unsloth/llama-3-8b-instruct", "unsloth/llama-3-70b-instruct") + ): # Add <|eot_id|> extra_eos_tokens.append("<|eot_id|>") - elif ("<|eot_id|>" in extra_eos_tokens or "<|eot_id|>" in chat_template) and \ - tokenizer_name.startswith(("unsloth/llama-3-8b", "unsloth/llama-3-70b")): + elif ( + "<|eot_id|>" in extra_eos_tokens or "<|eot_id|>" in chat_template + ) and tokenizer_name.startswith(("unsloth/llama-3-8b", "unsloth/llama-3-70b")): # Warn logger.warning( - "Unsloth: Base llama-3 models did not train <|eot_id|>.\n"\ + "Unsloth: Base llama-3 models did not train <|eot_id|>.\n" "Please use the instruct version or use <|end_of_text|>" ) extra_eos_tokens = list(set(extra_eos_tokens)) @@ -2653,42 +2932,51 @@ extra_eos_tokens = None, try: # O(N^2) search finding 2 repeatted pieces of text - j = len(chat_template)-1 + j = len(chat_template) - 1 at_least_one = False while j > 0: found = chat_template.rfind(chat_template[j:], 0, j) - if found == -1: break + if found == -1: + break j -= 1 at_least_one = True - if j > 0: j += 1 - else: raise RuntimeError(error_msg) + if j > 0: + j += 1 + else: + raise RuntimeError(error_msg) - if not at_least_one: raise RuntimeError(error_msg) + if not at_least_one: + raise RuntimeError(error_msg) # Must be equivalent to left final_combined_check = True # Repeatted text instruction_response = chat_template[j:] - if instruction_response.count("{INPUT}") != 1 or instruction_response.count("{OUTPUT}") != 1: + if ( + instruction_response.count("{INPUT}") != 1 + or instruction_response.count("{OUTPUT}") != 1 + ): raise RuntimeError(error_msg) # 1st System, Instruction, Output pair - left = chat_template[:j] + left = chat_template[:j] # 2nd Instruction, Output pair right = chat_template[j:] final_combined_check = left if final_combined_check else chat_template # Isolate input - extra_eos_tokens_regex = "|".join(f"(?:{re.escape(x)})" for x in extra_eos_tokens) + extra_eos_tokens_regex = "|".join( + f"(?:{re.escape(x)})" for x in extra_eos_tokens + ) if len(extra_eos_tokens_regex) != 0: find_end = f"(?:{extra_eos_tokens_regex})?" else: find_end = "" find_end = r"\{INPUT\}[\s\n]{0,}" + find_end input_end = list(re.finditer(find_end, right)) - assert(len(input_end) == 1) + assert len(input_end) == 1 input_end = input_end[0] input_end = input_end.span(0)[1] input_part = right[:input_end] @@ -2698,46 +2986,63 @@ extra_eos_tokens = None, # Isolate system where_system = left.find(input_part) - system_part = left[:where_system if where_system != -1 else len(left)] + system_part = left[: where_system if where_system != -1 else len(left)] # Check if the user provided a correct prompt combined = system_part + input_part + output_part if combined != final_combined_check: - combined_changed = combined .replace('\n', '\\n') - left_changed = final_combined_check.replace('\n', '\\n') + combined_changed = combined.replace("\n", "\\n") + left_changed = final_combined_check.replace("\n", "\\n") raise RuntimeError( - "Unsloth: The prompt template you provided isn't correct. You gave:\n"\ - f"{combined_changed}\n\n"\ - "But we require the following:\n"\ + "Unsloth: The prompt template you provided isn't correct. You gave:\n" + f"{combined_changed}\n\n" + "But we require the following:\n" f"{left_changed}" ) except: - ending = chat_template[chat_template.find("{OUTPUT}") + len("{OUTPUT}"):] + ending = chat_template[chat_template.find("{OUTPUT}") + len("{OUTPUT}") :] ending = re.escape(ending) find_text = "{INPUT}" + ending + "(.+?{OUTPUT}" + ending + ")" - response_part = re.findall(find_text, chat_template, flags = re.DOTALL | re.MULTILINE) + response_part = re.findall( + find_text, chat_template, flags = re.DOTALL | re.MULTILINE + ) response_part = response_part[0] for j in range(1, len(response_part)): try_find = re.escape(response_part[:j]) - try: found = next(re.finditer("(" + try_find + ").+?\\{INPUT\\}", chat_template, flags = re.DOTALL | re.MULTILINE)) - except: break + try: + found = next( + re.finditer( + "(" + try_find + ").+?\\{INPUT\\}", + chat_template, + flags = re.DOTALL | re.MULTILINE, + ) + ) + except: + break separator = found.group(1) response_start = chat_template.find(response_part) start_instruction = chat_template[:response_start].rfind(separator) - if start_instruction == -1: start_instruction = 0 + if start_instruction == -1: + start_instruction = 0 instruction_part = chat_template[start_instruction:response_start] combined = instruction_part + response_part where = chat_template.find(combined) system_part = chat_template[:where] - system_part, input_part, output_part = system_part, instruction_part, response_part + system_part, input_part, output_part = ( + system_part, + instruction_part, + response_part, + ) if count_eos == 0: - logger.warning("Unsloth: We automatically added an EOS token to stop endless generations.") + logger.warning( + "Unsloth: We automatically added an EOS token to stop endless generations." + ) eos = extra_eos_tokens[0] output_part = output_part + eos @@ -2751,54 +3056,73 @@ extra_eos_tokens = None, always_bos_token = True if ollama_system.startswith(tokenizer.bos_token): has_bos_token = True - ollama_system = ollama_system[len(tokenizer.bos_token):] + ollama_system = ollama_system[len(tokenizer.bos_token) :] # Check system if "{SYSTEM}" in ollama_system: - system_modelfile = "{{ if .System }}" + ollama_system.replace("{SYSTEM}", "{{ .System }}") + "{{ end }}" + system_modelfile = ( + "{{ if .System }}" + + ollama_system.replace("{SYSTEM}", "{{ .System }}") + + "{{ end }}" + ) else: system_modelfile = ollama_system - input_modelfile = "{{ if .Prompt }}" + input_part .replace("{INPUT}", "{{ .Prompt }}") + "{{ end }}" + input_modelfile = ( + "{{ if .Prompt }}" + + input_part.replace("{INPUT}", "{{ .Prompt }}") + + "{{ end }}" + ) output_modelfile = output_part.replace("{OUTPUT}", "{{ .Response }}") # Ollama EOS ollama_eos = get_ollama_eos_tokens(tokenizer, extra_eos_tokens) - ollama_eos = '\n'.join(f'PARAMETER stop "{eos}"' for eos in ollama_eos) + ollama_eos = "\n".join(f'PARAMETER stop "{eos}"' for eos in ollama_eos) # Add temperature and min_p to counteract gibberish ollama_eos += "\nPARAMETER temperature 1.5\nPARAMETER min_p 0.1" # Ollama modelfile part = '"""' - modelfile = 'FROM {__FILE_LOCATION__}\n\n'\ - 'TEMPLATE ' + part + system_modelfile + input_modelfile + output_modelfile + \ - part + '\n\n' + ollama_eos + modelfile = ( + "FROM {__FILE_LOCATION__}\n\n" + "TEMPLATE " + + part + + system_modelfile + + input_modelfile + + output_modelfile + + part + + "\n\n" + + ollama_eos + ) # HF Jinja Chat template def process(part, which, content = "message['content']"): if part.endswith(which): - part = "'" + part[:part.find(which)] + f"' + {content}" + part = "'" + part[: part.find(which)] + f"' + {content}" elif part.startswith(which): - part = f"{content} + '" + part[part.find(which):] + "'" + part = f"{content} + '" + part[part.find(which) :] + "'" else: part = "'" + part.replace(which, f"' + {content} + '") + "'" - if part.startswith("'' + "): part = part[5:] + if part.startswith("'' + "): + part = part[5:] return part - input_jinja = process(input_part, "{INPUT}") + + input_jinja = process(input_part, "{INPUT}") output_jinja = process(output_part, "{OUTPUT}") - jinja_template = \ - "{% for message in loop_messages %}"\ - "{% if message['role'] == 'user' %}"\ - "{{ " + input_jinja + " }}"\ - "{% elif message['role'] == 'assistant' %}"\ - "{{ " + output_jinja + " }}"\ - "{% else %}"\ - "{{ raise_exception('Only user and assistant roles are supported!') }}"\ - "{% endif %}"\ - "{% endfor %}"\ - "{% if add_generation_prompt %}"\ - "{{ '" + output_part[:output_part.find("{OUTPUT}")] + "' }}"\ + jinja_template = ( + "{% for message in loop_messages %}" + "{% if message['role'] == 'user' %}" + "{{ " + input_jinja + " }}" + "{% elif message['role'] == 'assistant' %}" + "{{ " + output_jinja + " }}" + "{% else %}" + "{{ raise_exception('Only user and assistant roles are supported!') }}" "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '" + output_part[: output_part.find("{OUTPUT}")] + "' }}" + "{% endif %}" + ) # Now add system prompt to jinja if len(system_part) != 0: @@ -2812,20 +3136,23 @@ extra_eos_tokens = None, # Separate the BOS if has_bos_token: partial_system = partial_system.replace(tokenizer.bos_token, "", 1) - system_part = system_part .replace(tokenizer.bos_token, "", 1) + system_part = system_part.replace(tokenizer.bos_token, "", 1) - partial_system = \ - "{% if messages[0]['role'] == 'system' %}"\ - "{{ " + partial_system + " }}"\ - "{% set loop_messages = messages[1:] %}" + partial_system = ( + "{% if messages[0]['role'] == 'system' %}" + "{{ " + partial_system + " }}" + "{% set loop_messages = messages[1:] %}" + ) if default_system_message is not None: full_system = system_part.replace("{SYSTEM}", default_system_message) if "{SYSTEM}" in system_part: modelfile += '\nSYSTEM "' + default_system_message + '"' - partial_system += "{% else %}"\ - "{{ '" + full_system + "' }}"\ - "{% set loop_messages = messages %}"\ - "{% endif %}" + partial_system += ( + "{% else %}" + "{{ '" + full_system + "' }}" + "{% set loop_messages = messages %}" + "{% endif %}" + ) else: partial_system += "{% endif %}" @@ -2839,17 +3166,18 @@ extra_eos_tokens = None, jinja_template = jinja_template.replace( "{% for message in loop_messages %}", "{% for message in messages %}", - 1, # Only replace the first one + 1, # Only replace the first one ) # Check if system part is the same! jinja_template = re.sub( - r"\{\% if messages\[0\]\['role'\] \=\= 'system' \%\}\{\{ '(.+?)' \}\}"\ - r"\{\% set loop\_messages \= messages\[1\:\] \%\}"\ - r"\{\% else \%\}\{\{ '\1' \}\}\{\% set loop\_messages \= messages \%\}\{\% endif \%\}"\ + r"\{\% if messages\[0\]\['role'\] \=\= 'system' \%\}\{\{ '(.+?)' \}\}" + r"\{\% set loop\_messages \= messages\[1\:\] \%\}" + r"\{\% else \%\}\{\{ '\1' \}\}\{\% set loop\_messages \= messages \%\}\{\% endif \%\}" r"\{\% for message in loop\_messages \%\}", r"{{ '\1' }}{% for message in messages %}", - jinja_template, flags = re.MULTILINE | re.DOTALL, + jinja_template, + flags = re.MULTILINE | re.DOTALL, ) # Check jinja template for bos @@ -2858,15 +3186,18 @@ extra_eos_tokens = None, jinja_template = "{{ bos_token }}" + jinja_template # Get instruction and output parts for train_on_inputs = False - input_part = input_part [:input_part .find("{INPUT}")] - output_part = output_part[:output_part.find("{OUTPUT}")] + input_part = input_part[: input_part.find("{INPUT}")] + output_part = output_part[: output_part.find("{OUTPUT}")] return modelfile, jinja_template, input_part, output_part def test_construct_chat_template(): token = "hf_" from transformers import AutoTokenizer - tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct", token = token) + + tokenizer = AutoTokenizer.from_pretrained( + "meta-llama/Meta-Llama-3-8B-Instruct", token = token + ) chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> @@ -2880,8 +3211,7 @@ def test_construct_chat_template(): {OUTPUT}<|eot_id|>""" - default_system_message = \ - "Below are some instructions that describe some tasks. Write responses that appropriately complete each request." + default_system_message = "Below are some instructions that describe some tasks. Write responses that appropriately complete each request." extra_eos_tokens = None @@ -2899,19 +3229,21 @@ def test_construct_chat_template(): {"role": "assistant", "content": "Anything else?"}, {"role": "user", "content": "What's 2x2?"}, ] - correct_output = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + correct_output = tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) tokenizer.chat_template = jinja_template - new_output = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) - assert(correct_output == new_output) + new_output = tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) + assert correct_output == new_output -def apply_chat_template( \ - -dataset, -tokenizer = None, - -chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> +def apply_chat_template( + dataset, + tokenizer = None, + chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> {SYSTEM}<|eot_id|><|start_header_id|>user<|end_header_id|> @@ -2922,12 +3254,8 @@ chat_template = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> {INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> {OUTPUT}<|eot_id|>""", - -default_system_message = \ - "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.", - -extra_eos_tokens = None, - + default_system_message = "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.", + extra_eos_tokens = None, ): """ Creates an Ollama modelfile and a HF Jinja template from a custom @@ -2942,27 +3270,42 @@ extra_eos_tokens = None, default_system_message = default_system_message, extra_eos_tokens = extra_eos_tokens, ) + def formatting_prompts_func(examples): convos = examples["conversations"] - texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for convo in convos] - return { "text" : texts, } + texts = [ + tokenizer.apply_chat_template( + convo, tokenize = False, add_generation_prompt = False + ) + for convo in convos + ] + return { + "text": texts, + } tokenizer.chat_template = jinja_template tokenizer._ollama_modelfile = modelfile - tokenizer._unsloth_input_part = input_part + tokenizer._unsloth_input_part = input_part tokenizer._unsloth_output_part = output_part if hasattr(tokenizer, "tokenizer"): tokenizer.tokenizer.chat_template = jinja_template tokenizer.tokenizer._ollama_modelfile = modelfile - tokenizer.tokenizer._unsloth_input_part = input_part + tokenizer.tokenizer._unsloth_input_part = input_part tokenizer.tokenizer._unsloth_output_part = output_part - return dataset.map(formatting_prompts_func, batched = True,) + return dataset.map( + formatting_prompts_func, + batched = True, + ) def create_stopping_criteria(tokenizer, stop_word = "eos_token"): class StoppingCriteriaSub(StoppingCriteria): - __slots__ = "stop_token", "single_match", "length", + __slots__ = ( + "stop_token", + "single_match", + "length", + ) def __init__(self, stops = "eos_token", device = "cuda", encounters = 1): super().__init__() @@ -2970,7 +3313,9 @@ def create_stopping_criteria(tokenizer, stop_word = "eos_token"): self.stop_token = torch.tensor(tokenizer.eos_token_id, device = "cuda") self.length = 1 else: - self.stop_token = tokenizer(["\n" + stops], add_special_tokens = False, return_tensors = "pt") + self.stop_token = tokenizer( + ["\n" + stops], add_special_tokens = False, return_tensors = "pt" + ) self.stop_token = self.stop_token.input_ids.ravel()[1:].to("cuda") self.length = self.stop_token.shape[0] self.single_match = self.length == 1 @@ -2978,18 +3323,26 @@ def create_stopping_criteria(tokenizer, stop_word = "eos_token"): def __call__(self, input_ids: LongTensor, scores: FloatTensor) -> bool: input_ids = input_ids.ravel() last_token = input_ids[-1] - if self.single_match and (last_token == self.stop_token): return True + if self.single_match and (last_token == self.stop_token): + return True - if input_ids.shape[0] >= self.length and \ - (input_ids[-self.length:] == self.stop_token).all(): return True + if ( + input_ids.shape[0] >= self.length + and (input_ids[-self.length :] == self.stop_token).all() + ): + return True return False + stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops = stop_word)]) return stopping_criteria def test_chat_templates(): messages = [ - {"role": "system","content": " You are a friendly chatbot.",}, + { + "role": "system", + "content": " You are a friendly chatbot.", + }, {"role": "user", "content": "What is 2+2?"}, {"role": "assistant", "content": "It's 4."}, {"role": "user", "content": " But 2+2 is equal to 5. "}, @@ -2999,36 +3352,57 @@ def test_chat_templates(): # Zephyr from transformers import AutoTokenizer + template = zephyr_template correct_tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-beta") - correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + correct_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) - assert(correct_prompt == our_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) + assert correct_prompt == our_prompt # Chatml template = chatml_template - correct_tokenizer = AutoTokenizer.from_pretrained("teknium/OpenHermes-2.5-Mistral-7B") - correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + correct_tokenizer = AutoTokenizer.from_pretrained( + "teknium/OpenHermes-2.5-Mistral-7B" + ) + correct_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) - assert(correct_prompt == our_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) + assert correct_prompt == our_prompt # Mistral template = mistral_template - correct_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2") - correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) + correct_tokenizer = AutoTokenizer.from_pretrained( + "mistralai/Mistral-7B-Instruct-v0.2" + ) + correct_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) - assert(correct_prompt == our_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) + assert correct_prompt == our_prompt # Llama template = llama_template correct_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-2-7b-chat") - correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + correct_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) - assert(correct_prompt == our_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) + assert correct_prompt == our_prompt # Vicuna try: @@ -3037,16 +3411,20 @@ def test_chat_templates(): os.system("pip -qqq install git+https://github.com/lm-sys/FastChat.git") from fastchat.conversation import get_conv_template correct_prompt = get_conv_template("vicuna_v1.1") - for j in range(len(messages)-1): - correct_prompt.append_message(correct_prompt.roles[j%2==1], messages[j+1]["content"]) + for j in range(len(messages) - 1): + correct_prompt.append_message( + correct_prompt.roles[j % 2 == 1], messages[j + 1]["content"] + ) correct_prompt.append_message(correct_prompt.roles[1], "") correct_prompt = tokenizer.bos_token + correct_prompt.get_prompt() template = vicuna_template correct_tokenizer = AutoTokenizer.from_pretrained("lmsys/vicuna-7b-v1.5") correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) - assert(correct_prompt == our_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) + assert correct_prompt == our_prompt try: from fastchat.conversation import get_conv_template @@ -3054,49 +3432,68 @@ def test_chat_templates(): os.system("pip -qqq install git+https://github.com/lm-sys/FastChat.git") from fastchat.conversation import get_conv_template correct_prompt = get_conv_template("zero_shot") - for j in range(len(messages)-1): - correct_prompt.append_message(correct_prompt.roles[j%2==1], messages[j+1]["content"]) + for j in range(len(messages) - 1): + correct_prompt.append_message( + correct_prompt.roles[j % 2 == 1], messages[j + 1]["content"] + ) correct_prompt.append_message(correct_prompt.roles[1], "") correct_prompt = tokenizer.bos_token + correct_prompt.get_prompt() template = vicuna_old_template correct_tokenizer = AutoTokenizer.from_pretrained("lmsys/vicuna-7b-v1.5") correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) + our_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) # We add ourselves - assert(correct_prompt == our_prompt.replace("", "")) + assert correct_prompt == our_prompt.replace("", "") # Gemma correct_tokenizer = AutoTokenizer.from_pretrained("unsloth/gemma-7b-it") - correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) + correct_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) correct_tokenizer.chat_template = gemma_template - our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) - assert(our_prompt == correct_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) + assert our_prompt == correct_prompt # Llama-3 template = llama3_template correct_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b-Instruct") - correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + correct_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) - assert(correct_prompt == our_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) + assert correct_prompt == our_prompt # Phi-3 template = phi3_template - correct_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct") - correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) + correct_tokenizer = AutoTokenizer.from_pretrained( + "microsoft/Phi-3-mini-4k-instruct" + ) + correct_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) - assert(correct_prompt == our_prompt) + our_prompt = correct_tokenizer.apply_chat_template( + messages[1:], tokenize = False, add_generation_prompt = True + ) + assert correct_prompt == our_prompt def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf"): """ - Carefully checks the output of GGUF's tokenization and HF. - Can catch all tokenization bugs. + Carefully checks the output of GGUF's tokenization and HF. + Can catch all tokenization bugs. """ import subprocess import re + messages = [ {"role": "user", "content": "What is 2+2?"}, {"role": "assistant", "content": "It's 4."}, @@ -3115,31 +3512,51 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf") ### Response: {}""".format( - "Describe the city given eloquently.", # instruction - "The lost city of Atlantis.", # input - "", # output - leave this blank for generation! + "Describe the city given eloquently.", # instruction + "The lost city of Atlantis.", # input + "", # output - leave this blank for generation! ) - prompts = [ prompt, ] + prompts = [ + prompt, + ] if tokenizer.chat_template is not None: - prompt = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) - prompt = prompt.replace("'", "") # Subprocess does not like '' + prompt = tokenizer.apply_chat_template( + messages, tokenize = False, add_generation_prompt = True + ) + prompt = prompt.replace("'", "") # Subprocess does not like '' prompt = remove_special_tokens(tokenizer, prompt) prompts.append(prompt) for prompt in prompts: - command = f"./llama.cpp/llama-cli -m {gguf_model} -n 0 --temp 0.0 --verbose-prompt "\ + command = ( + f"./llama.cpp/llama-cli -m {gguf_model} -n 0 --temp 0.0 --verbose-prompt " f"--check-tensors -p '{prompt}'" + ) datas = [] - with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1) as sp: + with subprocess.Popen( + command, + shell = True, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + bufsize = 1, + ) as sp: for line in sp.stdout: datas.append(line.decode("utf-8", errors = "replace")) gguf_tokens = "".join(datas) # Now extract GGUF tokenization attempt - gguf_tokenized = re.findall(r"([\d]{1,}) \-\> \'([^\']{1,})\'", gguf_tokens, flags = re.MULTILINE) - gguf_tokenized = [(int(x[0]), x[1],) for x in gguf_tokenized] + gguf_tokenized = re.findall( + r"([\d]{1,}) \-\> \'([^\']{1,})\'", gguf_tokens, flags = re.MULTILINE + ) + gguf_tokenized = [ + ( + int(x[0]), + x[1], + ) + for x in gguf_tokenized + ] input_ids = tokenizer(prompt).input_ids tokens = tokenizer.batch_decode(input_ids) @@ -3147,7 +3564,7 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf") # Compare to Huggingface for j, (hf_token, gguf_token) in enumerate(zip(hf_tokenized, gguf_tokenized)): - if (hf_token[0] != gguf_token[0]): + if hf_token[0] != gguf_token[0]: print("Failed GGUF != HF at", j) print("HF =", hf_token) print("GGUF =", gguf_token) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index ba010edabb..a643340d44 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re import json import csv -from typing import List, Dict, Any, Union, Optional from datasets import Dataset from pathlib import Path diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 7735b1ecd2..ac7fd52319 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -25,7 +25,6 @@ os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" import requests import torch import gc -import time import re from unsloth_zoo.vllm_utils import ( load_vllm, diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 4fbab6a94c..d4e0ac3509 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -69,7 +69,7 @@ class HideLoggingMessage(logging.Filter): self.text = text def filter(self, x): - return not (self.text in x.getMessage()) + return self.text not in x.getMessage() class HidePrintMessage: diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py index fbb14013ff..ad3e3984dd 100644 --- a/unsloth/kernels/cross_entropy_loss.py +++ b/unsloth/kernels/cross_entropy_loss.py @@ -23,12 +23,10 @@ from .utils import ( torch_gpu_device, is_cdna, ) -from transformers.models.llama.modeling_llama import logger from unsloth_zoo.utils import Version from unsloth_zoo.loss_utils import ( patch_loss_functions as _patch_loss_functions, - post_patch_loss_function, ) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index f1c0e298d9..0a5eb62223 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -18,7 +18,6 @@ from .utils import ( fast_dequantize, QUANT_STATE, get_lora_parameters, - get_lora_parameters_bias, matmul_lora, torch_amp_custom_fwd, torch_amp_custom_bwd, diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.py index b94ff56dec..e65dc7b7b0 100644 --- a/unsloth/kernels/flex_attention.py +++ b/unsloth/kernels/flex_attention.py @@ -13,8 +13,6 @@ # limitations under the License. import torch -from functools import lru_cache -from transformers.models.llama.modeling_llama import logger import os torch_compile_options = { @@ -80,7 +78,8 @@ else: # See https://github.com/pytorch-labs/attention-gym/blob/main/examples/flex_attn.ipynb # for more examples # BSD 3-Clause License Copyright (c) 2023, Driss Guessous, Horace He et al - import functools, math + import functools + import math def generate_tanh_softcap(t): def tanh_softcap(x, b, h, q_idx, kv_idx): diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py index e9f9161709..fa9bb2f339 100644 --- a/unsloth/kernels/fp8.py +++ b/unsloth/kernels/fp8.py @@ -13,11 +13,8 @@ # limitations under the License. import os import torch -import torch.nn as nn import triton import triton.language as tl -from torch.nn import functional as F -import math from unsloth_zoo.utils import Version from unsloth_zoo.log import logger from unsloth_zoo.temporary_patches.common import torch_compile @@ -582,7 +579,7 @@ try: # This check is a must for consumer grade GPUs which fail if test_has_fbgemm(): os.environ["UNSLOTH_HAS_FBGEMM"] = "1" - logger.info(f"Using fbgemm_gpu block quantized FP8 matmul") + logger.info("Using fbgemm_gpu block quantized FP8 matmul") fp8_block_quant_linear = fp8_fbgemm_block_linear else: os.environ["UNSLOTH_HAS_FBGEMM"] = "0" diff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.py index 50b4e521d3..e3d51715a7 100644 --- a/unsloth/kernels/geglu.py +++ b/unsloth/kernels/geglu.py @@ -16,7 +16,6 @@ import triton import triton.language as tl import torch from .utils import ( - calculate_settings, triton_tanh, torch_gpu_device, ) diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py index 5e2e3af2f8..adcb4aba6e 100644 --- a/unsloth/kernels/layernorm.py +++ b/unsloth/kernels/layernorm.py @@ -17,9 +17,6 @@ import triton import triton.language as tl import torch from .utils import calculate_settings, torch_gpu_device -from unsloth_zoo.patching_utils import ( - patch_layernorm, -) @triton.jit diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.py index f23d9688ea..9b5d1d7d40 100644 --- a/unsloth/kernels/moe/autotune_cache.py +++ b/unsloth/kernels/moe/autotune_cache.py @@ -23,9 +23,8 @@ import json import logging import os import time -from typing import Dict, List, Optional, Tuple, Any +from typing import Dict, Optional, Tuple, Any import torch -import triton logger = logging.getLogger(__name__) diff --git a/unsloth/kernels/moe/benchmark/utils.py b/unsloth/kernels/moe/benchmark/utils.py index 21905d8df1..0e2ab317d9 100644 --- a/unsloth/kernels/moe/benchmark/utils.py +++ b/unsloth/kernels/moe/benchmark/utils.py @@ -178,7 +178,7 @@ def save_autotune_results(autotune_cache, mode, ref_time, fused_time, results_di for key, config in autotune_cache.items(): key = [ - str(k) if not "torch" in str(k) else str(k.split("torch.")[-1]) for k in key + str(k) if "torch" not in str(k) else str(k.split("torch.")[-1]) for k in key ] filename = "_".join(key) save_path = f"{save_dir}/{filename}.json" diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py index 572b02702b..787322d589 100644 --- a/unsloth/kernels/moe/grouped_gemm/interface.py +++ b/unsloth/kernels/moe/grouped_gemm/interface.py @@ -3,7 +3,6 @@ import logging import warnings -from dataclasses import asdict import torch import triton diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py b/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py index d25913975e..38d06ebec0 100644 --- a/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py +++ b/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py @@ -336,7 +336,6 @@ def exceeds_smem_capacity( def common_prune_criteria(config: triton.Config, kwargs: dict, dtype): - from ..interface import supports_tma from .tuning import get_device_properties smem_size = get_device_properties().SIZE_SMEM diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py index 5e07056b52..0ea1be8294 100644 --- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py +++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GNU Affero General Public License v3.0 # Copyright 2023-present the Unsloth team. All rights reserved. -import torch import triton import triton.language as tl diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py index a42ec5ffe9..cb8264ea9d 100644 --- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py +++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GNU Affero General Public License v3.0 # Copyright 2023-present the Unsloth team. All rights reserved. -import torch import triton import triton.language as tl diff --git a/unsloth/kernels/moe/tests/test_grouped_gemm.py b/unsloth/kernels/moe/tests/test_grouped_gemm.py index bd98b6a276..23e1910e38 100644 --- a/unsloth/kernels/moe/tests/test_grouped_gemm.py +++ b/unsloth/kernels/moe/tests/test_grouped_gemm.py @@ -7,7 +7,6 @@ import pytest import torch from grouped_gemm.interface import ( - grouped_gemm, grouped_gemm_dW, grouped_gemm_dX, grouped_gemm_forward, @@ -582,7 +581,6 @@ def _test_grouped_gemm_backward_dX( kernel_config_bwd_dW = KernelConfigBackward_dW() else: from grouped_gemm.kernels.backward import ( - _autotuned_grouped_gemm_dW_kernel, _autotuned_grouped_gemm_dX_kernel, ) from grouped_gemm.kernels.forward import ( diff --git a/unsloth/kernels/moe/tests/test_llama4_moe.py b/unsloth/kernels/moe/tests/test_llama4_moe.py index 13ad552bf4..0b22b94dbd 100644 --- a/unsloth/kernels/moe/tests/test_llama4_moe.py +++ b/unsloth/kernels/moe/tests/test_llama4_moe.py @@ -2,7 +2,6 @@ # Copyright 2023-present the Unsloth team. All rights reserved. import argparse -import sys from contextlib import contextmanager from functools import partial diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py index b3ae9d40e6..6ed686839e 100644 --- a/unsloth/kernels/swiglu.py +++ b/unsloth/kernels/swiglu.py @@ -15,7 +15,7 @@ import triton import triton.language as tl import torch -from .utils import calculate_settings, torch_gpu_device +from .utils import torch_gpu_device # signed int32 max is 2**31-1 so num_elements cannot exceed 2**31 NUM_INT32_ELEMENTS = 2**31 diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 5dcc7c232c..d750d094fb 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -19,18 +19,13 @@ import ctypes MAX_FUSED_SIZE: int = 65536 next_power_of_2 = triton.next_power_of_2 import functools -from typing import Optional from ..device_type import ( is_hip, - get_device_type, DEVICE_TYPE, - DEVICE_TYPE_TORCH, DEVICE_COUNT, - ALLOW_PREQUANTIZED_MODELS, ) from .fp8 import weight_dequant, fp8_linear -import functools # torch.cuda.amp.custom_fwd is deprecated >= 2.4 import torch diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 48e5683076..a5f8560d90 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -85,34 +85,25 @@ from typing import Union, Optional, List, Any, Callable, Tuple, Iterator from platform import system as platform_system platform_system = platform_system() -import numpy as np -import contextlib import re from dataclasses import dataclass, field import functools import textwrap import logging -import warnings, subprocess, inspect, psutil, os, math +import warnings +import inspect +import psutil +import os from unsloth_zoo.utils import Version, get_quant_type from importlib.metadata import version as importlib_version from ..device_type import ( - is_hip, - get_device_type, DEVICE_TYPE, - DEVICE_TYPE_TORCH, DEVICE_COUNT, - ALLOW_PREQUANTIZED_MODELS, ) from unsloth_zoo.log import logger from unsloth_zoo.tokenizer_utils import ( patch_tokenizer as _patch_tokenizer, ) -from unsloth_zoo.rl_environments import ( - check_python_modules, - create_locked_down_function, - execute_with_time_limit, - Benchmarker, -) from unsloth_zoo.patching_utils import ( patch_compiling_bitsandbytes, patch_layernorm, @@ -125,8 +116,6 @@ from unsloth_zoo.gradient_checkpointing import ( unsloth_offloaded_gradient_checkpoint, patch_unsloth_gradient_checkpointing, unpatch_unsloth_gradient_checkpointing, - Unsloth_Gradient_Checkpointer, - unsloth_gradient_checkpoint, patch_gradient_checkpointing, unpatch_gradient_checkpointing, patch_unsloth_smart_gradient_checkpointing, @@ -252,7 +241,7 @@ class HideLoggingMessage(logging.Filter): self.text = text def filter(self, x): - return not (self.text in x.getMessage()) + return self.text not in x.getMessage() # Stop vLLM messages @@ -598,21 +587,21 @@ def patch_mistral_nemo_config(config): try: # Some Config files use layer_type_validation # for eg Gemma-2, so we must import it to stop errors. - from transformers.configuration_utils import layer_type_validation + pass except: pass try: # Transformers 5.0+ uses RotaryEmbeddingConfigMixin as a base class for configs - from transformers.modeling_rope_utils import RotaryEmbeddingConfigMixin + pass except: pass from transformers import __version__ as transformers_version try: - from transformers import PreTrainedConfig + pass except: - from transformers import PretrainedConfig + pass model_architectures = [ "llama", @@ -711,7 +700,7 @@ from transformers.utils import is_openai_available if is_openai_available(): try: - from openai import OpenAI + pass except: print("Unsloth: OpenAI failed to import - ignoring for now.") import transformers.utils @@ -723,9 +712,7 @@ if is_openai_available(): # ============================================= # Get Flash Attention v2 if Ampere (RTX 30xx, A100) -import bitsandbytes as bnb -from transformers import AutoTokenizer from transformers.utils.import_utils import _is_package_available SUPPORTS_BFLOAT16 = False @@ -743,9 +730,9 @@ if DEVICE_TYPE == "cuda": try: try: # See https://github.com/unslothai/unsloth/issues/1437 - from flash_attn.flash_attn_interface import flash_attn_gpu + pass except: - from flash_attn.flash_attn_interface import flash_attn_cuda + pass HAS_FLASH_ATTENTION = True # Also check for softcapping @@ -795,9 +782,9 @@ elif DEVICE_TYPE == "hip": try: try: # See https://github.com/unslothai/unsloth/issues/1437 - from flash_attn.flash_attn_interface import flash_attn_gpu + pass except: - from flash_attn.flash_attn_interface import flash_attn_cuda + pass HAS_FLASH_ATTENTION = True # Also check for softcapping @@ -1148,7 +1135,7 @@ USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" if USE_MODELSCOPE: if importlib.util.find_spec("modelscope") is None: raise ImportError( - f"You are using the modelscope hub, please install modelscope by `pip install modelscope -U`" + "You are using the modelscope hub, please install modelscope by `pip install modelscope -U`" ) import socket @@ -1166,13 +1153,10 @@ def has_internet(host = "8.8.8.8", port = 53, timeout = 3): return True finally: sock.close() - except socket.error as ex: + except socket.error: return False -import psutil - - def _get_statistics(statistics = None, force_download = True): # We log some basic stats about which environment is being used. # We simply download a README.md file from HF - all data is made public. @@ -1339,7 +1323,6 @@ def get_statistics(local_files_only = False): # Fixes Bitsandbytes to remove missing warnings from transformers.utils.quantization_config import ( BitsAndBytesConfig, - QuantizationMethod, ) BitsAndBytesConfig__init__ = inspect.getsource(BitsAndBytesConfig.__init__) diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index c33317ee02..e84c0697a4 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -13,7 +13,6 @@ # limitations under the License. from .llama import * -from ._utils import __version__ from unsloth_zoo.hf_utils import dtype_from_config from unsloth_zoo.utils import _get_dtype, Version from ..utils.packing import get_packed_info_from_kwargs @@ -30,9 +29,6 @@ try: CohereDecoderLayer, CohereModel, CohereForCausalLM, - CohereRotaryEmbedding, - apply_rotary_pos_emb, - repeat_kv, ) except: transformers_version = Version(transformers_version) diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py index 428f49d727..bf32b3be03 100644 --- a/unsloth/models/falcon_h1.py +++ b/unsloth/models/falcon_h1.py @@ -13,8 +13,6 @@ # limitations under the License. from .llama import * -import os -from ._utils import __version__ from unsloth_zoo.utils import Version, _get_dtype from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import get_packed_info_from_kwargs @@ -28,7 +26,6 @@ from ..utils.attention_dispatch import ( from .llama import ( LlamaRotaryEmbedding, LlamaLinearScalingRotaryEmbedding, - _LlamaModel_fast_forward_inference, ) try: @@ -37,7 +34,6 @@ try: FalconH1DecoderLayer, FalconH1Model, FalconH1ForCausalLM, - FalconHybridMambaAttentionDynamicCache, ) except: from transformers import __version__ as transformers_version diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 1789a9cd92..86f6b8925c 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -13,14 +13,8 @@ # limitations under the License. from .llama import * -from ._utils import __version__ from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config -from ..utils.packing import ( - build_sdpa_packed_attention_mask, - build_xformers_block_causal_mask, - get_packed_info_from_kwargs, -) import math try: @@ -29,9 +23,6 @@ try: GemmaDecoderLayer, GemmaModel, GemmaForCausalLM, - GemmaRotaryEmbedding, - apply_rotary_pos_emb, - repeat_kv, ) except: transformers_version = Version(transformers_version) diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py index 16d04955d3..87e422d0b1 100644 --- a/unsloth/models/gemma2.py +++ b/unsloth/models/gemma2.py @@ -13,7 +13,6 @@ # limitations under the License. from .llama import * -from ._utils import __version__ from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import get_packed_info_from_kwargs @@ -22,7 +21,6 @@ from ..utils.attention_dispatch import ( AttentionContext, run_attention, select_attention_backend, - SDPA, ) from .gemma import ( GemmaFixedRotaryEmbedding, @@ -36,9 +34,6 @@ try: Gemma2DecoderLayer, Gemma2Model, Gemma2ForCausalLM, - Gemma2RotaryEmbedding, - apply_rotary_pos_emb, - repeat_kv, ) except: transformers_version = Version(transformers_version) @@ -65,7 +60,7 @@ except: Gemma2FlashAttention2 = Gemma2Attention if HAS_FLASH_ATTENTION_SOFTCAPPING: - from flash_attn import flash_attn_func + pass # Logit softcapping diff --git a/unsloth/models/glm4_moe.py b/unsloth/models/glm4_moe.py index 5d04b2f1d0..05a7d77e79 100644 --- a/unsloth/models/glm4_moe.py +++ b/unsloth/models/glm4_moe.py @@ -25,20 +25,11 @@ Key architecture differences from Qwen3 MoE: from .llama import * import os -from ._utils import __version__ from .llama import ( - LlamaRotaryEmbedding, - LlamaLinearScalingRotaryEmbedding, - fix_prepare_inputs_for_generation, fast_rms_layernorm_inference, fast_swiglu_inference, - LlamaModel_fast_forward, - LlamaModel_fast_forward_inference, - CausalLM_fast_forward, - PeftModel_fast_forward, ) import torch -import torch.nn.functional as F from typing import Optional, Tuple from ..kernels import fast_rms_layernorm diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index aae746aed1..8d9ca55478 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -13,7 +13,6 @@ # limitations under the License. from .llama import * -import os from ._utils import __version__ from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config @@ -265,8 +264,6 @@ def GraniteDecoderLayer_fast_forward( return outputs -from math import sqrt as math_sqrt - KV_CACHE_INCREMENT = 256 # KV Cache update size torch_nn_functional_softmax = torch.nn.functional.softmax torch_matmul = torch.matmul @@ -285,7 +282,7 @@ def GraniteAttention_fast_forward_inference( ): assert ( position_embeddings is not None - ), f"Granite model requires position embeddings to be specified" + ), "Granite model requires position embeddings to be specified" Xn = hidden_states bsz, _, hd = hidden_states.size() diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index f18a07ac3c..60acae25b9 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -46,12 +46,9 @@ from unsloth_zoo.hf_utils import ( ) from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES from ..device_type import ( - is_hip, - get_device_type, DEVICE_TYPE, DEVICE_TYPE_TORCH, DEVICE_COUNT, - ALLOW_PREQUANTIZED_MODELS, ) transformers_version = Version(transformers_version) @@ -93,7 +90,6 @@ except: LlamaFlashAttention2 = LlamaAttention from transformers import ( - AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification, BitsAndBytesConfig, @@ -104,14 +100,16 @@ from transformers import set_seed as transformers_set_seed from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model from peft import PeftModelForCausalLM, PeftModelForSequenceClassification from ..save import patch_saving_functions -import re, os, inspect, math, sys +import re +import os +import inspect import types try: - from huggingface_hub.utils import get_token + pass except: # Old HF Hub versions <= 0.0.25 - from huggingface_hub.utils._token import get_token + pass from triton import __version__ as triton_version HAS_XFORMERS = xformers is not None @@ -2903,7 +2901,7 @@ class FastLlamaModel: try: assert module in accepted_modules final_modules.append(module) - except AssertionError as e: + except AssertionError: final_modules.append(module) print( "Unsloth: You added custom modules, but Unsloth hasn't optimized for this.\n" diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index fd869c7b5f..e3f9570f8f 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -15,20 +15,17 @@ from ._utils import ( _prepare_model_for_qat, is_bfloat16_supported, - is_vLLM_available, HAS_FLASH_ATTENTION, HAS_FLASH_ATTENTION_SOFTCAPPING, USE_MODELSCOPE, get_transformers_model_type, hf_login, ) -from .granite import FastGraniteModel from .llama import FastLlamaModel, logger from .mistral import FastMistralModel from .qwen2 import FastQwen2Model from .qwen3 import FastQwen3Model from .qwen3_moe import FastQwen3MoeModel -from .cohere import FastCohereModel from transformers import AutoConfig from transformers import __version__ as transformers_version from peft import PeftConfig, PeftModel @@ -38,22 +35,20 @@ from .loader_utils import ( _tag_model_with_fp8_torchao_config, get_model_name, ) -import os, contextlib, sys +import os +import contextlib try: - from huggingface_hub import get_token + pass except: try: - from huggingface_hub.utils import get_token + pass except: # For older versions of huggingface_hub - from huggingface_hub.utils._token import get_token + pass from huggingface_hub import HfFileSystem import importlib.util from ..device_type import ( - is_hip, - get_device_type, - DEVICE_TYPE, DEVICE_TYPE_TORCH, DEVICE_COUNT, ALLOW_PREQUANTIZED_MODELS, @@ -82,15 +77,12 @@ if SUPPORTS_GEMMA: if SUPPORTS_GEMMA2: from .gemma2 import FastGemma2Model if SUPPORTS_FALCON_H1: - from .falcon_h1 import FastFalconH1Model + pass import torch from ._utils import ( patch_compiling_bitsandbytes, - patch_model_and_tokenizer, - prepare_model_for_kbit_training, apply_unsloth_gradient_checkpointing, patch_compiled_autograd, - process_vision_info, unsloth_compile_transformers, fast_inference_setup, ) @@ -679,7 +671,6 @@ class FastLanguageModel(FastLlamaModel): from ..kernels import ( patch_loss_functions, - post_patch_loss_function, ) from .vision import FastBaseModel from transformers import ( diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 1e5533c25c..8557b95a33 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -16,7 +16,6 @@ from ..device_type import DEVICE_TYPE_TORCH import importlib import os import torch -import re import tempfile from typing import Union from .mapper import ( diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index e7296291f1..aea9d2811b 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -20,1249 +20,1228 @@ __all__ = [ "FLOAT_TO_FP8_ROW_MAPPER", ] -__INT_TO_FLOAT_MAPPER = \ -{ - "unsloth/mistral-7b-bnb-4bit" : ( +__INT_TO_FLOAT_MAPPER = { + "unsloth/mistral-7b-bnb-4bit": ( "unsloth/mistral-7b", "mistralai/Mistral-7B-v0.1", ), - "unsloth/llama-2-7b-bnb-4bit" : ( + "unsloth/llama-2-7b-bnb-4bit": ( "unsloth/llama-2-7b", "meta-llama/Llama-2-7b-hf", ), - "unsloth/llama-2-13b-bnb-4bit" : ( + "unsloth/llama-2-13b-bnb-4bit": ( "unsloth/llama-2-13b", "meta-llama/Llama-2-13b-hf", ), - "unsloth/codellama-34b-bnb-4bit" : ( - "codellama/CodeLlama-34b-hf", - ), - "unsloth/zephyr-sft-bnb-4bit" : ( + "unsloth/codellama-34b-bnb-4bit": ("codellama/CodeLlama-34b-hf",), + "unsloth/zephyr-sft-bnb-4bit": ( "unsloth/zephyr-sft", "HuggingFaceH4/mistral-7b-sft-beta", ), - "unsloth/tinyllama-bnb-4bit" : ( + "unsloth/tinyllama-bnb-4bit": ( "unsloth/tinyllama", "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", ), - "unsloth/tinyllama-chat-bnb-4bit" : ( + "unsloth/tinyllama-chat-bnb-4bit": ( "unsloth/tinyllama-chat", "TinyLlama/TinyLlama-1.1B-Chat-v1.0", ), - "unsloth/mistral-7b-instruct-v0.1-bnb-4bit" : ( + "unsloth/mistral-7b-instruct-v0.1-bnb-4bit": ( "unsloth/mistral-7b-instruct-v0.1", "mistralai/Mistral-7B-Instruct-v0.1", ), - "unsloth/mistral-7b-instruct-v0.2-bnb-4bit" : ( + "unsloth/mistral-7b-instruct-v0.2-bnb-4bit": ( "unsloth/mistral-7b-instruct-v0.2", "mistralai/Mistral-7B-Instruct-v0.2", ), - "unsloth/llama-2-7b-chat-bnb-4bit" : ( + "unsloth/llama-2-7b-chat-bnb-4bit": ( "unsloth/llama-2-7b-chat", "meta-llama/Llama-2-7b-chat-hf", ), - "unsloth/llama-2-7b-chat-bnb-4bit" : ( + "unsloth/llama-2-7b-chat-bnb-4bit": ( "unsloth/llama-2-7b-chat", "meta-llama/Llama-2-7b-chat-hf", ), - "unsloth/Mixtral-8x7B-v0.1-unsloth-bnb-4bit" : ( + "unsloth/Mixtral-8x7B-v0.1-unsloth-bnb-4bit": ( "unsloth/Mixtral-8x7B-v0.1", "mistralai/Mixtral-8x7B-v0.1", "unsloth/Mixtral-8x7B-v0.1-bnb-4bit", ), - "unsloth/Mixtral-8x7B-Instruct-v0.1-unsloth-bnb-4bit" : ( + "unsloth/Mixtral-8x7B-Instruct-v0.1-unsloth-bnb-4bit": ( "unsloth/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x7B-Instruct-v0.1", "unsloth/Mixtral-8x7B-Instruct-v0.1-bnb-4bit", ), - "unsloth/codellama-7b-bnb-4bit" : ( + "unsloth/codellama-7b-bnb-4bit": ( "unsloth/codellama-7b", "codellama/CodeLlama-7b-hf", ), - "unsloth/codellama-13b-bnb-4bit" : ( - "codellama/CodeLlama-13b-hf", - ), - "unsloth/yi-6b-bnb-4bit" : ( + "unsloth/codellama-13b-bnb-4bit": ("codellama/CodeLlama-13b-hf",), + "unsloth/yi-6b-bnb-4bit": ( "unsloth/yi-6b", "01-ai/Yi-6B", ), - "unsloth/solar-10.7b-bnb-4bit" : ( - "upstage/SOLAR-10.7B-v1.0", - ), - "unsloth/gemma-7b-bnb-4bit" : ( + "unsloth/solar-10.7b-bnb-4bit": ("upstage/SOLAR-10.7B-v1.0",), + "unsloth/gemma-7b-bnb-4bit": ( "unsloth/gemma-7b", "google/gemma-7b", ), - "unsloth/gemma-2b-bnb-4bit" : ( + "unsloth/gemma-2b-bnb-4bit": ( "unsloth/gemma-2b", "google/gemma-2b", ), - "unsloth/gemma-7b-it-bnb-4bit" : ( + "unsloth/gemma-7b-it-bnb-4bit": ( "unsloth/gemma-7b-it", "google/gemma-7b-it", ), - "unsloth/gemma-2b-bnb-4bit" : ( + "unsloth/gemma-2b-bnb-4bit": ( "unsloth/gemma-2b-it", "google/gemma-2b-it", ), - "unsloth/mistral-7b-v0.2-bnb-4bit" : ( + "unsloth/mistral-7b-v0.2-bnb-4bit": ( "unsloth/mistral-7b-v0.2", "alpindale/Mistral-7B-v0.2-hf", ), - "unsloth/gemma-1.1-2b-it-bnb-4bit" : ( + "unsloth/gemma-1.1-2b-it-bnb-4bit": ( "unsloth/gemma-1.1-2b-it", "google/gemma-1.1-2b-it", ), - "unsloth/gemma-1.1-7b-it-bnb-4bit" : ( + "unsloth/gemma-1.1-7b-it-bnb-4bit": ( "unsloth/gemma-1.1-7b-it", "google/gemma-1.1-7b-it", ), - "unsloth/Starling-LM-7B-beta" : ( + "unsloth/Starling-LM-7B-beta": ( "unsloth/Starling-LM-7B-beta", "Nexusflow/Starling-LM-7B-beta", ), - "unsloth/Hermes-2-Pro-Mistral-7B-bnb-4bit" : ( + "unsloth/Hermes-2-Pro-Mistral-7B-bnb-4bit": ( "unsloth/Hermes-2-Pro-Mistral-7B", "NousResearch/Hermes-2-Pro-Mistral-7B", ), - "unsloth/OpenHermes-2.5-Mistral-7B-bnb-4bit" : ( + "unsloth/OpenHermes-2.5-Mistral-7B-bnb-4bit": ( "unsloth/OpenHermes-2.5-Mistral-7B", "teknium/OpenHermes-2.5-Mistral-7B", ), - "unsloth/codegemma-2b-bnb-4bit" : ( + "unsloth/codegemma-2b-bnb-4bit": ( "unsloth/codegemma-2b", "google/codegemma-2b", ), - "unsloth/codegemma-7b-bnb-4bit" : ( + "unsloth/codegemma-7b-bnb-4bit": ( "unsloth/codegemma-7b", "google/codegemma-7b", ), - "unsloth/codegemma-7b-it-bnb-4bit" : ( + "unsloth/codegemma-7b-it-bnb-4bit": ( "unsloth/codegemma-7b-it", "google/codegemma-7b-it", ), - "unsloth/llama-3-8b-bnb-4bit" : ( + "unsloth/llama-3-8b-bnb-4bit": ( "unsloth/llama-3-8b", "meta-llama/Meta-Llama-3-8B", ), - "unsloth/llama-3-8b-Instruct-bnb-4bit" : ( + "unsloth/llama-3-8b-Instruct-bnb-4bit": ( "unsloth/llama-3-8b-Instruct", "meta-llama/Meta-Llama-3-8B-Instruct", ), - "unsloth/llama-3-70b-bnb-4bit" : ( - "meta-llama/Meta-Llama-3-70B", - ), - "unsloth/llama-3-70b-Instruct-bnb-4bit" : ( - "meta-llama/Meta-Llama-3-70B-Instruct", - ), - "unsloth/Phi-3-mini-4k-instruct-bnb-4bit" : ( + "unsloth/llama-3-70b-bnb-4bit": ("meta-llama/Meta-Llama-3-70B",), + "unsloth/llama-3-70b-Instruct-bnb-4bit": ("meta-llama/Meta-Llama-3-70B-Instruct",), + "unsloth/Phi-3-mini-4k-instruct-bnb-4bit": ( "unsloth/Phi-3-mini-4k-instruct", "microsoft/Phi-3-mini-4k-instruct", ), - "unsloth/mistral-7b-v0.3-bnb-4bit" : ( + "unsloth/mistral-7b-v0.3-bnb-4bit": ( "unsloth/mistral-7b-v0.3", "mistralai/Mistral-7B-v0.3", ), - "unsloth/mistral-7b-instruct-v0.3-bnb-4bit" : ( + "unsloth/mistral-7b-instruct-v0.3-bnb-4bit": ( "unsloth/mistral-7b-instruct-v0.3", "mistralai/Mistral-7B-Instruct-v0.3", ), - "unsloth/Phi-3-medium-4k-instruct-bnb-4bit" : ( + "unsloth/Phi-3-medium-4k-instruct-bnb-4bit": ( "unsloth/Phi-3-medium-4k-instruct", "microsoft/Phi-3-medium-4k-instruct", ), - "unsloth/Qwen2-0.5B-bnb-4bit" : ( + "unsloth/Qwen2-0.5B-bnb-4bit": ( "unsloth/Qwen2-0.5B", "Qwen/Qwen2-0.5B", ), - "unsloth/Qwen2-0.5B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit": ( "unsloth/Qwen2-0.5B-Instruct", "Qwen/Qwen2-0.5B-Instruct", ), - "unsloth/Qwen2-1.5B-bnb-4bit" : ( + "unsloth/Qwen2-1.5B-bnb-4bit": ( "unsloth/Qwen2-1.5B", "Qwen/Qwen2-1.5B", ), - "unsloth/Qwen2-1.5B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit": ( "unsloth/Qwen2-1.5B-Instruct", "Qwen/Qwen2-1.5B-Instruct", ), - "unsloth/Qwen2-7B-bnb-4bit" : ( + "unsloth/Qwen2-7B-bnb-4bit": ( "unsloth/Qwen2-7B", "Qwen/Qwen2-7B", ), - "unsloth/Qwen2-7B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2-7B-Instruct-bnb-4bit": ( "unsloth/Qwen2-7B-Instruct", "Qwen/Qwen2-7B-Instruct", ), - "unsloth/Qwen2-70B-bnb-4bit" : ( - "Qwen/Qwen2-70B", - ), - "unsloth/Qwen2-70B-Instruct-bnb-4bit" : ( - "Qwen/Qwen2-70B-Instruct", - ), - "mistralai/Codestral-22B-v0.1" : ( - "mistral-community/Codestral-22B-v0.1", - ), - "unsloth/gemma-2-9b-bnb-4bit" : ( + "unsloth/Qwen2-70B-bnb-4bit": ("Qwen/Qwen2-70B",), + "unsloth/Qwen2-70B-Instruct-bnb-4bit": ("Qwen/Qwen2-70B-Instruct",), + "mistralai/Codestral-22B-v0.1": ("mistral-community/Codestral-22B-v0.1",), + "unsloth/gemma-2-9b-bnb-4bit": ( "unsloth/gemma-2-9b", "google/gemma-2-9b", ), - "unsloth/gemma-2-27b-bnb-4bit" : ( + "unsloth/gemma-2-27b-bnb-4bit": ( "unsloth/gemma-2-27b", "google/gemma-2-27b", ), - "unsloth/gemma-2-9b-it-bnb-4bit" : ( + "unsloth/gemma-2-9b-it-bnb-4bit": ( "unsloth/gemma-2-9b-it", "google/gemma-2-9b-it", ), - "unsloth/gemma-2-27b-it-bnb-4bit" : ( + "unsloth/gemma-2-27b-it-bnb-4bit": ( "unsloth/gemma-2-27b-it", "google/gemma-2-27b-it", ), - "unsloth/Phi-3-mini-4k-instruct-v0-bnb-4bit" : ( # Old Phi pre July + "unsloth/Phi-3-mini-4k-instruct-v0-bnb-4bit": ( # Old Phi pre July "unsloth/Phi-3-mini-4k-instruct-v0", ), - "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit" : ( # New 12b Mistral models + "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit": ( # New 12b Mistral models "unsloth/Mistral-Nemo-Instruct-2407", "mistralai/Mistral-Nemo-Instruct-2407", ), - "unsloth/Mistral-Nemo-Base-2407-bnb-4bit" : ( # New 12b Mistral models + "unsloth/Mistral-Nemo-Base-2407-bnb-4bit": ( # New 12b Mistral models "unsloth/Mistral-Nemo-Base-2407", "mistralai/Mistral-Nemo-Base-2407", ), - "unsloth/Meta-Llama-3.1-8B-unsloth-bnb-4bit" : ( + "unsloth/Meta-Llama-3.1-8B-unsloth-bnb-4bit": ( "unsloth/Meta-Llama-3.1-8B", "meta-llama/Meta-Llama-3.1-8B", "unsloth/Meta-Llama-3.1-8B-bnb-4bit", ), - "unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit": { + "8": ( "RedHatAI/Llama-3.1-8B-Instruct-FP8", "unsloth/Llama-3.1-8B-Instruct-FP8-Block", "unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic", ), - "16" : ( + "16": ( "unsloth/Meta-Llama-3.1-8B-Instruct", "meta-llama/Meta-Llama-3.1-8B-Instruct", "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit", ), }, - "unsloth/Llama-3.1-8B-unsloth-bnb-4bit" : ( + "unsloth/Llama-3.1-8B-unsloth-bnb-4bit": ( "unsloth/Llama-3.1-8B", "meta-llama/Llama-3.1-8B", "unsloth/Llama-3.1-8B-bnb-4bit", ), - "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit": { + "8": ( "RedHatAI/Llama-3.1-8B-Instruct-FP8", "unsloth/Llama-3.1-8B-Instruct-FP8-Block", "unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic", ), - "16" : ( + "16": ( "unsloth/Llama-3.1-8B-Instruct", "meta-llama/Llama-3.1-8B-Instruct", "unsloth/Llama-3.1-8B-Instruct-bnb-4bit", ), }, - "unsloth/Meta-Llama-3.1-70B-bnb-4bit" : ( + "unsloth/Meta-Llama-3.1-70B-bnb-4bit": ( "unsloth/Meta-Llama-3.1-70B", "meta-llama/Meta-Llama-3.1-70B", ), - "unsloth/Meta-Llama-3.1-405B-bnb-4bit" : ( - "meta-llama/Meta-Llama-3.1-405B", - ), - "unsloth/Meta-Llama-3.1-405B-Instruct-bnb-4bit" : ( + "unsloth/Meta-Llama-3.1-405B-bnb-4bit": ("meta-llama/Meta-Llama-3.1-405B",), + "unsloth/Meta-Llama-3.1-405B-Instruct-bnb-4bit": ( "meta-llama/Meta-Llama-3.1-405B-Instruct", ), - "unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit" : ( + "unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit": ( "unsloth/Meta-Llama-3.1-70B-Instruct", "meta-llama/Meta-Llama-3.1-70B-Instruct", ), - "unsloth/Mistral-Large-Instruct-2407-bnb-4bit" : ( + "unsloth/Mistral-Large-Instruct-2407-bnb-4bit": ( "mistralai/Mistral-Large-Instruct-2407", ), - "unsloth/gemma-2-2b-bnb-4bit" : ( + "unsloth/gemma-2-2b-bnb-4bit": ( "unsloth/gemma-2-2b", "google/gemma-2-2b", ), - "unsloth/gemma-2-2b-it-bnb-4bit" : ( + "unsloth/gemma-2-2b-it-bnb-4bit": ( "unsloth/gemma-2-2b-it", "google/gemma-2-2b-it", ), - "unsloth/Phi-3.5-mini-instruct-bnb-4bit" : ( + "unsloth/Phi-3.5-mini-instruct-bnb-4bit": ( "unsloth/Phi-3.5-mini-instruct", "microsoft/Phi-3.5-mini-instruct", ), - "unsloth/c4ai-command-r-08-2024-bnb-4bit" : ( - "CohereForAI/c4ai-command-r-08-2024", - ), - "unsloth/c4ai-command-r-plus-08-2024-bnb-4bit" : ( + "unsloth/c4ai-command-r-08-2024-bnb-4bit": ("CohereForAI/c4ai-command-r-08-2024",), + "unsloth/c4ai-command-r-plus-08-2024-bnb-4bit": ( "CohereForAI/c4ai-command-r-plus-08-2024", ), - "unsloth/Llama-3.1-Storm-8B-bnb-4bit" : ( + "unsloth/Llama-3.1-Storm-8B-bnb-4bit": ( "unsloth/Llama-3.1-Storm-8B", "akjindal53244/Llama-3.1-Storm-8B", ), - "unsloth/Hermes-3-Llama-3.1-8B-bnb-4bit" : ( + "unsloth/Hermes-3-Llama-3.1-8B-bnb-4bit": ( "unsloth/Hermes-3-Llama-3.1-8B", "NousResearch/Hermes-3-Llama-3.1-8B", ), - "unsloth/Hermes-3-Llama-3.1-70B-bnb-4bit" : ( + "unsloth/Hermes-3-Llama-3.1-70B-bnb-4bit": ( "unsloth/Hermes-3-Llama-3.1-70B", "NousResearch/Hermes-3-Llama-3.1-70B", ), - "unsloth/Hermes-3-Llama-3.1-405B-bnb-4bit" : ( + "unsloth/Hermes-3-Llama-3.1-405B-bnb-4bit": ( "NousResearch/Hermes-3-Llama-3.1-405B", ), - "unsloth/SmolLM-135M-bnb-4bit" : ( + "unsloth/SmolLM-135M-bnb-4bit": ( "unsloth/SmolLM-135M", "HuggingFaceTB/SmolLM-135M", ), - "unsloth/SmolLM-360M-bnb-4bit" : ( + "unsloth/SmolLM-360M-bnb-4bit": ( "unsloth/SmolLM-360M", "HuggingFaceTB/SmolLM-360M", ), - "unsloth/SmolLM-1.7B-bnb-4bit" : ( + "unsloth/SmolLM-1.7B-bnb-4bit": ( "unsloth/SmolLM-1.7B", "HuggingFaceTB/SmolLM-1.7B", ), - "unsloth/SmolLM-135M-Instruct-bnb-4bit" : ( + "unsloth/SmolLM-135M-Instruct-bnb-4bit": ( "unsloth/SmolLM-135M-Instruct", "HuggingFaceTB/SmolLM-135M-Instruct", ), - "unsloth/SmolLM-360M-Instruct-bnb-4bit" : ( + "unsloth/SmolLM-360M-Instruct-bnb-4bit": ( "unsloth/SmolLM-360M-Instruct", "HuggingFaceTB/SmolLM-360M-Instruct", ), - "unsloth/SmolLM-1.7B-Instruct-bnb-4bit" : ( + "unsloth/SmolLM-1.7B-Instruct-bnb-4bit": ( "unsloth/SmolLM-1.7B-Instruct", "HuggingFaceTB/SmolLM-1.7B-Instruct", ), - "unsloth/Mistral-Small-Instruct-2409-bnb-4bit" : ( + "unsloth/Mistral-Small-Instruct-2409-bnb-4bit": ( "unsloth/Mistral-Small-Instruct-2409", "mistralai/Mistral-Small-Instruct-2409", ), - "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-0.5B-Instruct", "Qwen/Qwen2.5-0.5B-Instruct", "unsloth/Qwen2.5-0.5B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-1.5B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-1.5B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-1.5B-Instruct", "Qwen/Qwen2.5-1.5B-Instruct", "unsloth/Qwen2.5-1.5B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-3B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-3B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-3B-Instruct", "Qwen/Qwen2.5-3B-Instruct", "unsloth/Qwen2.5-3B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-7B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-7B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-7B-Instruct", "Qwen/Qwen2.5-7B-Instruct", "unsloth/Qwen2.5-7B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-14B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-14B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-14B-Instruct", "Qwen/Qwen2.5-14B-Instruct", "unsloth/Qwen2.5-14B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-32B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-32B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-32B-Instruct", "Qwen/Qwen2.5-32B-Instruct", ), - "unsloth/Qwen2.5-72B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-72B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-72B-Instruct", "Qwen/Qwen2.5-72B-Instruct", ), - "unsloth/Qwen2.5-0.5B-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-0.5B-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-0.5B", "Qwen/Qwen2.5-0.5B", "unsloth/Qwen2.5-0.5B-bnb-4bit", ), - "unsloth/Qwen2.5-1.5B-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-1.5B-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-1.5B", "Qwen/Qwen2.5-1.5B", "unsloth/Qwen2.5-1.5B-bnb-4bit", ), - "unsloth/Qwen2.5-3B-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-3B-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-3B", "Qwen/Qwen2.5-3B", "unsloth/Qwen2.5-3B-bnb-4bit", ), - "unsloth/Qwen2.5-7B-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-7B-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-7B", "Qwen/Qwen2.5-7B", "unsloth/Qwen2.5-7B-bnb-4bit", ), - "unsloth/Qwen2.5-14B-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-14B-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-14B", "Qwen/Qwen2.5-14B", "unsloth/Qwen2.5-14B-bnb-4bit", ), - "unsloth/Qwen2.5-32B-bnb-4bit" : ( + "unsloth/Qwen2.5-32B-bnb-4bit": ( "unsloth/Qwen2.5-32B", "Qwen/Qwen2.5-32B", ), - "unsloth/Qwen2.5-72B-bnb-4bit" : ( + "unsloth/Qwen2.5-72B-bnb-4bit": ( "unsloth/Qwen2.5-72B", "Qwen/Qwen2.5-72B", ), - "unsloth/Qwen2.5-Math-1.5B-bnb-4bit" : ( + "unsloth/Qwen2.5-Math-1.5B-bnb-4bit": ( "unsloth/Qwen2.5-Math-1.5B", "Qwen/Qwen2.5-Math-1.5B", ), - "unsloth/Qwen2.5-Math-7B-bnb-4bit" : ( + "unsloth/Qwen2.5-Math-7B-bnb-4bit": ( "unsloth/Qwen2.5-Math-7B", "Qwen/Qwen2.5-Math-7B", ), - "unsloth/Qwen2.5-Math-72B-bnb-4bit" : ( + "unsloth/Qwen2.5-Math-72B-bnb-4bit": ( "unsloth/Qwen2.5-Math-72B", "Qwen/Qwen2.5-Math-72B", ), - "unsloth/Qwen2.5-Math-1.5B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Math-1.5B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Math-1.5B-Instruct", "Qwen/Qwen2.5-Math-1.5B-Instruct", ), - "unsloth/Qwen2.5-Math-7B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Math-7B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Math-7B-Instruct", "Qwen/Qwen2.5-Math-7B-Instruct", ), - "unsloth/Qwen2.5-Math-72B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Math-72B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Math-72B-Instruct", "Qwen/Qwen2.5-Math-72B-Instruct", ), - "unsloth/Qwen2.5-Coder-0.5B-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-0.5B-bnb-4bit": ( "unsloth/Qwen2.5-Coder-0.5B", "Qwen/Qwen2.5-Coder-0.5B", ), - "unsloth/Qwen2.5-Coder-1.5B-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-1.5B-bnb-4bit": ( "unsloth/Qwen2.5-Coder-1.5B", "Qwen/Qwen2.5-Coder-1.5B", ), - "unsloth/Qwen2.5-Coder-3B-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-3B-bnb-4bit": ( "unsloth/Qwen2.5-Coder-3B", "Qwen/Qwen2.5-Coder-3B", ), - "unsloth/Qwen2.5-Coder-7B-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-7B-bnb-4bit": ( "unsloth/Qwen2.5-Coder-7B", "Qwen/Qwen2.5-Coder-7B", ), - "unsloth/Qwen2.5-Coder-14B-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-14B-bnb-4bit": ( "unsloth/Qwen2.5-Coder-14B", "Qwen/Qwen2.5-Coder-14B", ), - "unsloth/Qwen2.5-Coder-32B-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-32B-bnb-4bit": ( "unsloth/Qwen2.5-Coder-32B", "Qwen/Qwen2.5-Coder-32B", ), - "unsloth/Qwen2.5-Coder-0.5B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-0.5B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Coder-0.5B-Instruct", "Qwen/Qwen2.5-Coder-0.5B-Instruct", ), - "unsloth/Qwen2.5-Coder-1.5B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-1.5B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Coder-1.5B-Instruct", "Qwen/Qwen2.5-Coder-1.5B-Instruct", ), - "unsloth/Qwen2.5-Coder-3B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-3B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Coder-3B-Instruct", "Qwen/Qwen2.5-Coder-3B-Instruct", ), - "unsloth/Qwen2.5-Coder-7B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-7B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Coder-7B-Instruct", "Qwen/Qwen2.5-Coder-7B-Instruct", ), - "unsloth/Qwen2.5-Coder-14B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-14B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Coder-14B-Instruct", "Qwen/Qwen2.5-Coder-14B-Instruct", ), - "unsloth/Qwen2.5-Coder-32B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2.5-Coder-32B-Instruct-bnb-4bit": ( "unsloth/Qwen2.5-Coder-32B-Instruct", "Qwen/Qwen2.5-Coder-32B-Instruct", ), - "unsloth/Llama-3.2-1B-unsloth-bnb-4bit" : ( + "unsloth/Llama-3.2-1B-unsloth-bnb-4bit": ( "unsloth/Llama-3.2-1B", "meta-llama/Llama-3.2-1B", "unsloth/Llama-3.2-1B-bnb-4bit", ), - "unsloth/Llama-3.2-3B-unsloth-bnb-4bit" : ( + "unsloth/Llama-3.2-3B-unsloth-bnb-4bit": ( "unsloth/Llama-3.2-3B", "meta-llama/Llama-3.2-3B", "unsloth/Llama-3.2-3B-bnb-4bit", ), - "unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit" : { + "unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit": { "8": ( "RedHatAI/Llama-3.2-1B-Instruct-FP8", "unsloth/Llama-3.2-1B-Instruct-FP8-Block", "unsloth/Llama-3.2-1B-Instruct-FP8-Dynamic", ), - "16" : ( + "16": ( "unsloth/Llama-3.2-1B-Instruct", "meta-llama/Llama-3.2-1B-Instruct", "unsloth/Llama-3.2-1B-Instruct-bnb-4bit", ), }, - "unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit" : { + "unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit": { "8": ( "RedHatAI/Llama-3.2-3B-Instruct-FP8", "unsloth/Llama-3.2-3B-Instruct-FP8-Block", "unsloth/Llama-3.2-3B-Instruct-FP8-Dynamic", ), - "16" : ( + "16": ( "unsloth/Llama-3.2-3B-Instruct", "meta-llama/Llama-3.2-3B-Instruct", "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", ), }, - "unsloth/Llama-3.1-Nemotron-70B-Instruct-bnb-4bit" : ( + "unsloth/Llama-3.1-Nemotron-70B-Instruct-bnb-4bit": ( "unsloth/Llama-3.1-Nemotron-70B-Instruct", "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", ), - "unsloth/Qwen2-VL-2B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2-VL-2B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2-VL-2B-Instruct", "Qwen/Qwen2-VL-2B-Instruct", "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit", ), - "unsloth/Qwen2-VL-7B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2-VL-7B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2-VL-7B-Instruct", "Qwen/Qwen2-VL-7B-Instruct", "unsloth/Qwen2-VL-7B-Instruct-bnb-4bit", ), - "unsloth/Qwen2-VL-72B-Instruct-bnb-4bit" : ( + "unsloth/Qwen2-VL-72B-Instruct-bnb-4bit": ( "unsloth/Qwen2-VL-72B-Instruct", "Qwen/Qwen2-VL-72B-Instruct", ), - "unsloth/Qwen2-VL-2B-bnb-4bit" : ( + "unsloth/Qwen2-VL-2B-bnb-4bit": ( "unsloth/Qwen2-VL-2B", "Qwen/Qwen2-VL-2B", ), - "unsloth/Qwen2-VL-7B-bnb-4bit" : ( + "unsloth/Qwen2-VL-7B-bnb-4bit": ( "unsloth/Qwen2-VL-7B", "Qwen/Qwen2-VL-7B", ), - "unsloth/Qwen2-VL-72B-bnb-4bit" : ( + "unsloth/Qwen2-VL-72B-bnb-4bit": ( "unsloth/Qwen2-VL-72B", "Qwen/Qwen2-VL-72B", ), - "unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit": ( "unsloth/Llama-3.2-11B-Vision-Instruct", "meta-llama/Llama-3.2-11B-Vision-Instruct", "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit", ), - "unsloth/Llama-3.2-90B-Vision-Instruct-bnb-4bit" : ( + "unsloth/Llama-3.2-90B-Vision-Instruct-bnb-4bit": ( "unsloth/Llama-3.2-90B-Vision-Instruct", "meta-llama/Llama-3.2-90B-Vision-Instruct", ), - "unsloth/Llama-3.2-11B-Vision-unsloth-bnb-4bit" : ( + "unsloth/Llama-3.2-11B-Vision-unsloth-bnb-4bit": ( "unsloth/Llama-3.2-11B-Vision", "meta-llama/Llama-3.2-11B-Vision", "unsloth/Llama-3.2-11B-Vision-bnb-4bit", ), - "unsloth/Llama-3.2-90B-Vision-bnb-4bit" : ( + "unsloth/Llama-3.2-90B-Vision-bnb-4bit": ( "unsloth/Llama-3.2-90B-Vision", "meta-llama/Llama-3.2-90B-Vision", ), - "unsloth/Pixtral-12B-2409-unsloth-bnb-4bit" : ( + "unsloth/Pixtral-12B-2409-unsloth-bnb-4bit": ( "unsloth/Pixtral-12B-2409", "mistralai/Pixtral-12B-2409", "unsloth/Pixtral-12B-2409-bnb-4bit", ), - "unsloth/Pixtral-12B-2409-Base-bnb-4bit" : ( + "unsloth/Pixtral-12B-2409-Base-bnb-4bit": ( "unsloth/Pixtral-12B-Base-2409", "mistralai/Pixtral-12B-Base-2409", ), - "unsloth/llava-1.5-7b-hf-bnb-4bit" : ( + "unsloth/llava-1.5-7b-hf-bnb-4bit": ( "unsloth/llava-1.5-7b-hf", "llava-hf/llava-1.5-7b-hf", ), - "unsloth/llava-v1.6-mistral-7b-hf-bnb-4bit" : ( + "unsloth/llava-v1.6-mistral-7b-hf-bnb-4bit": ( "unsloth/llava-v1.6-mistral-7b-hf", "llava-hf/llava-v1.6-mistral-7b-hf", ), - "unsloth/Llama-3.1-Tulu-3-8B-bnb-4bit" : ( + "unsloth/Llama-3.1-Tulu-3-8B-bnb-4bit": ( "unsloth/Llama-3.1-Tulu-3-8B", "allenai/Llama-3.1-Tulu-3-8B", ), - "unsloth/Llama-3.1-Tulu-3-70B-bnb-4bit" : ( + "unsloth/Llama-3.1-Tulu-3-70B-bnb-4bit": ( "unsloth/Llama-3.1-Tulu-3-70B", "allenai/Llama-3.1-Tulu-3-70B", ), - "unsloth/QwQ-32B-Preview-bnb-4bit" : ( + "unsloth/QwQ-32B-Preview-bnb-4bit": ( "unsloth/QwQ-32B-Preview", "Qwen/QwQ-32B-Preview", ), - "unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit": { + "8": ( "RedHatAI/Llama-3.3-70B-Instruct-FP8", "unsloth/Llama-3.3-70B-Instruct-FP8-Block", "unsloth/Llama-3.3-70B-Instruct-FP8-Dynamic", ), - "16" : ( + "16": ( "unsloth/Llama-3.3-70B-Instruct", "meta-llama/Llama-3.3-70B-Instruct", "unsloth/Llama-3.3-70B-Instruct-bnb-4bit", ), }, - "unsloth/phi-4-unsloth-bnb-4bit" : ( + "unsloth/phi-4-unsloth-bnb-4bit": ( "unsloth/phi-4", "microsoft/phi-4", "unsloth/phi-4-bnb-4bit", ), - "unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit" : ( + "unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit": ( "unsloth/DeepSeek-R1-Distill-Qwen-32B", "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", ), - "unsloth/DeepSeek-R1-Distill-Qwen-14B-unsloth-bnb-4bit" : ( + "unsloth/DeepSeek-R1-Distill-Qwen-14B-unsloth-bnb-4bit": ( "unsloth/DeepSeek-R1-Distill-Qwen-14B", "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", "unsloth/DeepSeek-R1-Distill-Qwen-14B-bnb-4bit", ), - "unsloth/DeepSeek-R1-Distill-Qwen-7B-unsloth-bnb-4bit" : ( + "unsloth/DeepSeek-R1-Distill-Qwen-7B-unsloth-bnb-4bit": ( "unsloth/DeepSeek-R1-Distill-Qwen-7B", "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "unsloth/DeepSeek-R1-Distill-Qwen-7B-bnb-4bit", ), - "unsloth/DeepSeek-R1-Distill-Qwen-1.5B-unsloth-bnb-4bit" : ( + "unsloth/DeepSeek-R1-Distill-Qwen-1.5B-unsloth-bnb-4bit": ( "unsloth/DeepSeek-R1-Distill-Qwen-1.5B", "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", "unsloth/DeepSeek-R1-Distill-Qwen-1.5B-bnb-4bit", ), - "unsloth/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit" : ( + "unsloth/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit": ( "unsloth/DeepSeek-R1-Distill-Llama-8B", "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", "unsloth/DeepSeek-R1-Distill-Llama-8B-bnb-4bit", ), - "unsloth/DeepSeek-R1-Distill-Llama-70B-bnb-4bit" : ( + "unsloth/DeepSeek-R1-Distill-Llama-70B-bnb-4bit": ( "unsloth/DeepSeek-R1-Distill-Llama-70B", "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", ), - "unsloth/Mistral-Small-24B-Base-2501-unsloth-bnb-4bit" : ( + "unsloth/Mistral-Small-24B-Base-2501-unsloth-bnb-4bit": ( "unsloth/Mistral-Small-24B-Base-2501", "mistralai/Mistral-Small-24B-Base-2501", "unsloth/Mistral-Small-24B-Base-2501-bnb-4bit", ), - "unsloth/Mistral-Small-24B-Instruct-2501-unsloth-bnb-4bit" : ( + "unsloth/Mistral-Small-24B-Instruct-2501-unsloth-bnb-4bit": ( "unsloth/Mistral-Small-24B-Instruct-2501", "mistralai/Mistral-Small-24B-Instruct-2501", "unsloth/Mistral-Small-24B-Instruct-2501-bnb-4bit", ), - "unsloth/Qwen2.5-VL-3B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-VL-3B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-VL-3B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct", "unsloth/Qwen2.5-VL-3B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-VL-7B-Instruct", "Qwen/Qwen2.5-VL-7B-Instruct", "unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-VL-32B-Instruct", "Qwen/Qwen2.5-VL-32B-Instruct", "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-VL-72B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-VL-72B-Instruct-unsloth-bnb-4bit": ( "unsloth/Qwen2.5-VL-72B-Instruct", "Qwen/Qwen2.5-VL-72B-Instruct", "unsloth/Qwen2.5-VL-72B-Instruct-bnb-4bit", ), - "unsloth/DeepScaleR-1.5B-Preview-unsloth-bnb-4bit" : ( + "unsloth/DeepScaleR-1.5B-Preview-unsloth-bnb-4bit": ( "unsloth/DeepHermes-3-Llama-3-8B-Preview", "agentica-org/DeepScaleR-1.5B-Preview", "unsloth/DeepScaleR-1.5B-Preview-bnb-4bit", ), - "unsloth/OpenThinker-7B-unsloth-bnb-4bit" : ( + "unsloth/OpenThinker-7B-unsloth-bnb-4bit": ( "unsloth/OpenThinker-7B", "open-thoughts/OpenThinker-7B", "unsloth/OpenThinker-7B-bnb-4bit", ), - "unsloth/granite-3.2-2b-instruct-unsloth-bnb-4bit" : ( + "unsloth/granite-3.2-2b-instruct-unsloth-bnb-4bit": ( "unsloth/granite-3.2-2b-instruct", "ibm-granite/granite-3.2-2b-instruct", "unsloth/granite-3.2-2b-instruct-bnb-4bit", ), - "unsloth/granite-3.2-8b-instruct-unsloth-bnb-4bit" : ( + "unsloth/granite-3.2-8b-instruct-unsloth-bnb-4bit": ( "unsloth/granite-3.2-8b-instruct", "ibm-granite/granite-3.2-8b-instruct", "unsloth/granite-3.2-8b-instruct-bnb-4bit", ), - "unsloth/QwQ-32B-unsloth-bnb-4bit" : ( + "unsloth/QwQ-32B-unsloth-bnb-4bit": ( "unsloth/QwQ-32B", "Qwen/QwQ-32B", "unsloth/QwQ-32B-bnb-4bit", ), - "unsloth/gemma-3-1b-it-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-1b-it-unsloth-bnb-4bit": ( "unsloth/gemma-3-1b-it", "google/gemma-3-1b-it", "unsloth/gemma-3-1b-it-bnb-4bit", ), - "unsloth/gemma-3-4b-it-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-4b-it-unsloth-bnb-4bit": ( "unsloth/gemma-3-4b-it", "google/gemma-3-4b-it", "unsloth/gemma-3-4b-it-bnb-4bit", ), - "unsloth/gemma-3-12b-it-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-12b-it-unsloth-bnb-4bit": ( "unsloth/gemma-3-12b-it", "google/gemma-3-12b-it", "unsloth/gemma-3-12b-it-bnb-4bit", ), - "unsloth/gemma-3-27b-it-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-27b-it-unsloth-bnb-4bit": ( "unsloth/gemma-3-27b-it", "google/gemma-3-27b-it", "unsloth/gemma-3-27b-it-bnb-4bit", ), - "unsloth/gemma-3-1b-pt-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-1b-pt-unsloth-bnb-4bit": ( "unsloth/gemma-3-1b-pt", "google/gemma-3-1b-pt", "unsloth/gemma-3-1b-pt-bnb-4bit", ), - "unsloth/gemma-3-4b-pt-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-4b-pt-unsloth-bnb-4bit": ( "unsloth/gemma-3-4b-pt", "google/gemma-3-4b-pt", "unsloth/gemma-3-4b-pt-bnb-4bit", ), - "unsloth/gemma-3-12b-pt-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-12b-pt-unsloth-bnb-4bit": ( "unsloth/gemma-3-12b-pt", "google/gemma-3-12b-pt", "unsloth/gemma-3-12b-pt-bnb-4bit", ), - "unsloth/gemma-3-27b-pt-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-27b-pt-unsloth-bnb-4bit": ( "unsloth/gemma-3-27b-pt", "google/gemma-3-27b-pt", "unsloth/gemma-3-27b-pt-bnb-4bit", ), - "unsloth/reka-flash-3-unsloth-bnb-4bit" : ( + "unsloth/reka-flash-3-unsloth-bnb-4bit": ( "unsloth/reka-flash-3", "RekaAI/reka-flash-3", "unsloth/reka-flash-3-bnb-4bit", ), - "unsloth/c4ai-command-a-03-2025-unsloth-bnb-4bit" : ( + "unsloth/c4ai-command-a-03-2025-unsloth-bnb-4bit": ( "unsloth/c4ai-command-a-03-2025", "CohereForAI/c4ai-command-a-03-2025", "unsloth/c4ai-command-a-03-2025-bnb-4bit", ), - "unsloth/aya-vision-32b-unsloth-bnb-4bit" : ( + "unsloth/aya-vision-32b-unsloth-bnb-4bit": ( "unsloth/aya-vision-32b", "CohereForAI/aya-vision-32b", "unsloth/aya-vision-32b-bnb-4bit", ), - "unsloth/aya-vision-8b-unsloth-bnb-4bit" : ( + "unsloth/aya-vision-8b-unsloth-bnb-4bit": ( "unsloth/aya-vision-8b", "CohereForAI/aya-vision-8b", "unsloth/aya-vision-8b-bnb-4bit", ), - "unsloth/granite-vision-3.2-2b-unsloth-bnb-4bit" : ( + "unsloth/granite-vision-3.2-2b-unsloth-bnb-4bit": ( "unsloth/granite-vision-3.2-2b", "ibm-granite/granite-vision-3.2-2b", "unsloth/granite-vision-3.2-2b-bnb-4bit", ), - "unsloth/OLMo-2-0325-32B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/OLMo-2-0325-32B-Instruct-unsloth-bnb-4bit": ( "unsloth/OLMo-2-0325-32B-Instruct", "allenai/OLMo-2-0325-32B-Instruct", "unsloth/OLMo-2-0325-32B-Instruct-bnb-4bit", ), - "unsloth/Mistral-Small-3.1-24B-Instruct-2503-unsloth-bnb-4bit" : ( + "unsloth/Mistral-Small-3.1-24B-Instruct-2503-unsloth-bnb-4bit": ( "unsloth/Mistral-Small-3.1-24B-Instruct-2503", "mistralai/Mistral-Small-3.1-24B-Instruct-2503", "unsloth/Mistral-Small-3.1-24B-Instruct-2503-bnb-4bit", ), - "unsloth/Mistral-Small-3.1-24B-Base-2503-unsloth-bnb-4bit" : ( + "unsloth/Mistral-Small-3.1-24B-Base-2503-unsloth-bnb-4bit": ( "unsloth/Mistral-Small-3.1-24B-Base-2503", "mistralai/Mistral-Small-3.1-24B-Base-2503", "unsloth/Mistral-Small-3.1-24B-Base-2503-bnb-4bit", ), - "unsloth/Qwen3-0.6B-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-0.6B-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-0.6B-FP8", "unsloth/Qwen3-0.6B-FP8", "unsloth/Qwen3-0.6B-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-0.6B", "Qwen/Qwen3-0.6B", "unsloth/Qwen3-0.6B-bnb-4bit", ), }, - "unsloth/Qwen3-1.7B-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-1.7B-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-1.7B-FP8", "unsloth/Qwen3-1.7B-FP8", "unsloth/Qwen3-1.7B-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-1.7B", "Qwen/Qwen3-1.7B", "unsloth/Qwen3-1.7B-bnb-4bit", ), }, - "unsloth/Qwen3-4B-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-4B-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-4B-FP8", "unsloth/Qwen3-4B-FP8", "unsloth/Qwen3-4B-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-4B", "Qwen/Qwen3-4B", "unsloth/Qwen3-4B-bnb-4bit", ), }, - "unsloth/Qwen3-8B-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-8B-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-8B-FP8", "unsloth/Qwen3-8B-FP8", "unsloth/Qwen3-8B-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-8B", "Qwen/Qwen3-8B", "unsloth/Qwen3-8B-bnb-4bit", ), }, - "unsloth/Qwen3-14B-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-14B-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-14B-FP8", "unsloth/Qwen3-14B-FP8", "unsloth/Qwen3-14B-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-14B", "Qwen/Qwen3-14B", "unsloth/Qwen3-14B-bnb-4bit", ), }, - "unsloth/Qwen3-32B-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-32B-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-32B-FP8", "unsloth/Qwen3-32B-FP8", "unsloth/Qwen3-32B-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-32B", "Qwen/Qwen3-32B", "unsloth/Qwen3-32B-bnb-4bit", ), }, - "unsloth/Qwen3-30B-A3B-unsloth-bnb-4bit" : ( + "unsloth/Qwen3-30B-A3B-unsloth-bnb-4bit": ( "unsloth/Qwen3-30B-A3B", "Qwen/Qwen3-30B-A3B", "unsloth/Qwen3-30B-A3B-bnb-4bit", ), - "unsloth/Qwen3-0.6B-Base-unsloth-bnb-4bit" : ( + "unsloth/Qwen3-0.6B-Base-unsloth-bnb-4bit": ( "unsloth/Qwen3-0.6B-Base", "Qwen/Qwen3-0.6B-Base", "unsloth/Qwen3-0.6B-Base-bnb-4bit", ), - "unsloth/Qwen3-1.7B-Base-unsloth-bnb-4bit" : ( + "unsloth/Qwen3-1.7B-Base-unsloth-bnb-4bit": ( "unsloth/Qwen3-1.7B-Base", "Qwen/Qwen3-1.7B-Base", "unsloth/Qwen3-1.7B-Base-bnb-4bit", ), - "unsloth/Qwen3-4B-Base-unsloth-bnb-4bit" : ( + "unsloth/Qwen3-4B-Base-unsloth-bnb-4bit": ( "unsloth/Qwen3-4B-Base", "Qwen/Qwen3-4B-Base", "unsloth/Qwen3-4B-Base-bnb-4bit", ), - "unsloth/Qwen3-8B-Base-unsloth-bnb-4bit" : ( + "unsloth/Qwen3-8B-Base-unsloth-bnb-4bit": ( "unsloth/Qwen3-8B-Base", "Qwen/Qwen3-8B-Base", "unsloth/Qwen3-8B-Base-bnb-4bit", ), - "unsloth/Qwen3-14B-Base-unsloth-bnb-4bit" : ( + "unsloth/Qwen3-14B-Base-unsloth-bnb-4bit": ( "unsloth/Qwen3-14B-Base", "Qwen/Qwen3-14B-Base", "unsloth/Qwen3-14B-Base-bnb-4bit", ), - "unsloth/Qwen3-30B-A3B-Base-bnb-4bit" : ( + "unsloth/Qwen3-30B-A3B-Base-bnb-4bit": ( "unsloth/Qwen3-30B-A3B-Base", "Qwen/Qwen3-30B-A3B-Base", ), - "unsloth/phi-4-reasoning-unsloth-bnb-4bit" : ( + "unsloth/phi-4-reasoning-unsloth-bnb-4bit": ( "unsloth/phi-4-reasoning", "microsoft/Phi-4-reasoning", "unsloth/phi-4-reasoning-bnb-4bit", ), - "unsloth/phi-4-reasoning-plus-unsloth-bnb-4bit" : ( + "unsloth/phi-4-reasoning-plus-unsloth-bnb-4bit": ( "unsloth/phi-4-reasoning-plus", "microsoft/Phi-4-reasoning-plus", "unsloth/phi-4-reasoning-plus-bnb-4bit", ), - "unsloth/phi-4-mini-reasoning-unsloth-bnb-4bit" : ( + "unsloth/phi-4-mini-reasoning-unsloth-bnb-4bit": ( "unsloth/phi-4-mini-reasoning", "microsoft/Phi-4-mini-reasoning", "unsloth/phi-4-mini-reasoning-bnb-4bit", ), - "unsloth/Phi-4-mini-instruct-unsloth-bnb-4bit" : ( + "unsloth/Phi-4-mini-instruct-unsloth-bnb-4bit": ( "unsloth/Phi-4-mini-instruct", "microsoft/Phi-4-mini-instruct", "unsloth/Phi-4-mini-instruct-bnb-4bit", ), - "unsloth/orpheus-3b-0.1-pretrained-unsloth-bnb-4bit" : ( + "unsloth/orpheus-3b-0.1-pretrained-unsloth-bnb-4bit": ( "unsloth/orpheus-3b-0.1-pretrained", "canopylabs/orpheus-3b-0.1-pretrained", "unsloth/orpheus-3b-0.1-pretrained-bnb-4bit", ), - "unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit" : ( + "unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit": ( "unsloth/orpheus-3b-0.1-ft", "canopylabs/orpheus-3b-0.1-ft", "unsloth/orpheus-3b-0.1-ft-bnb-4bit", ), - "unsloth/csm-1b" : ( + "unsloth/csm-1b": ( "unsloth/csm-1b", "sesame/csm-1b", ), - "unsloth/whisper-large-v3" : ( + "unsloth/whisper-large-v3": ( "unsloth/whisper-large-v3", "openai/whisper-large-v3", ), - "unsloth/whisper-large-v3-turbo" : ( + "unsloth/whisper-large-v3-turbo": ( "unsloth/whisper-large-v3-turbo", "openai/whisper-large-v3-turbo", ), - "unsloth/whisper-small" : ( + "unsloth/whisper-small": ( "unsloth/whisper-small", "openai/whisper-small", ), - "unsloth/CrisperWhisper" : ( + "unsloth/CrisperWhisper": ( "unsloth/CrisperWhisper", "nyrahealth/CrisperWhisper", ), - "unsloth/Llasa-1B" : ( + "unsloth/Llasa-1B": ( "unsloth/Llasa-1B", "HKUSTAudio/Llasa-1B", ), - "unsloth/Spark-TTS-0.5B" : ( + "unsloth/Spark-TTS-0.5B": ( "unsloth/Spark-TTS-0.5B", "SparkAudio/Spark-TTS-0.5B", ), - "unsloth/Llama-OuteTTS-1.0-1B" : ( + "unsloth/Llama-OuteTTS-1.0-1B": ( "unsloth/Llama-OuteTTS-1.0-1B", "OuteAI/Llama-OuteTTS-1.0-1B", ), - "unsloth/medgemma-4b-it-unsloth-bnb-4bit" : ( + "unsloth/medgemma-4b-it-unsloth-bnb-4bit": ( "unsloth/medgemma-4b-it", "google/medgemma-4b-it", "unsloth/medgemma-4b-it-bnb-4bit", ), - "unsloth/medgemma-27b-text-it-unsloth-bnb-4bit" : ( + "unsloth/medgemma-27b-text-it-unsloth-bnb-4bit": ( "unsloth/medgemma-27b-text-it", "google/medgemma-27b-text-it", "unsloth/medgemma-27b-text-it-bnb-4bit", ), - "unsloth/Devstral-Small-2505-unsloth-bnb-4bit" : ( + "unsloth/Devstral-Small-2505-unsloth-bnb-4bit": ( "unsloth/Devstral-Small-2505", "mistralai/Devstral-Small-2505", "unsloth/Devstral-Small-2505-bnb-4bit", ), - "unsloth/DeepSeek-R1-0528-Qwen3-8B-unsloth-bnb-4bit" : ( + "unsloth/DeepSeek-R1-0528-Qwen3-8B-unsloth-bnb-4bit": ( "unsloth/DeepSeek-R1-0528-Qwen3-8B", "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", "unsloth/DeepSeek-R1-0528-Qwen3-8B-bnb-4bit", ), - "unsloth/Magistral-Small-2506-unsloth-bnb-4bit" : ( + "unsloth/Magistral-Small-2506-unsloth-bnb-4bit": ( "unsloth/Magistral-Small-2506", "mistralai/Magistral-Small-2506", "unsloth/Magistral-Small-2506-bnb-4bit", ), - "unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : ( + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit": ( "unsloth/Mistral-Small-3.2-24B-Instruct-2506", "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit", ), - "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit" : ( + "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit": ( "unsloth/gemma-3n-E4B-it", "google/gemma-3n-E4B-it", "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit", ), - "unsloth/gemma-3n-E2B-it-unsloth-bnb-4bit" : ( + "unsloth/gemma-3n-E2B-it-unsloth-bnb-4bit": ( "unsloth/gemma-3n-E2B-it", "google/gemma-3n-E2B-it", "unsloth/gemma-3n-E2B-it-unsloth-bnb-4bit", ), - "unsloth/gemma-3n-E4B-unsloth-bnb-4bit" : ( + "unsloth/gemma-3n-E4B-unsloth-bnb-4bit": ( "unsloth/gemma-3n-E4B", "google/gemma-3n-E4B", "unsloth/gemma-3n-E4B-unsloth-bnb-4bit", ), - "unsloth/gemma-3n-E2B-unsloth-bnb-4bit" : ( + "unsloth/gemma-3n-E2B-unsloth-bnb-4bit": ( "unsloth/gemma-3n-E2B", "google/gemma-3n-E2B", "unsloth/gemma-3n-E2B-unsloth-bnb-4bit", ), - "unsloth/Devstral-Small-2507-unsloth-bnb-4bit" : ( + "unsloth/Devstral-Small-2507-unsloth-bnb-4bit": ( "unsloth/Devstral-Small-2507", "mistralai/Devstral-Small-2507", "unsloth/Devstral-Small-2507-bnb-4bit", ), - "unsloth/Qwen3-30B-A3B-Thinking-2507" : ( + "unsloth/Qwen3-30B-A3B-Thinking-2507": ( "unsloth/Qwen3-30B-A3B-Thinking-2507", "Qwen/Qwen3-30B-A3B-Thinking-2507", ), - "unsloth/Qwen3-30B-A3B-Instruct-2507" : ( + "unsloth/Qwen3-30B-A3B-Instruct-2507": ( "unsloth/Qwen3-30B-A3B-Instruct-2507", "Qwen/Qwen3-30B-A3B-Instruct-2507", ), - "unsloth/Qwen3-Coder-30B-A3B-Instruct" : ( + "unsloth/Qwen3-Coder-30B-A3B-Instruct": ( "unsloth/Qwen3-Coder-30B-A3B-Instruct", "Qwen/Qwen3-Coder-30B-A3B-Instruct", ), - "unsloth/gpt-oss-20b-unsloth-bnb-4bit" : ( + "unsloth/gpt-oss-20b-unsloth-bnb-4bit": ( "unsloth/gpt-oss-20b", "openai/gpt-oss-20b", "unsloth/gpt-oss-20b-unsloth-bnb-4bit", ), - "unsloth/gpt-oss-120b-unsloth-bnb-4bit" : ( + "unsloth/gpt-oss-120b-unsloth-bnb-4bit": ( "unsloth/gpt-oss-120b", "openai/gpt-oss-120b", "unsloth/gpt-oss-120b-unsloth-bnb-4bit", ), - "unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-4B-Instruct-2507-FP8", "unsloth/Qwen3-4B-Instruct-2507-FP8", "unsloth/Qwen3-4B-Instruct-2507-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-4B-Instruct-2507", "Qwen/Qwen3-4B-Instruct-2507", "unsloth/Qwen3-4B-Instruct-2507-bnb-4bit", ), }, - "unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-4B-Thinking-2507-FP8", "unsloth/Qwen3-4B-Thinking-2507-FP8", "unsloth/Qwen3-4B-Thinking-2507-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-4B-Thinking-2507", "Qwen/Qwen3-4B-Thinking-2507", "unsloth/Qwen3-4B-Thinking-2507-bnb-4bit", ), }, - "unsloth/gemma-3-270m-it-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-270m-it-unsloth-bnb-4bit": ( "unsloth/gemma-3-270m-it", "google/gemma-3-270m-it", "unsloth/gemma-3-270m-it-bnb-4bit", ), - "unsloth/gemma-3-270m-unsloth-bnb-4bit" : ( + "unsloth/gemma-3-270m-unsloth-bnb-4bit": ( "unsloth/gemma-3-270m", "google/gemma-3-270m", "unsloth/gemma-3-270m-bnb-4bit", ), - "unsloth/Magistral-Small-2507-unsloth-bnb-4bit" : ( + "unsloth/Magistral-Small-2507-unsloth-bnb-4bit": ( "unsloth/Magistral-Small-2507", "mistralai/Magistral-Small-2507", "unsloth/Magistral-Small-2507-bnb-4bit", ), - "unsloth/Magistral-Small-2509-unsloth-bnb-4bit" : ( + "unsloth/Magistral-Small-2509-unsloth-bnb-4bit": ( "unsloth/Magistral-Small-2509", "mistralai/Magistral-Small-2509", "unsloth/Magistral-Small-2509-bnb-4bit", ), - "unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit" : ( + "unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit": ( "unsloth/Apertus-70B-Instruct-2509", "swiss-ai/Apertus-70B-2509", "unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit", ), - "unsloth/Apertus-8B-Instruct-2509-unsloth-bnb-4bit" : ( + "unsloth/Apertus-8B-Instruct-2509-unsloth-bnb-4bit": ( "unsloth/Apertus-8B-Instruct-2509", "swiss-ai/Apertus-8B-2509", "unsloth/Apertus-8B-Instruct-2509-unsloth-bnb-4bit", ), - "unsloth/granite-4.0-micro-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-micro-unsloth-bnb-4bit": ( "unsloth/granite-4.0-micro", "ibm-granite/granite-4.0-micro", "unsloth/granite-4.0-micro-bnb-4bit", ), - "unsloth/granite-4.0-h-micro-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-h-micro-unsloth-bnb-4bit": ( "unsloth/granite-4.0-h-micro", "ibm-granite/granite-4.0-h-micro", "unsloth/granite-4.0-h-micro-bnb-4bit", ), - "unsloth/granite-4.0-micro-base-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-micro-base-unsloth-bnb-4bit": ( "unsloth/granite-4.0-micro-base", "ibm-granite/granite-4.0-micro-base", "unsloth/granite-4.0-micro-base-bnb-4bit", ), - "unsloth/granite-4.0-h-micro-base-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-h-micro-base-unsloth-bnb-4bit": ( "unsloth/granite-4.0-h-micro-base", "ibm-granite/granite-4.0-h-micro-base", "unsloth/granite-4.0-h-micro-base-bnb-4bit", ), - "unsloth/granite-4.0-h-tiny" : ( + "unsloth/granite-4.0-h-tiny": ( "unsloth/granite-4.0-h-tiny", "ibm-granite/granite-4.0-h-tiny", ), - "unsloth/granite-4.0-h-small" : ( + "unsloth/granite-4.0-h-small": ( "unsloth/granite-4.0-h-small", "ibm-granite/granite-4.0-h-small", ), - "unsloth/granite-4.0-h-tiny-base" : ( + "unsloth/granite-4.0-h-tiny-base": ( "unsloth/granite-4.0-h-tiny-base", "ibm-granite/granite-4.0-h-tiny-base", ), - "unsloth/granite-4.0-h-small-base" : ( + "unsloth/granite-4.0-h-small-base": ( "unsloth/granite-4.0-h-small-base", "ibm-granite/granite-4.0-h-small-base", ), - "unsloth/Qwen3-VL-4B-Thinking-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-4B-Thinking-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-4B-Thinking-FP8", "unsloth/Qwen3-VL-4B-Thinking-FP8", "unsloth/Qwen3-VL-4B-Thinking-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-4B-Thinking", "Qwen/Qwen3-VL-4B-Thinking", "unsloth/Qwen3-VL-4B-Thinking-bnb-4bit", ), }, - "unsloth/Qwen3-VL-8B-Thinking-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-8B-Thinking-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-8B-Thinking-FP8", "unsloth/Qwen3-VL-8B-Thinking-FP8", "unsloth/Qwen3-VL-8B-Thinking-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-8B-Thinking", "Qwen/Qwen3-VL-8B-Thinking", "unsloth/Qwen3-VL-8B-Thinking-bnb-4bit", ), }, - "unsloth/Qwen3-VL-4B-Instruct-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-4B-Instruct-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-4B-Instruct-FP8", "unsloth/Qwen3-VL-4B-Instruct-FP8", "unsloth/Qwen3-VL-4B-Instruct-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-4B-Instruct", "Qwen/Qwen3-VL-4B-Instruct", "unsloth/Qwen3-VL-4B-Instruct-bnb-4bit", ), }, - "unsloth/Qwen3-VL-8B-Instruct-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-8B-Instruct-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-8B-Instruct-FP8", "unsloth/Qwen3-VL-8B-Instruct-FP8", "unsloth/Qwen3-VL-8B-Instruct-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-8B-Instruct", "Qwen/Qwen3-VL-8B-Instruct", "unsloth/Qwen3-VL-8B-Instruct-bnb-4bit", ), }, - "unsloth/Qwen3-VL-2B-Thinking-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-2B-Thinking-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-2B-Thinking-FP8", "unsloth/Qwen3-VL-2B-Thinking-FP8", "unsloth/Qwen3-VL-2B-Thinking-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-2B-Thinking", "Qwen/Qwen3-VL-2B-Thinking", "unsloth/Qwen3-VL-2B-Thinking-bnb-4bit", ), }, - "unsloth/Qwen3-VL-32B-Thinking-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-32B-Thinking-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-32B-Thinking-FP8", "unsloth/Qwen3-VL-32B-Thinking-FP8", "unsloth/Qwen3-VL-32B-Thinking-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-32B-Thinking", "Qwen/Qwen3-VL-32B-Thinking", "unsloth/Qwen3-VL-32B-Thinking-bnb-4bit", ), }, - "unsloth/Qwen3-VL-2B-Instruct-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-2B-Instruct-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-2B-Instruct-FP8", "unsloth/Qwen3-VL-2B-Instruct-FP8", "unsloth/Qwen3-VL-2B-Instruct-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-2B-Instruct", "Qwen/Qwen3-VL-2B-Instruct", "unsloth/Qwen3-VL-2B-Instruct-bnb-4bit", ), }, - "unsloth/Qwen3-VL-32B-Instruct-unsloth-bnb-4bit" : { - "8" : ( + "unsloth/Qwen3-VL-32B-Instruct-unsloth-bnb-4bit": { + "8": ( "Qwen/Qwen3-VL-32B-Instruct-FP8", "unsloth/Qwen3-VL-32B-Instruct-FP8", "unsloth/Qwen3-VL-32B-Instruct-FP8", ), - "16" : ( + "16": ( "unsloth/Qwen3-VL-32B-Instruct", "Qwen/Qwen3-VL-32B-Instruct", "unsloth/Qwen3-VL-32B-Instruct-bnb-4bit", ), }, - "unsloth/granite-4.0-350m-base-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-350m-base-unsloth-bnb-4bit": ( "unsloth/granite-4.0-350m-base", "ibm-granite/granite-4.0-350m-base", "unsloth/granite-4.0-350m-base-bnb-4bit", ), - "unsloth/granite-4.0-350m-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-350m-unsloth-bnb-4bit": ( "unsloth/granite-4.0-350m", "ibm-granite/granite-4.0-350m", "unsloth/granite-4.0-350m-bnb-4bit", ), - "unsloth/granite-4.0-h-350m-base-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-h-350m-base-unsloth-bnb-4bit": ( "unsloth/granite-4.0-h-350m-base", "ibm-granite/granite-4.0-h-350m-base", "unsloth/granite-4.0-h-350m-base-bnb-4bit", ), - "unsloth/granite-4.0-h-350m-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-h-350m-unsloth-bnb-4bit": ( "unsloth/granite-4.0-h-350m", "ibm-granite/granite-4.0-h-350m", "unsloth/granite-4.0-h-350m-bnb-4bit", ), - "unsloth/granite-4.0-1b-base-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-1b-base-unsloth-bnb-4bit": ( "unsloth/granite-4.0-1b-base", "ibm-granite/granite-4.0-1b-base", "unsloth/granite-4.0-1b-base-bnb-4bit", ), - "unsloth/granite-4.0-1b-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-1b-unsloth-bnb-4bit": ( "unsloth/granite-4.0-1b", "ibm-granite/granite-4.0-1b", "unsloth/granite-4.0-1b-bnb-4bit", ), - "unsloth/granite-4.0-h-1b-base-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-h-1b-base-unsloth-bnb-4bit": ( "unsloth/granite-4.0-h-1b-base", "ibm-granite/granite-4.0-h-1b-base", "unsloth/granite-4.0-h-1b-base-bnb-4bit", ), - "unsloth/granite-4.0-h-1b-unsloth-bnb-4bit" : ( + "unsloth/granite-4.0-h-1b-unsloth-bnb-4bit": ( "unsloth/granite-4.0-h-1b", "ibm-granite/granite-4.0-h-1b", "unsloth/granite-4.0-h-1b-bnb-4bit", ), - "unsloth/gpt-oss-safeguard-20b" : ( + "unsloth/gpt-oss-safeguard-20b": ( "unsloth/gpt-oss-safeguard-20b", "openai/gpt-oss-safeguard-20b", ), - "unsloth/gpt-oss-safeguard-120b" : ( + "unsloth/gpt-oss-safeguard-120b": ( "unsloth/gpt-oss-safeguard-120b", "openai/gpt-oss-safeguard-120b", ), - "unsloth/functiongemma-270m-it-unsloth-bnb-4bit" : ( + "unsloth/functiongemma-270m-it-unsloth-bnb-4bit": ( "unsloth/functiongemma-270m-it", "google/functiongemma-270m-it", "unsloth/functiongemma-270m-it-unsloth-bnb-4bit", ), } -INT_TO_FLOAT_MAPPER = {} -FLOAT_TO_INT_MAPPER = {} +INT_TO_FLOAT_MAPPER = {} +FLOAT_TO_INT_MAPPER = {} MAP_TO_UNSLOTH_16bit = {} FLOAT_TO_FP8_BLOCK_MAPPER = {} -FLOAT_TO_FP8_ROW_MAPPER = {} +FLOAT_TO_FP8_ROW_MAPPER = {} for key, values in __INT_TO_FLOAT_MAPPER.items(): block, row = None, None diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 5e893d2b6f..6ddb0a2ef7 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -14,7 +14,6 @@ from .llama import * import os -from ._utils import __version__ from unsloth_zoo.utils import _get_dtype from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import ( @@ -47,7 +46,6 @@ try: except: MistralSdpaAttention = MistralAttention MistralFlashAttention2 = MistralAttention -from unsloth_zoo.utils import Version, _get_dtype def MistralAttention_fast_forward( diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py index ea06016d72..723df47e8c 100644 --- a/unsloth/models/qwen3.py +++ b/unsloth/models/qwen3.py @@ -13,9 +13,7 @@ # limitations under the License. from .llama import * -import os -from ._utils import __version__ -from unsloth_zoo.utils import Version, _get_dtype +from unsloth_zoo.utils import Version from ..utils.packing import get_packed_info_from_kwargs from ..utils.attention_dispatch import ( AttentionConfig, @@ -47,9 +45,6 @@ except: f'Try `pip install --upgrade "transformers>=4.50.3"`\n' f"to obtain the latest transformers build, then restart this session." ) -from transformers.modeling_attn_mask_utils import ( - _prepare_4d_causal_attention_mask_for_sdpa, -) # For Pytorch 2.1.1 try: diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.py index e1f8c71b6b..efd58fc72e 100644 --- a/unsloth/models/qwen3_moe.py +++ b/unsloth/models/qwen3_moe.py @@ -13,8 +13,6 @@ # limitations under the License. from .llama import * -import os -from ._utils import __version__ from .llama import ( LlamaRotaryEmbedding, LlamaLinearScalingRotaryEmbedding, @@ -43,7 +41,6 @@ from transformers.models.qwen3_moe.modeling_qwen3_moe import ( # Qwen3SdpaAttention = Qwen3Attention # Qwen3FlashAttention2 = Qwen3Attention # pass -from unsloth_zoo.utils import Version, _get_dtype torch_nn_functional_softmax = torch.nn.functional.softmax diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index eacfecc6c3..bf8c3ac4f9 100755 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -18,7 +18,6 @@ __all__ = [ ] import torch -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union import inspect import os import re @@ -414,8 +413,6 @@ def _wrap_grpo_generate_and_score(trainer_cls): def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Patch for vLLM and Unsloth PEFT - import trl - import trl.trainer try: trainer = eval(f"trl.trainer.{trainer_file}") diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 8208dc922a..89e4961640 100755 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -33,12 +33,7 @@ from unsloth_zoo.log import logger from unsloth_zoo.device_type import device_synchronize import importlib.util from ..device_type import ( - is_hip, - get_device_type, DEVICE_TYPE, - DEVICE_TYPE_TORCH, - DEVICE_COUNT, - ALLOW_PREQUANTIZED_MODELS, ) import textwrap from ._utils import _get_inference_mode_context_manager diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py index b66ac7cf8a..76cab645a1 100644 --- a/unsloth/models/sentence_transformer.py +++ b/unsloth/models/sentence_transformer.py @@ -346,9 +346,9 @@ class FastSentenceTransformer(FastModel): print(f"Pooling mode detected as {mode}, updating...") return mode - except Exception as e: + except Exception: print( - f"Failed to detect pooling mode, not a sentence-transformers model. Using default pooling mode 'mean', this may or may not work." + "Failed to detect pooling mode, not a sentence-transformers model. Using default pooling mode 'mean', this may or may not work." ) return "mean" diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 93811c2668..efe66090b6 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -17,7 +17,6 @@ from transformers import ( BitsAndBytesConfig, AutoProcessor, AutoTokenizer, - AutoModelForCausalLM, ) try: @@ -61,29 +60,22 @@ from unsloth_zoo.patching_utils import patch_model_and_tokenizer from unsloth_zoo.training_utils import prepare_model_for_training from unsloth_zoo.utils import Version -from transformers import __version__ as transformers_version import types import functools import os import gc -import math -from typing import Optional, Tuple, List, Union -import re, inspect, sys -import contextlib +import inspect try: - from huggingface_hub.utils import get_token + pass except: # Old HF Hub versions <= 0.0.25 - from huggingface_hub.utils._token import get_token + pass from ..device_type import ( - is_hip, - get_device_type, DEVICE_TYPE, DEVICE_TYPE_TORCH, DEVICE_COUNT, - ALLOW_PREQUANTIZED_MODELS, ) __all__ = [ @@ -107,7 +99,7 @@ PRE_COMPILE_INFERENCE = [ "gpt_oss", ] -from transformers import GenerationConfig, CompileConfig, AutoConfig +from transformers import CompileConfig, AutoConfig try: from transformers import PreTrainedConfig @@ -125,10 +117,6 @@ _compile_config = CompileConfig( ) _compile_config.disable = True # Must set manually -from unsloth_zoo.vllm_utils import ( - convert_lora_modules, - return_lora_modules, -) try: torch_compiler_set_stance = torch.compiler.set_stance @@ -532,7 +520,7 @@ class FastBaseModel: flex_attn_impl = prefer_flex_attn_if_supported(model_class, auto_config) default_attn_impl = "flex_attention" if flex_attn_impl else "sdpa" - if not ("attn_implementation" in kwargs): + if "attn_implementation" not in kwargs: kwargs["attn_implementation"] = default_attn_impl if not supports_sdpa and kwargs.get("attn_implementation") == "sdpa": if os.environ.get("UNSLOTH_ENABLE_FLEX_ATTENTION", "0") == "0": @@ -580,13 +568,13 @@ class FastBaseModel: if dtype == torch.bfloat16: if float32_mixed_precision != True: print( - f"Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.\n" - f"To enable float32 training, use `float32_mixed_precision = True` during FastLanguageModel.from_pretrained" + "Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.\n" + "To enable float32 training, use `float32_mixed_precision = True` during FastLanguageModel.from_pretrained" ) else: print( - f"Unsloth: Using full float32 full finetuning. " - f"To enable bfloat16 training to reduce VRAM usage by 50% albeit with a slightly higher loss, do:\n" + "Unsloth: Using full float32 full finetuning. " + "To enable bfloat16 training to reduce VRAM usage by 50% albeit with a slightly higher loss, do:\n" "use `float32_mixed_precision = False` during FastLanguageModel.from_pretrained" ) os.environ["UNSLOTH_BFLOAT16_MIXED_PRECISION"] = "1" diff --git a/unsloth/save.py b/unsloth/save.py index 071e032c53..11fa05ecff 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -28,19 +28,16 @@ from peft.tuners.lora import Linear4bit as Peft_Linear4bit from peft.tuners.lora import Linear as Peft_Linear from typing import Optional, Callable, Union, List import sys -import requests import torch import os import shutil import pickle import gc from transformers.models.llama.modeling_llama import logger -from .kernels import fast_dequantize, QUANT_STATE, get_lora_parameters_bias +from .kernels import fast_dequantize, get_lora_parameters_bias import subprocess import psutil import re -from transformers.models.llama.modeling_llama import logger -from .tokenizer_utils import fix_sentencepiece_gguf from .models.loader_utils import get_model_name from .models._utils import _convert_torchao_model from .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPER @@ -1224,7 +1221,7 @@ def save_to_gguf( print( f"Unsloth: [1] Converting model into {first_conversion_dtype} GGUF format." ) - print(f"This might take 3 minutes...") + print("This might take 3 minutes...") initial_files, is_vlm_update = convert_to_gguf( model_name = model_name, @@ -1289,7 +1286,7 @@ def save_to_gguf( ) all_saved_locations.append(quantized_file) quants_created = True - except Exception as e: + except Exception: if IS_KAGGLE_ENVIRONMENT: raise RuntimeError( f"Unsloth: Quantization failed for {output_location}\n" @@ -1328,7 +1325,7 @@ def save_to_gguf( else: want_full_precision = first_conversion in frozenset(quantization_method) - print(f"Unsloth: All GGUF conversions completed successfully!") + print("Unsloth: All GGUF conversions completed successfully!") print(f"Generated files: {all_saved_locations}") return all_saved_locations, want_full_precision, is_vlm @@ -2118,7 +2115,7 @@ def unsloth_push_to_hub_gguf( cleanup_temp = False # Step 2: Call save_pretrained_gguf to do the conversion - print(f"Unsloth: Converting model to GGUF format...") + print("Unsloth: Converting model to GGUF format...") try: # Call save_pretrained_gguf - it returns all the info we need @@ -2512,13 +2509,11 @@ def unsloth_convert_lora_to_ggml_and_save_locally( ) -from .models.loader_utils import get_model_name from unsloth_zoo.saving_utils import ( merge_and_overwrite_lora, prepare_saving, ) from unsloth_zoo.llama_cpp import ( - install_llama_cpp, convert_to_gguf as _convert_to_gguf, ) @@ -2829,7 +2824,6 @@ def _unsloth_save_torchao_with_given_config( AutoModelForImageTextToText, AutoProcessor, ) - from torchao import quantize_ if isinstance(torchao_config, TorchAoConfig): quantization_config = torchao_config @@ -2967,7 +2961,6 @@ def not_implemented_save(*args, **kwargs): def patch_saving_functions(model, vision = False): import inspect import types - from typing import Callable, Optional, Union, List # And now re add our saving methods! if model.push_to_hub.__name__ == "unsloth_push_to_hub": diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 087a9a7f8a..c2687e6f17 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -18,22 +18,11 @@ from transformers import PreTrainedTokenizerFast import re import os from transformers.models.llama.modeling_llama import logger -from peft import PeftModelForCausalLM import torch -import itertools -import collections import numpy as np -import gc -import subprocess -import psutil from unsloth_zoo.tokenizer_utils import ( - mean_of_trained_tokens, add_new_tokens, - fix_untrained_tokens, -) -from unsloth_zoo.training_utils import ( - fix_zero_training_loss, ) __all__ = [ @@ -356,7 +345,7 @@ def fix_sentencepiece_tokenizer( from transformers.convert_slow_tokenizer import import_protobuf sentencepiece_model_pb2 = import_protobuf() - except Exception as e: + except Exception: try: import google.protobuf from unsloth_zoo.utils import Version @@ -896,7 +885,6 @@ def check_tokenizer( return convert_to_fast_tokenizer(tokenizer) -import inspect from inspect import getsource import trl import trl.trainer.sft_trainer @@ -941,7 +929,7 @@ def patch_sft_trainer_tokenizer(): Patches the trainer with changes """ try: - sft_trainer = eval(f"trl.trainer.sft_trainer.SFTTrainer") + sft_trainer = eval("trl.trainer.sft_trainer.SFTTrainer") except: return all_imports = dir(trl.trainer.sft_trainer) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 858dcf2cd3..f34ae9a143 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -14,16 +14,11 @@ import logging import os -import psutil -import warnings -from dataclasses import dataclass, field -from typing import Optional from functools import wraps import trl import inspect from trl import SFTTrainer -from . import is_bfloat16_supported from unsloth.utils import ( configure_padding_free, configure_sample_packing,