diff --git a/README.md b/README.md index 555e08089e..6df6616225 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ -### Finetune Mistral, Gemma, Llama 2-5x faster with 80% less memory! +### Finetune Llama 3, Mistral & Gemma 2-5x faster with 80% less memory! ![](https://i.ibb.co/sJ7RhGG/image-41.png) @@ -22,12 +22,11 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and | Unsloth supports | Free Notebooks | Performance | Memory use | |-----------------|--------------------------------------------------------------------------------------------------------------------------|-------------|----------| -| **Llama-3 8b** | [▶️ Start on Colab](https://colab.research.google.com/drive/135ced7oHytdxu3N2DNe1Z0kqjyYIkDXp?usp=sharing) | 2x faster | 60% less | -| **Gemma 7b** | [▶️ Start on Colab](https://colab.research.google.com/drive/10NbwlsRChbma1v55m8LAPYG15uQv6HLo?usp=sharing) | 2.4x faster | 71% less | -| **Mistral 7b** | [▶️ Start on Colab](https://colab.research.google.com/drive/1Dyauq4kTZoLewQ1cApceUQVNcnnNTzg_?usp=sharing) | 2.2x faster | 73% less | -| **TinyLlama** | [▶️ Start on Colab](https://colab.research.google.com/drive/1AZghoNBQaMDgWJpi4RbffGM1h6raLUj9?usp=sharing) | 3.9x faster | 82% less | -| **CodeLlama 34b** A100 | [▶️ Start on Colab](https://colab.research.google.com/drive/1y7A0AxE3y8gdj4AVkl2aZX47Xu3P1wJT?usp=sharing) | 1.9x faster | 49% less | -| **Mistral 7b** 1xT4 | [▶️ Start on Kaggle](https://www.kaggle.com/code/danielhanchen/kaggle-mistral-7b-unsloth-notebook) | 5x faster\* | 73% less | +| **Llama 3 (8B)** | [▶️ Start on Colab](https://colab.research.google.com/drive/135ced7oHytdxu3N2DNe1Z0kqjyYIkDXp?usp=sharing) | 2x faster | 60% less | +| **Mistral (7B)** | [▶️ Start on Colab](https://colab.research.google.com/drive/1Dyauq4kTZoLewQ1cApceUQVNcnnNTzg_?usp=sharing) | 2.2x faster | 73% less | +| **Gemma (7B)** | [▶️ Start on Colab](https://colab.research.google.com/drive/10NbwlsRChbma1v55m8LAPYG15uQv6HLo?usp=sharing) | 2.4x faster | 71% less | +| **Llama 3 (8B)** 1xT4 | [▶️ Start on Kaggle](https://www.kaggle.com/code/danielhanchen/kaggle-llama-3-8b-unsloth-notebook) | 5x faster\* | 73% less | +| **ORPO** | [▶️ Start on Colab](https://colab.research.google.com/drive/11t4njE3c4Lxl-07OD8lJSMKkfyJml3Tn?usp=sharing) | 1.9x faster | 43% less | | **DPO - Zephyr** | [▶️ Start on Colab](https://colab.research.google.com/drive/15vttTpzzVXv_tJwEk-hIcQ0S9FcEWvwP?usp=sharing) | 1.9x faster | 43% less | - Benchmarking compared to FA2 + Hugging Face combined. @@ -36,7 +35,8 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and - \* Kaggle has 2x T4s, but we use 1. Due to overhead, 1x T4 is 5x faster. ## 🦥 Unsloth.ai News -- 📣 NEW! [Llama-3 8b](https://colab.research.google.com/drive/135ced7oHytdxu3N2DNe1Z0kqjyYIkDXp?usp=sharing) now works! Llama-3 70b also works (just change the model name in the notebook). +- 📣 NEW! [Llama-3 8b](https://colab.research.google.com/drive/135ced7oHytdxu3N2DNe1Z0kqjyYIkDXp?usp=sharing) now works! Llama-3 70b also works (change the model name in the notebook). +- 📣 NEW! [ORPO support](https://colab.research.google.com/drive/11t4njE3c4Lxl-07OD8lJSMKkfyJml3Tn?usp=sharing) is here! - 📣 NEW! We cut memory usage by a [further 30%](https://unsloth.ai/blog/long-context) and now support fine-tuning of LLMs with [4x longer context windows](https://unsloth.ai/blog/long-context)! No change required if you're using our notebooks. To enable, simply change 1 line: ```python model = FastLanguageModel.get_peft_model( @@ -46,8 +46,6 @@ model = FastLanguageModel.get_peft_model( ``` - 📣 [CodeGemma](https://colab.research.google.com/drive/19lwcRk_ZQ_ZtX-qzFP3qZBBHZNcMD1hh?usp=sharing) now works along with [Gemma 7b](https://colab.research.google.com/drive/10NbwlsRChbma1v55m8LAPYG15uQv6HLo?usp=sharing) and [Gemma 2b](https://colab.research.google.com/drive/15gGm7x_jTm017_Ic8e317tdIpDG53Mtu?usp=sharing) - 📣 [2x faster inference](https://colab.research.google.com/drive/15vttTpzzVXv_tJwEk-hIcQ0S9FcEWvwP?usp=sharing) added for all our models -- 📣 [DPO support](https://colab.research.google.com/drive/15vttTpzzVXv_tJwEk-hIcQ0S9FcEWvwP?usp=sharing) is now included. [More info](#DPO) on DPO -- 📣 We did a [blog](https://huggingface.co/blog/unsloth-trl) with 🤗Hugging Face and are in their official docs! Check out the [SFT docs](https://huggingface.co/docs/trl/main/en/sft_trainer#accelerate-fine-tuning-2x-using-unsloth) and [DPO docs](https://huggingface.co/docs/trl/main/en/dpo_trainer#accelerate-dpo-fine-tuning-using-unsloth) ## 🔗 Links and Resources | Type | Links | @@ -182,18 +180,20 @@ max_seq_length = 2048 # Supports RoPE Scaling interally, so choose any! url = "https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl" dataset = load_dataset("json", data_files = {"train" : url}, split = "train") -# 4bit pre quantized models we support - 4x faster downloading! +# 4bit pre quantized models we support for 4x faster downloading + no OOMs. fourbit_models = [ "unsloth/mistral-7b-bnb-4bit", + "unsloth/mistral-7b-instruct-v0.2-bnb-4bit", "unsloth/llama-2-7b-bnb-4bit", - "unsloth/llama-2-13b-bnb-4bit", - "unsloth/codellama-34b-bnb-4bit", - "unsloth/tinyllama-bnb-4bit", -] # Go to https://huggingface.co/unsloth for more 4-bit models! + "unsloth/gemma-7b-bnb-4bit", + "unsloth/gemma-7b-it-bnb-4bit", # Instruct version of Gemma 7b + "unsloth/gemma-2b-bnb-4bit", + "unsloth/gemma-2b-it-bnb-4bit", # Instruct version of Gemma 2b + "unsloth/llama-3-8b-bnb-4bit", # [NEW] 15 Trillion token Llama-3 +] # More models at https://huggingface.co/unsloth -# Load Llama model model, tokenizer = FastLanguageModel.from_pretrained( - model_name = "unsloth/mistral-7b-bnb-4bit", # Supports Llama, Mistral - replace this! + model_name = "unsloth/llama-3-8b-bnb-4bit", max_seq_length = max_seq_length, dtype = None, load_in_4bit = True, @@ -208,7 +208,8 @@ model = FastLanguageModel.get_peft_model( lora_alpha = 16, lora_dropout = 0, # Supports any, but = 0 is optimized bias = "none", # Supports any, but = "none" is optimized - use_gradient_checkpointing = True, + # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes! + use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context random_state = 3407, max_seq_length = max_seq_length, use_rslora = False, # We support rank stabilized LoRA @@ -272,7 +273,8 @@ model = FastLanguageModel.get_peft_model( lora_alpha = 64, lora_dropout = 0, # Supports any, but = 0 is optimized bias = "none", # Supports any, but = "none" is optimized - use_gradient_checkpointing = True, + # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes! + use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context random_state = 3407, max_seq_length = max_seq_length, ) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index d31b6cf7a4..4e7a71aee7 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -281,6 +281,17 @@ def get_chat_template( IS_GEMMA = True pass + # We add a check for Llama-3 + # if chat_template == "llama-3": + # tokenizer._using_llama3_template = True + # else: + # llama3_tokens = set(["<|end_header_id|>", "<|eot_id|>", "<|start_header_id|>"]) + # check_llama3_tokens = llama3_tokens & set(str(x) for x in tokenizer.added_tokens_decoder.values()) + # if len(check_llama3_tokens) == len(llama3_tokens): + # tokenizer._using_llama3_template = True + # pass + # pass + # We first check if the tokenizer is a fast one. If not, we cannot convert this! is_fast_tokenizer = getattr(tokenizer, "is_fast", False) old_padding_side = tokenizer.padding_side diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 45c75010b8..a7cacea9b9 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1284,6 +1284,15 @@ class FastLlamaModel: # Add save modules patch_saving_functions(model) + # Save tokenizer for inference purposes + tokenizer.padding_side = "left" # Force inference + internal_model = model + while hasattr(internal_model, "model"): + internal_model._saved_temp_tokenizer = tokenizer + internal_model = internal_model.model + pass + internal_model._saved_temp_tokenizer = tokenizer + return model, tokenizer pass @@ -1534,8 +1543,12 @@ class FastLlamaModel: if not SUPPORTS_LOFTQ: del arguments["loftq_config"] if not SUPPORTS_RSLORA: del arguments["use_rslora"] + _saved_temp_tokenizer = model._saved_temp_tokenizer + lora_config = LoraConfig(**arguments) model = _get_peft_model(model, lora_config) + + model._saved_temp_tokenizer = _saved_temp_tokenizer model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing) @@ -1554,6 +1567,18 @@ class FastLlamaModel: model.model.lm_head.modules_to_save.default.requires_grad_(True) pass + # Patch tokenizer to pad to the right + internal_model = model + while hasattr(internal_model, "model"): + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "right" + pass + internal_model = internal_model.model + pass + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "right" + pass + return model pass @@ -1751,6 +1776,18 @@ class FastLlamaModel: # Wrap model.generate model._unwrapped_old_generate = model.generate model.generate = _wrap_fast_inference(model.generate, device_type, dtype) + + # Patch tokenizer to pad to the left + internal_model = model + while hasattr(internal_model, "model"): + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "left" + pass + internal_model = internal_model.model + pass + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "left" + pass pass @@ -1777,8 +1814,18 @@ class FastLlamaModel: model.generate = model._unwrapped_old_generate del model._unwrapped_old_generate pass + + # Patch tokenizer to pad to the right + internal_model = model + while hasattr(internal_model, "model"): + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "right" + pass + internal_model = internal_model.model + pass + if hasattr(internal_model, "_saved_temp_tokenizer"): + internal_model._saved_temp_tokenizer.padding_side = "right" + pass pass pass - - diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index fa864a9a80..a107200eae 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -76,6 +76,7 @@ class FastLanguageModel(FastLlamaModel): fix_tokenizer = True, trust_remote_code = False, use_gradient_checkpointing = True, + resize_model_vocab = None, *args, **kwargs, ): if token is None and "HF_TOKEN" in os.environ: @@ -149,6 +150,9 @@ class FastLanguageModel(FastLlamaModel): trust_remote_code = trust_remote_code, *args, **kwargs, ) + + if resize_model_vocab is not None: + model.resize_token_embeddings(resize_model_vocab) # In case the model supports tagging, add the unsloth tag. if hasattr(model, "add_model_tags"): diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 56108939b9..80d0ffdf7f 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -559,6 +559,15 @@ class FastMistralModel(FastLlamaModel): # Add save modules patch_saving_functions(model) + + # Save tokenizer for inference purposes + tokenizer.padding_side = "left" # Force inference + internal_model = model + while hasattr(internal_model, "model"): + internal_model._saved_temp_tokenizer = tokenizer + internal_model = internal_model.model + pass + internal_model._saved_temp_tokenizer = tokenizer return model, tokenizer pass diff --git a/unsloth/save.py b/unsloth/save.py index 6e9d82c88a..a2c55bb539 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -689,7 +689,7 @@ pass def install_llama_cpp_clone_non_blocking(): - full_command = ["git", "clone", "https://github.com/ggerganov/llama.cpp"] + full_command = ["git", "clone", "--recursive", "https://github.com/ggerganov/llama.cpp"] run_installer = subprocess.Popen(full_command, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT) return run_installer pass @@ -742,7 +742,7 @@ def install_llama_cpp_old(version = -10): # Clone a specific commit # Also don't use the GPU! commands = [ - "git clone https://github.com/ggerganov/llama.cpp", + "git clone --recursive https://github.com/ggerganov/llama.cpp", f"cd llama.cpp && git reset --hard {version} && git clean -df", "make clean -C llama.cpp", f"make all -j{psutil.cpu_count()*2} -C llama.cpp", @@ -767,7 +767,7 @@ def install_llama_cpp_blocking(use_cuda = True): use_cuda = "LLAMA_CUDA=1" if use_cuda else "" commands = [ - "git clone https://github.com/ggerganov/llama.cpp", + "git clone --recursive https://github.com/ggerganov/llama.cpp", "make clean -C llama.cpp", f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp", "pip install gguf protobuf", @@ -922,16 +922,9 @@ def save_to_gguf( f"The output location will be {final_location}\n"\ "This will take 3 minutes...") - # We first check if tokenizer.model exists in the model_directory - if os.path.exists(f"{model_directory}/tokenizer.model"): - vocab_type = "hfft" - else: - vocab_type = "bpe" - pass - if use_fast_convert: command = f"python llama.cpp/convert.py {model_directory} "\ - f"--outfile {final_location} --vocab-type {vocab_type} "\ + f"--outfile {final_location} --vocab-type spm,hfft,bpe "\ f"--outtype {first_conversion} --concurrency {n_cpus}" else: # Need to fix convert-hf-to-gguf.py for some models! @@ -966,7 +959,7 @@ def save_to_gguf( "You might have to compile llama.cpp yourself, then run this again.\n"\ "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ "You must run this in the same folder as you're saving your model.\n"\ - "git clone https://github.com/ggerganov/llama.cpp\n"\ + "git clone --recursive https://github.com/ggerganov/llama.cpp\n"\ "cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\ "Once that's done, redo the quantization." ) @@ -1006,7 +999,7 @@ def save_to_gguf( "Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\ "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ "You must run this in the same folder as you're saving your model.\n"\ - "git clone https://github.com/ggerganov/llama.cpp\n"\ + "git clone --recursive https://github.com/ggerganov/llama.cpp\n"\ "cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\ "Once that's done, redo the quantization." ) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index f1a9daa998..5dc5856c2a 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -524,7 +524,7 @@ def add_new_tokens( tokenizer, new_tokens = [], method = "mean", - interpolation = 0.05, + interpolation = 0.5, ): """ Smartly resizes the tokenizer and adds new tokens to the model.