From dc8bca6713d97cb3afcfaf584dbce06a18af5dda Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 22:12:21 -0700 Subject: [PATCH 01/11] Update llama.py --- unsloth/models/llama.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index bae6d5b80d..2524ec1312 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1386,11 +1386,7 @@ def _wrap_fast_inference(generate, device_type, dtype, model): pass # For newer HF - if SUPPORTS_LLAMA32: - # kwargs["cache_implementation"] = "hybrid" - pass - else: - kwargs["cache_implementation"] = "dynamic" + kwargs["cache_implementation"] = "dynamic" # For num_logits_to_keep kwargs["num_logits_to_keep"] = 1 From 63e3a85efb184a7f4bfecf6430d08868fe813881 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 22:56:41 -0700 Subject: [PATCH 02/11] Update _utils.py --- unsloth/models/_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index af7e1eb293..309c0d913e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -306,6 +306,14 @@ if Version(xformers_version) >= Version("0.0.27"): accelerate_new_send_to_device = _fixed_send_to_device pass pass + +# Transformers 4.46 breaks dynamic caching. This is a hack +import transformers.generation.configuration_utils +if hasattr(transformers.generation.configuration_utils, "ALL_CACHE_IMPLEMENTATIONS"): + if type(transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS) is list: + transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS.append("dynamic") + pass +pass # ============================================= # ============================================= From c331c886ee0a8747157f9a41855048eabc93b38e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 23:00:09 -0700 Subject: [PATCH 03/11] Remove version checks --- unsloth/models/llama.py | 5 +---- unsloth/models/loader.py | 19 ------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 2524ec1312..a245330108 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -62,10 +62,7 @@ from peft.tuners.lora import Linear4bit as Peft_Linear4bit from ..save import patch_saving_functions import re, os, inspect, math, sys from huggingface_hub.utils._token import get_token -from transformers import __version__ as transformers_version -from packaging.version import Version -transformers_version = Version(transformers_version) -SUPPORTS_LLAMA32 = transformers_version > Version("4.45.0") + def original_apply_qkv(self, X): Q = self.q_proj(X) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index ee1f680ace..cb03088b64 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -138,17 +138,6 @@ def get_model_name(model_name, load_in_4bit = True): pass -LLAMA32_MODEL_NAMES = set(( - "unsloth/Llama-3.2-1B-bnb-4bit", - "unsloth/Llama-3.2-3B-bnb-4bit", - "unsloth/Llama-3.2-1B", - "unsloth/Llama-3.2-3B", - "unsloth/Llama-3.2-1B-Instruct-bnb-4bit", - "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", - "unsloth/Llama-3.2-1B-Instruct", - "unsloth/Llama-3.2-3B-Instruct", -)) - class FastLanguageModel(FastLlamaModel): @staticmethod def from_pretrained( @@ -255,14 +244,6 @@ class FastLanguageModel(FastLlamaModel): f"to obtain the latest transformers build, then restart this session."\ ) - elif model_name.lower() in LLAMA32_MODEL_NAMES and not SUPPORTS_LLAMA32: - raise ImportError( - f"Unsloth: Your transformers version of {transformers_version} does not support Llama 3.2.\n"\ - f"The minimum required version is 4.46\n"\ - f'Try `pip install --upgrade "transformers>=4.46"`\n'\ - f"to obtain the latest transformers build, then restart this session."\ - ) - dispatch_model = FastLlamaModel elif model_type == "mistral": dispatch_model = FastMistralModel From 5b345ec757b751c184670fa51ad93c3fc64c431e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 23:13:49 -0700 Subject: [PATCH 04/11] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b61908a690..6cec9627ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ exclude = ["images*"] huggingface = [ "packaging", "tyro", - "transformers>=4.43.2", + "transformers>=4.45.0", "datasets>=2.16.0", "sentencepiece>=0.2.0", "tqdm", From 930d2ad1a830abea6b3c38c5536cf040677bdf5d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 23:47:15 -0700 Subject: [PATCH 05/11] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6cec9627ba..26761ebc58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,7 +212,7 @@ colab-ampere-torch220 = [ colab-new = [ "packaging", "tyro", - "transformers>=4.43.2", + "transformers>=4.45.0", "datasets>=2.16.0", "sentencepiece>=0.2.0", "tqdm", From 4f4ef22035f89606399e85af690779c05bd72c11 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 26 Sep 2024 00:02:15 -0700 Subject: [PATCH 06/11] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f8944c5a77..0b6ebb9f89 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and | **DPO Zephyr** | [▶️ Start for free](https://colab.research.google.com/drive/15vttTpzzVXv_tJwEk-hIcQ0S9FcEWvwP?usp=sharing) | 1.9x faster | 43% less | - **Kaggle Notebooks** for [Llama 3.1 (8B)](https://www.kaggle.com/danielhanchen/kaggle-llama-3-1-8b-unsloth-notebook), [Gemma 2 (9B)](https://www.kaggle.com/code/danielhanchen/kaggle-gemma-7b-unsloth-notebook/), [Mistral (7B)](https://www.kaggle.com/code/danielhanchen/kaggle-mistral-7b-unsloth-notebook) +- Run [Llama 3.2 1B 3B notebook](https://colab.research.google.com/drive/1hoHFpf7ROqk_oZHzxQdfPW9yvTxnvItq?usp=sharing) and [Llama 3.2 conversational notebook](https://colab.research.google.com/drive/1T5-zKWM_5OD21QHwXHiV9ixTRR7k3iB9?usp=sharing) - Run [Llama 3.1 conversational notebook](https://colab.research.google.com/drive/15OyFkGoCImV9dSsewU1wa2JuKB4-mDE_?usp=sharing) and [Mistral v0.3 ChatML](https://colab.research.google.com/drive/15F1xyn8497_dUbxZP4zWmPZ3PJx1Oymv?usp=sharing) - This [text completion notebook](https://colab.research.google.com/drive/1ef-tab5bhkvWmBOObepl1WgJvfvSzn5Q?usp=sharing) is for continued pretraining / raw text - This [continued pretraining notebook](https://colab.research.google.com/drive/1tEd1FrOXWMnCU9UIvdYhs61tkxdMuKZu?usp=sharing) is for learning another language From 6bbca3aaa82690d1c6f4cf3bfeb706233bab43e5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 26 Sep 2024 00:05:38 -0700 Subject: [PATCH 07/11] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0b6ebb9f89..12fd793e5a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ -### Finetune Llama 3.1, Mistral, Phi-3.5 & Gemma 2-5x faster with 80% less memory! +### Finetune Llama 3.2, Mistral, Phi-3.5 & Gemma 2-5x faster with 80% less memory! ![](https://i.ibb.co/sJ7RhGG/image-41.png) @@ -39,9 +39,9 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and - Click [here](https://github.com/unslothai/unsloth/wiki) for detailed documentation for Unsloth. ## 🦥 Unsloth.ai News +- 📣 NEW! [Llama 3.2 Conversational notebook](https://colab.research.google.com/drive/1T5-zKWM_5OD21QHwXHiV9ixTRR7k3iB9?usp=sharing) includes training only on completions / outputs (increase accuracy), ShareGPT standardization and more! - 📣 NEW! [Qwen 2.5 7b notebook](https://colab.research.google.com/drive/1Kose-ucXO1IBaZq5BvbwWieuubP7hxvQ?usp=sharing) finetuning is supported! Qwen 2.5 comes in multiple sizes - check our [4bit uploads](https://huggingface.co/unsloth) for 4x faster downloads!. 14b fits in a Colab GPU! [Qwen 2.5 conversational notebook](https://colab.research.google.com/drive/1qN1CEalC70EO1wGKhNxs1go1W9So61R5?usp=sharing) - 📣 NEW! [Mistral Small 22b notebook](https://colab.research.google.com/drive/1oCEHcED15DzL8xXGU1VTx5ZfOJM8WY01?usp=sharing) finetuning fits in under 16GB of VRAM! -- 📣 NEW! [Llama 3.1 Conversational notebook](https://colab.research.google.com/drive/15OyFkGoCImV9dSsewU1wa2JuKB4-mDE_?usp=sharing) includes training only on completions / outputs (increase accuracy), ShareGPT standardization and more! - 📣 NEW! [Phi-3.5 (mini)](https://colab.research.google.com/drive/1lN6hPQveB_mHSnTOYifygFcrO8C1bxq4?usp=sharing) now supported - 📣 NEW! [Gemma-2-2b](https://colab.research.google.com/drive/1weTpKOjBZxZJ5PQ-Ql8i6ptAY2x-FWVA?usp=sharing) now supported! Try out [Chat interface](https://colab.research.google.com/drive/1i-8ESvtLRGNkkUQQr_-z_rcSAIo9c3lM?usp=sharing)! - 📣 NEW! [Llama 3.1 8b, 70b](https://colab.research.google.com/drive/1Ys44kVvmeZtnICzWz0xgpRnrIOjZAuxp?usp=sharing) & [Mistral Nemo-12b](https://colab.research.google.com/drive/17d3U-CAIwzmbDRqbZ9NnpHxCkmXB6LZ0?usp=sharing) both Base and Instruct are now supported From 88a542a1297f907c7b987f3b278284c1f7086e9c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 26 Sep 2024 00:12:42 -0700 Subject: [PATCH 08/11] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 12fd793e5a..2b8e79f021 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and ## 🦥 Unsloth.ai News - 📣 NEW! [Llama 3.2 Conversational notebook](https://colab.research.google.com/drive/1T5-zKWM_5OD21QHwXHiV9ixTRR7k3iB9?usp=sharing) includes training only on completions / outputs (increase accuracy), ShareGPT standardization and more! +- 📣 NEW! [Llama 3.2 Kaggle notebook](https://www.kaggle.com/danielhanchen/kaggle-llama-3-2-1b-3b-unsloth-notebook) and [Llama 3.2 Kaggle conversational notebook](https://www.kaggle.com/code/danielhanchen/kaggle-llama-3-2-1b-3b-conversational-unsloth/notebook) - 📣 NEW! [Qwen 2.5 7b notebook](https://colab.research.google.com/drive/1Kose-ucXO1IBaZq5BvbwWieuubP7hxvQ?usp=sharing) finetuning is supported! Qwen 2.5 comes in multiple sizes - check our [4bit uploads](https://huggingface.co/unsloth) for 4x faster downloads!. 14b fits in a Colab GPU! [Qwen 2.5 conversational notebook](https://colab.research.google.com/drive/1qN1CEalC70EO1wGKhNxs1go1W9So61R5?usp=sharing) - 📣 NEW! [Mistral Small 22b notebook](https://colab.research.google.com/drive/1oCEHcED15DzL8xXGU1VTx5ZfOJM8WY01?usp=sharing) finetuning fits in under 16GB of VRAM! - 📣 NEW! [Phi-3.5 (mini)](https://colab.research.google.com/drive/1lN6hPQveB_mHSnTOYifygFcrO8C1bxq4?usp=sharing) now supported From c0b4d640f245926f1b9afdeb9006a874d0a03a76 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 26 Sep 2024 01:23:40 -0700 Subject: [PATCH 09/11] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 477d993e69..04690d3566 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -1180,11 +1180,6 @@ def patch_sft_trainer_tokenizer(): check_text = \ "\n"\ - "if self._inner_training_loop.__name__ != '_fast_inner_training_loop':\n"\ - " raise RuntimeError(\n"\ - " 'Please do not edit specific areas of the Unsloth codebase or you will get CUDA segfaults.'\n"\ - " )\n"\ - "pass\n"\ "import subprocess, re, gc, numpy as np\n"\ "a = np.array([0,])\n"\ "try:\n"\ From b314837622c4ebe0f569e84a079384b273d13a40 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Sep 2024 01:36:45 -0700 Subject: [PATCH 10/11] Update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 26761ebc58..59ef1b8aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ exclude = ["images*"] huggingface = [ "packaging", "tyro", - "transformers>=4.45.0", + "transformers>=4.45.1", "datasets>=2.16.0", "sentencepiece>=0.2.0", "tqdm", @@ -212,7 +212,7 @@ colab-ampere-torch220 = [ colab-new = [ "packaging", "tyro", - "transformers>=4.45.0", + "transformers>=4.45.1", "datasets>=2.16.0", "sentencepiece>=0.2.0", "tqdm", From afbb140a793cdcd29b207069c64097bb940bbe83 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 29 Sep 2024 01:42:58 -0700 Subject: [PATCH 11/11] Update loader.py --- unsloth/models/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index cb03088b64..0ac9b02743 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -192,7 +192,7 @@ class FastLanguageModel(FastLlamaModel): pass # Cannot be both! - if is_model and is_peft: + if (is_model and is_peft) and not SUPPORTS_LLAMA32: raise RuntimeError( "Unsloth: Your repo has a LoRA adapter and a base model.\n"\ "You have 2 files `config.json` and `adapter_config.json`.\n"\