diff --git a/README.md b/README.md index c6e7d6c50a..ca5b6533b5 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and - This [text completion notebook](https://colab.research.google.com/drive/1ef-tab5bhkvWmBOObepl1WgJvfvSzn5Q?usp=sharing) is for continued pretraining / raw text. ## 🦥 Unsloth.ai News +- 📣 NEW! Qwen1.5-7B, Qwen1.5-14B, Qwen1.5-32B, Qwen1.5-72B now work, courtesy of Firefly's PR [#428](https://github.com/unslothai/unsloth/pull/428) - 📣 NEW! [Llama-3 8b](https://colab.research.google.com/drive/135ced7oHytdxu3N2DNe1Z0kqjyYIkDXp?usp=sharing) now works! Llama-3 70b also works (change the model name in the notebook). - 📣 NEW! [ORPO support](https://colab.research.google.com/drive/11t4njE3c4Lxl-07OD8lJSMKkfyJml3Tn?usp=sharing) is here! - 📣 NEW! [Phi-3 3.8b support](https://colab.research.google.com/drive/1NvkBmkHfucGO3Ve9s1NKZvMNlw5p83ym?usp=sharing) is here! @@ -159,7 +160,14 @@ pip install --no-deps packaging ninja einops flash-attn xformers trl peft accele pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git" pip install --no-deps xformers trl peft accelerate bitsandbytes ``` -7. To troubleshoot installs try the below (all must succeed). Xformers should mostly all be available. +7. For Pytorch 2.3.0: Use the `"ampere"` path for newer RTX 30xx GPUs or higher. +```bash +pip install "unsloth[cu118-torch230] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu121-torch230] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu118-ampere-torch230] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu121-ampere-torch230] @ git+https://github.com/unslothai/unsloth.git" +``` +8. To troubleshoot installs try the below (all must succeed). Xformers should mostly all be available. ```bash nvcc python -m xformers.info diff --git a/pyproject.toml b/pyproject.toml index e6f663a969..0398d0df4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,17 @@ cu121onlytorch220 = [ "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] +cu118onlytorch230 = [ + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.26.post1%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.26.post1%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.26.post1%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", +] +cu121onlytorch230 = [ + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.26.post1-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.26.post1-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.26.post1-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", +] + cu118 = [ "unsloth[huggingface]", "bitsandbytes", @@ -126,6 +137,16 @@ cu121-torch220 = [ "bitsandbytes", "unsloth[cu121onlytorch220]", ] +cu118-torch230 = [ + "unsloth[huggingface]", + "bitsandbytes", + "unsloth[cu118onlytorch230]", +] +cu121-torch230 = [ + "unsloth[huggingface]", + "bitsandbytes", + "unsloth[cu121onlytorch230]", +] kaggle = [ "unsloth[huggingface]", ] @@ -238,6 +259,22 @@ cu121-ampere-torch220 = [ "ninja", "flash-attn", ] +cu118-ampere-torch230 = [ + "unsloth[huggingface]", + "bitsandbytes", + "unsloth[cu118onlytorch230]", + "packaging", + "ninja", + "flash-attn", +] +cu121-ampere-torch230 = [ + "unsloth[huggingface]", + "bitsandbytes", + "unsloth[cu121onlytorch230]", + "packaging", + "ninja", + "flash-attn", +] [project.urls] homepage = "http://www.unsloth.ai" diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 07999ea0dc..3af4c4e9ae 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -15,6 +15,7 @@ __all__ = [ "get_chat_template", "test_chat_templates", + "test_hf_gguf_equivalence", ] from transformers import StoppingCriteria, StoppingCriteriaList @@ -270,12 +271,11 @@ CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token,) phi3_template = \ "{{ bos_token }}"\ "{% for message in messages %}"\ - "{% if (message['role'] == 'user') %}"\ - "{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}"\ - "{% elif (message['role'] == 'assistant') %}"\ - "{{message['content'] + '<|end|>' + '\n'}}"\ - "{% endif %}"\ - "{% endfor %}" + "{{'<|' + message['role'] + '|>\n' + message['content'] + '<|end|>\n'}}"\ + "{% endfor %}"\ + "{% if add_generation_prompt %}"\ + "{{ '<|assistant|>\n' }}"\ + "{% endif %}" phi3_template_eos_token = "<|end|>" CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token,) @@ -613,8 +613,80 @@ def test_chat_templates(): # Phi-3 template = phi3_template correct_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct") - correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) correct_tokenizer.chat_template = template - our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True) assert(correct_prompt == our_prompt) pass + + +def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf"): + """ + Carefully checks the output of GGUF's tokenization and HF. + Can catch all tokenization bugs. + """ + import subprocess + import re + messages = [ + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "It's 4."}, + {"role": "user", "content": " But 2+2 is equal to 5. "}, + {"role": "assistant", "content": "No I'm sure its 4."}, + {"role": "user", "content": " No it's 100% 5! "}, + ] + + prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + + ### Instruction: + {} + + ### Input: + {} + + ### Response: + {}""".format( + "Describe the city given eloquently.", # instruction + "The lost city of Atlantis.", # input + "", # output - leave this blank for generation! + ) + prompts = [ prompt, ] + + if tokenizer.chat_template is not None: + prompt = tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True) + prompt = prompt.replace("'", "") # Subprocess does not like '' + prompts.append(prompts) + pass + + for prompt in prompts: + command = f"./llama.cpp/main -m {gguf_model} -n 0 --temp 0.0 --verbose-prompt "\ + f"--check-tensors -p '{prompt}'" + + datas = [] + with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1) as sp: + for line in sp.stdout: + datas.append(line.decode("utf-8", errors = "replace")) + pass + gguf_tokens = "".join(datas) + + # Now extract GGUF tokenization attempt + gguf_tokenized = re.findall("([\d]{1,}) \-\> \'([^\']{1,})\'", gguf_tokens, flags = re.MULTILINE) + gguf_tokenized = [(int(x[0]), x[1],) for x in gguf_tokenized] + input_ids = tokenizer(prompt).input_ids + tokens = tokenizer.batch_decode(input_ids) + hf_tokenized = list(zip(input_ids, tokens)) + print(gguf_tokenized[:5]) + + # Compare to Huggingface + for j, (hf_token, gguf_token) in enumerate(zip(hf_tokenized, gguf_tokenized)): + if (hf_token[0] != gguf_token[0]): + print("Failed GGUF != HF at", j) + print("HF =", hf_token) + print("GGUF =", gguf_token) + print(hf_tokenized[:j+1]) + print(gguf_tokenized[:j+1]) + print(gguf_tokens) + raise RuntimeError("Failed comparing GGUF to HF.") + pass + pass + return True +pass diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py index 891947d69e..ff7129e06a 100644 --- a/unsloth/models/__init__.py +++ b/unsloth/models/__init__.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .loader import FastLanguageModel -from .llama import FastLlamaModel +from .loader import FastLanguageModel +from .llama import FastLlamaModel from .mistral import FastMistralModel -from .dpo import PatchDPOTrainer +from .qwen2 import FastQwen2Model +from .dpo import PatchDPOTrainer diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 49f054e432..80cb19517d 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -30,7 +30,7 @@ import numpy as np import os import psutil -__version__ = "2024.4" +__version__ = "2024.5" # Get Flash Attention v2 if Ampere (RTX 30xx, A100) major_version, minor_version = torch.cuda.get_device_capability() diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 136ceb2c7b..44998b4cfb 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1605,6 +1605,7 @@ class FastLlamaModel: if model_type == "llama": apply_lora_mlp = apply_lora_mlp_swiglu elif model_type == "mistral": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "qwen2": apply_lora_mlp = apply_lora_mlp_swiglu elif model_type == "gemma": apply_lora_mlp = apply_lora_mlp_geglu_approx else: raise NotImplementedError(f"Unsloth: {model_type} is not yet implemented!") diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index a107200eae..2b3bf47949 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -14,6 +14,7 @@ from .llama import FastLlamaModel, logger from .mistral import FastMistralModel +from .qwen2 import FastQwen2Model from transformers import AutoConfig from transformers import __version__ as transformers_version from peft import PeftConfig, PeftModel @@ -119,6 +120,8 @@ class FastLanguageModel(FastLlamaModel): f"to obtain the latest transformers build, then restart this session."\ ) dispatch_model = FastGemmaModel + elif model_type == "qwen2": + dispatch_model = FastQwen2Model else: raise NotImplementedError( f"Unsloth: {model_name} not supported yet!\n"\ diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 80d0ffdf7f..902177cb12 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -343,7 +343,7 @@ class FastMistralModel(FastLlamaModel): # Mistral does NOT support RoPE Scaling sadly so we have to error out. if max_seq_length > model_max_seq_length: raise RuntimeError( - "Unsloth: Unfortunately Mistral type models do not support RoPE scaling!\n"\ + f"Unsloth: Unfortunately {model_patcher.__name__[4:-5]} type models do not support RoPE scaling!\n"\ f"The maximum sequence length supported is {model_max_seq_length}.", ) pass diff --git a/unsloth/models/qwen2.py b/unsloth/models/qwen2.py new file mode 100644 index 0000000000..76fe31a6d1 --- /dev/null +++ b/unsloth/models/qwen2.py @@ -0,0 +1,91 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .llama import * +from .mistral import FastMistralModel +import os +from ._utils import __version__ + +from transformers.models.qwen2.modeling_qwen2 import ( + Qwen2Attention, + Qwen2DecoderLayer, + Qwen2Model, + Qwen2ForCausalLM, +) +# For Pytorch 2.1.1 +try: + from transformers.models.qwen2.modeling_qwen2 import ( + Qwen2SdpaAttention, + Qwen2FlashAttention2, + ) +except: + Qwen2SdpaAttention = Qwen2Attention + Qwen2FlashAttention2 = Qwen2Attention +pass + + +class FastQwen2Model(FastLlamaModel): + + @staticmethod + def pre_patch(): + Qwen2Attention .forward = LlamaAttention_fast_forward + Qwen2SdpaAttention .forward = LlamaAttention_fast_forward + Qwen2FlashAttention2.forward = LlamaAttention_fast_forward + Qwen2DecoderLayer .forward = LlamaDecoderLayer_fast_forward + Qwen2Model .forward = LlamaModel_fast_forward + Qwen2ForCausalLM .forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference) + PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward + + # Solves https://github.com/unslothai/unsloth/issues/168 + # Static KV Cache was introduced in 4.38.0, causing training to be much slower. + # Inferene can now be CUDAGraphed, but we shall retain the old rotary embeddings. + # https://github.com/huggingface/transformers/pull/27931 + # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py + import transformers.models.qwen2.modeling_qwen2 + transformers.models.qwen2.modeling_qwen2.Qwen2RotaryEmbedding = LlamaRotaryEmbedding + return + pass + + + @staticmethod + def from_pretrained( + model_name = "Qwen/Qwen1.5-7B", + max_seq_length = 4096, + dtype = None, + load_in_4bit = True, + token = None, + device_map = "sequential", + rope_scaling = None, # Qwen2 does not support RoPE scaling + fix_tokenizer = True, + model_patcher = None, + tokenizer_name = None, + trust_remote_code = False, + **kwargs, + ): + return FastMistralModel.from_pretrained( + model_name = model_name, + max_seq_length = max_seq_length, + dtype = dtype, + load_in_4bit = load_in_4bit, + token = token, + device_map = device_map, + rope_scaling = rope_scaling, + fix_tokenizer = fix_tokenizer, + model_patcher = FastQwen2Model, + tokenizer_name = tokenizer_name, + trust_remote_code = trust_remote_code, + **kwargs, + ) + pass +pass diff --git a/unsloth/save.py b/unsloth/save.py index b825b10fb7..39b18d0dd9 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -27,6 +27,7 @@ import subprocess import psutil import re from transformers.models.llama.modeling_llama import logger +from .tokenizer_utils import fix_sentencepiece_gguf __all__ = [ "print_quantization_methods", @@ -774,7 +775,7 @@ def install_llama_cpp_old(version = -10): f"make all -j{psutil.cpu_count()*2} -C llama.cpp", ] for command in commands: - with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, bufsize = 1) as sp: + with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1) as sp: for line in sp.stdout: print(line.decode("utf-8", errors = "replace"), flush = True, end = "") pass @@ -806,7 +807,7 @@ def install_llama_cpp_blocking(use_cuda = True): if os.path.exists("llama.cpp"): return for command in commands: - with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, bufsize = 1) as sp: + with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1) as sp: for line in sp.stdout: print(line.decode("utf-8", errors = "replace"), flush = True, end = "") pass @@ -865,11 +866,11 @@ def save_to_gguf( first_conversion : str = "f16", _run_installer = None, # Non blocking install of llama.cpp ): - logger.warning( - "NOTICE: llama.cpp GGUF conversion is currently unstable, since llama.cpp is\n"\ - "undergoing some major bug fixes as at 5th of May 2024. This is not an Unsloth issue.\n"\ - "Please be patient - GGUF saving should still work, but might not work as well." - ) + # logger.warning( + # "NOTICE: llama.cpp GGUF conversion is currently unstable, since llama.cpp is\n"\ + # "undergoing some major bug fixes as at 5th of May 2024. This is not an Unsloth issue.\n"\ + # "Please be patient - GGUF saving should still work, but might not work as well." + # ) if quantization_method.startswith("iq2"): raise RuntimeError("Unsloth: Currently iq2 type quantizations aren't supported yet - sorry!") @@ -962,6 +963,8 @@ def save_to_gguf( # We first check if tokenizer.model exists in the model_directory if os.path.exists(f"{model_directory}/tokenizer.model"): vocab_type = "spm,hfft,bpe" + # Fix Sentencepiece model as well! + fix_sentencepiece_gguf(model_directory) else: vocab_type = "bpe" pass @@ -969,7 +972,7 @@ def save_to_gguf( if use_fast_convert: command = f"python llama.cpp/convert.py {model_directory} "\ f"--outfile {final_location} --vocab-type {vocab_type} "\ - f"--outtype {first_conversion} --concurrency {n_cpus}" + f"--outtype {first_conversion} --concurrency {n_cpus} --pad-vocab" else: # Need to fix convert-hf-to-gguf.py for some models! # _fix_gemma_gguf() @@ -979,7 +982,7 @@ def save_to_gguf( f"--outtype {first_conversion}" pass - with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 1) as sp: + with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1) as sp: for line in sp.stdout: print(line.decode("utf-8", errors = "replace"), flush = True, end = "") if sp.returncode is not None and sp.returncode != 0: @@ -1020,8 +1023,8 @@ def save_to_gguf( f"{final_location} {quantization_method} {n_cpus}" # quantize uses stderr - with subprocess.Popen(command, shell = True, stderr = subprocess.PIPE, bufsize = 1) as sp: - for line in sp.stderr: + with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1) as sp: + for line in sp.stdout: print(line.decode("utf-8", errors = "replace"), flush = True, end = "") if sp.returncode is not None and sp.returncode != 0: raise subprocess.CalledProcessError(sp.returncode, sp.args) @@ -1073,7 +1076,7 @@ def unsloth_save_pretrained_merged( save_peft_format : bool = True, tags : List[str] = None, temporary_location : str = "_unsloth_temporary_saved_buffers", - maximum_memory_usage : float = 0.85, + maximum_memory_usage : float = 0.75, ): """ Same as .save_pretrained(...) except 4bit weights are auto @@ -1116,7 +1119,7 @@ def unsloth_push_to_hub_merged( commit_description : str = "Upload model trained with Unsloth 2x faster", tags : Optional[List[str]] = None, temporary_location : str = "_unsloth_temporary_saved_buffers", - maximum_memory_usage : float = 0.85, + maximum_memory_usage : float = 0.75, ): """ Same as .push_to_hub(...) except 4bit weights are auto diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 0d6dadf7dd..87cba843d4 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -26,6 +26,7 @@ __all__ = [ "fix_sentencepiece_tokenizer", "check_tokenizer", "add_new_tokens", + "fix_sentencepiece_gguf", ] @@ -267,6 +268,71 @@ def fix_sentencepiece_tokenizer( pass +def fix_sentencepiece_gguf(saved_location): + """ + Fixes sentencepiece tokenizers which did not extend the vocabulary with + user defined tokens. + Inspiration from https://github.com/ggerganov/llama.cpp/blob/master/convert-hf-to-gguf.py + """ + import numpy as np + from copy import deepcopy + from transformers.utils import sentencepiece_model_pb2 + import json + from enum import IntEnum + import os + + class SentencePieceTokenTypes(IntEnum): + NORMAL = 1 + UNKNOWN = 2 + CONTROL = 3 + USER_DEFINED = 4 + UNUSED = 5 + BYTE = 6 + pass + + # Load tokenizer.model + tokenizer_file = sentencepiece_model_pb2.ModelProto() + if not os.path.isfile(f"{saved_location}/tokenizer.model"): return + tokenizer_file.ParseFromString(open(f"{saved_location}/tokenizer.model", "rb").read()) + sentence_piece_size = len(tokenizer_file.pieces) + + # Load added_tokens_json + if not os.path.isfile(f"{saved_location}/added_tokens.json"): return + with open(f"{saved_location}/added_tokens.json", "r", encoding = "utf-8") as file: + added_tokens_json = json.load(file) + pass + if len(added_tokens_json) == 0: return + + added_tokens_json = dict(sorted(added_tokens_json.items(), key = lambda item: item[1])) + + # Confirm added_tokens_json is correct + added_tokens_ids = np.array(list(added_tokens_json.values())) + diff = np.diff(added_tokens_ids) + if (diff.min() != 1 or diff.max() != 1): return + if (added_tokens_ids.min() != sentence_piece_size): return + + # Edit sentence piece tokens with added_tokens_json + logger.warning("Unsloth: Extending tokenizer.model with added_tokens.json!") + new_tokens = deepcopy(tokenizer_file.pieces[-len(added_tokens_ids):]) + for new_token, added_token in zip(new_tokens, added_tokens_json.keys()): + new_token.piece = added_token.encode("utf-8") + new_token.score = -1000.0 + new_token.type = SentencePieceTokenTypes.USER_DEFINED + pass + + tokenizer_file.pieces.extend(new_tokens) + + with open(f"{saved_location}/tokenizer.model", "wb") as file: + file.write(tokenizer_file.SerializeToString()) + pass + + # Add padding tokens + # actual_vocab_size = model.config.vocab_size + # padding = actual_vocab_size - len(tokenizer_file.pieces) + return +pass + + def load_correct_tokenizer( tokenizer_name, model_max_length = None,