From 0f1e607d938ca4c6be027277342c58d51a322294 Mon Sep 17 00:00:00 2001 From: Yang JianXin <995462226@qq.com> Date: Sat, 4 May 2024 11:37:01 +0800 Subject: [PATCH] support Qwen2 --- unsloth/models/llama.py | 1 + unsloth/models/loader.py | 3 + unsloth/models/qwen2.py | 342 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 346 insertions(+) create mode 100644 unsloth/models/qwen2.py diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index a7cacea9b9..7dd2ba98ec 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1599,6 +1599,7 @@ class FastLlamaModel: if model_type == "llama": apply_lora_mlp = apply_lora_mlp_swiglu elif model_type == "mistral": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "qwen2": apply_lora_mlp = apply_lora_mlp_swiglu elif model_type == "gemma": apply_lora_mlp = apply_lora_mlp_geglu_approx else: raise NotImplementedError(f"Unsloth: {model_type} is not yet implemented!") diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index a107200eae..dc90c83584 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -14,6 +14,7 @@ from .llama import FastLlamaModel, logger from .mistral import FastMistralModel +from .qwen2 import FastQwen2Model from transformers import AutoConfig from transformers import __version__ as transformers_version from peft import PeftConfig, PeftModel @@ -119,6 +120,8 @@ class FastLanguageModel(FastLlamaModel): f"to obtain the latest transformers build, then restart this session."\ ) dispatch_model = FastGemmaModel + elif model_type == 'qwen2': + dispatch_model = FastQwen2Model else: raise NotImplementedError( f"Unsloth: {model_name} not supported yet!\n"\ diff --git a/unsloth/models/qwen2.py b/unsloth/models/qwen2.py new file mode 100644 index 0000000000..5b6919f910 --- /dev/null +++ b/unsloth/models/qwen2.py @@ -0,0 +1,342 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .llama import * +import os +from ._utils import __version__ + +from transformers.models.qwen2.modeling_qwen2 import ( + Qwen2Attention, + Qwen2DecoderLayer, + Qwen2Model, + Qwen2ForCausalLM +) + +# For Pytorch 2.1.1 +try: + from transformers.models.qwen2.modeling_qwen2 import ( + Qwen2SdpaAttention, + Qwen2FlashAttention2, + ) +except: + Qwen2SdpaAttention = Qwen2Attention + Qwen2FlashAttention2 = Qwen2Attention + + +class FastQwen2Model(FastLlamaModel): + + @staticmethod + def pre_patch(): + Qwen2Attention.forward = LlamaAttention_fast_forward + Qwen2SdpaAttention.forward = LlamaAttention_fast_forward + Qwen2FlashAttention2.forward = LlamaAttention_fast_forward + Qwen2DecoderLayer.forward = LlamaDecoderLayer_fast_forward + Qwen2Model.forward = LlamaModel_fast_forward + Qwen2ForCausalLM.forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference) + PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward + + # Solves https://github.com/unslothai/unsloth/issues/168 + # Static KV Cache was introduced in 4.38.0, causing training to be much slower. + # Inferene can now be CUDAGraphed, but we shall retain the old rotary embeddings. + # https://github.com/huggingface/transformers/pull/27931 + # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py + import transformers.models.qwen2.modeling_qwen2 + transformers.models.qwen2.modeling_qwen2.Qwen2RotaryEmbedding = LlamaRotaryEmbedding + return + + pass + + @staticmethod + def from_pretrained( + model_name="Qwen/Qwen1.5-7B", + max_seq_length=4096, + dtype=None, + load_in_4bit=True, + token=None, + device_map="sequential", + rope_scaling=None, # Qwen2 does not support RoPE scaling + fix_tokenizer=True, + model_patcher=None, + tokenizer_name=None, + trust_remote_code=False, + **kwargs, + ): + if token is None and "HF_TOKEN" in os.environ: + token = os.environ["HF_TOKEN"] + + if token is None and "HUGGINGFACE_TOKEN" in os.environ: + token = os.environ["HUGGINGFACE_TOKEN"] + + if model_patcher is None: model_patcher = FastQwen2Model + # Qwen2 does NOT support RoPE Scaling! + if rope_scaling is not None: + logger.warning_once("Unsloth: Qwen2 models do not support RoPE scaling.") + pass + + SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported() + gpu_stats = torch.cuda.get_device_properties(0) + max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) + + statistics = \ + f"==((====))== Unsloth: Fast {model_patcher.__name__[4:-5]} patching release {__version__}\n" \ + f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n" \ + f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n" \ + f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n" \ + f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' + print(statistics) + model_patcher.pre_patch() + # get_statistics() + + if dtype is None: + dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 + elif dtype == torch.bfloat16 and not SUPPORTS_BFLOAT16: + logger.warning_once("Device does not support bfloat16. Will change to float16.") + dtype = torch.float16 + + assert (dtype == torch.float16 or dtype == torch.bfloat16 or dtype == torch.float32) + + # Check max sequence length + model_config = AutoConfig.from_pretrained(model_name, token=token) + model_max_seq_length = model_config.max_position_embeddings + + # Qwen2 does NOT support RoPE Scaling sadly so we have to error out. + if max_seq_length > model_max_seq_length: + raise RuntimeError( + "Unsloth: Unfortunately Qwen2 type models do not support RoPE scaling!\n" \ + f"The maximum sequence length supported is {model_max_seq_length}.", + ) + pass + + bnb_config = None + if load_in_4bit: + bnb_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=dtype, + ) + + max_position_embeddings = max(max_seq_length, model_max_seq_length) + model = AutoModelForCausalLM.from_pretrained( + model_name, + device_map=device_map, + torch_dtype=dtype, + quantization_config=bnb_config, + token=token, + # rope_scaling = rope_scaling, + trust_remote_code=trust_remote_code, + **kwargs, + ) + + # Counteract saved tokenizers + tokenizer_name = model_name if tokenizer_name is None else tokenizer_name + tokenizer = load_correct_tokenizer( + tokenizer_name, + model_max_length=max_position_embeddings, + padding_side="right", + token=token, + trust_remote_code=trust_remote_code, + ) + + model, tokenizer = patch_tokenizer(model, tokenizer) + model = model_patcher.post_patch(model) + + # Patch up QKV / O and MLP + for idx, layer in enumerate(model.model.layers): + layer.self_attn.apply_qkv = original_apply_qkv + layer.self_attn.apply_o = original_apply_o + pass + + # Patch Trainer + from transformers.trainer import Trainer + if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": + try: + inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + except: + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + pass + + # Patch Trainer + from transformers.trainer import Trainer + try: + if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": + inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + Trainer._original_training_loop = inner_training_loop + else: + inner_training_loop = Trainer._original_training_loop + except: + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + + import transformers.trainer + items_in_trainer = dir(transformers.trainer) + good_items = [] + for item in items_in_trainer: + # TODO: Support Deepspeed + if item.startswith(("deepspeed", "xm", "met", "smp")): continue + if item in inner_training_loop: good_items.append(item) + pass + exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")", globals()) + + start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0] + end = inner_training_loop.find("\n\n", start) + original_debug = inner_training_loop[start:end] + spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:] + front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0) + + debug_info = """debug_info = \\ + f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs = {args.world_size}\\n"\\ + f" \\\\\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\ + f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\ + f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\ + f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}' + logger.warning_once(debug_info)""" + + debug_info = debug_info.split('\n') + debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]]) + inner_training_loop = inner_training_loop.replace(original_debug, debug_info) + + debug_info = """n_total_devices = total_train_batch_size // \\ + args.gradient_accumulation_steps // self._train_batch_size + if n_total_devices > 2: + logger.warning_once( + "Our OSS was designed for people with few GPU resources to level the playing field.\\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + debug_info =""" + debug_info = debug_info.split('\n') + debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]]) + inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1) + + front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0) + inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags=re.MULTILINE) + inner_training_loop = inner_training_loop.replace( + "train_dataloader = tpu_spmd_dataloader(train_dataloader)", + "raise RuntimeError('Unsloth: TPUs are not yet supported!')" + ) + inner_training_loop = inner_training_loop.replace( + "self.accelerator.free_memory()", + "self.accelerator.free_memory()\n" + \ + front_spaces + "if self.is_deepspeed_enabled:" \ + "raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1, + ) + + check_batches = """train_dataloader = self.get_train_dataloader() + ga = args.gradient_accumulation_steps + bsz = self._train_batch_size + total_batches = bsz * ga * args.world_size + n_total_devices = total_batches // ga // bsz + if n_total_devices > 2: + logger.warning_once( + "Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n" + "The OSS currently works on 4 GPUs - we're a 2 person team, so please help fund\\n" + "our development costs by supporting us through Ko-fi or buying a license! Thanks!", + ) + divisor = n_total_devices / 2 + bsz = self._train_batch_size = max(int(bsz / divisor), 1) + if total_batches // ga // bsz > 2: + divisor = n_total_devices / 2 + ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)""" + check_batches = check_batches.split('\n') + check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]]) + inner_training_loop = inner_training_loop.replace( + "train_dataloader = self.get_train_dataloader()", + check_batches, 1, + ) + inner_training_loop = inner_training_loop.replace( + "_inner_training_loop", + "_fast_inner_training_loop", 1, + ) + exec(inner_training_loop, globals()) + + Trainer._inner_training_loop = _fast_inner_training_loop + inner_training_loop = inner_training_loop.replace( + "is_torch_tpu_available()", + "False", + ) + if "n_total_devices >" not in inner_training_loop: + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + inner_training_loop = inner_training_loop.replace( + "is_sagemaker_mp_enabled()", + "False", + ) + Trainer._inner_training_loop = _fast_inner_training_loop + + # Save max_seq_length + max_position_embeddings = max(max_seq_length, model.config.max_position_embeddings) + model.max_seq_length = max_position_embeddings + internal_model = model + while hasattr(internal_model, "model"): + internal_model.max_seq_length = max_position_embeddings + internal_model = internal_model.model + pass + internal_model.max_seq_length = max_position_embeddings + + # We check the tokenizer first for errors + if fix_tokenizer: + tokenizer = check_tokenizer( + model=model, + tokenizer=tokenizer, + model_name=model_name, + model_max_length=max_position_embeddings, + padding_side="right", + token=token, + ) + pass + patch_saving_functions(tokenizer) + + # Fix up config for transformers uploading PEFT + # Not necessary anymore since we require transformers>=4.37 + if False: + name = model.config._name_or_path + if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): + name = name[:len(name) - len("-bnb-4bit")] + model.config.update({"_name_or_path": name}) + pass + + # Log Unsloth version for future fastpaths for inference + model.config.update({"unsloth_version": __version__}) + + # Add save modules + patch_saving_functions(model) + + # Save tokenizer for inference purposes + tokenizer.padding_side = "left" # Force inference + internal_model = model + while hasattr(internal_model, "model"): + internal_model._saved_temp_tokenizer = tokenizer + internal_model = internal_model.model + pass + internal_model._saved_temp_tokenizer = tokenizer + + return model, tokenizer \ No newline at end of file