diff --git a/README.md b/README.md index 8a6c970fb0..66f4c55646 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,8 @@ pip install --upgrade --force-reinstall --no-cache-dir torch==2.1.0 triton \ ```bash pip install "unsloth[cu118] @ git+https://github.com/unslothai/unsloth.git" pip install "unsloth[cu121] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu118_ampere] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu121_ampere] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu118ampere] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu121ampere] @ git+https://github.com/unslothai/unsloth.git" ``` 3. For Pytorch 2.1.1: Use the `"ampere"` path for newer RTX 30xx GPUs or higher. ```bash @@ -119,10 +119,10 @@ pip install --upgrade --force-reinstall --no-cache-dir torch==2.1.1 triton \ --index-url https://download.pytorch.org/whl/cu121 ``` ```bash -pip install "unsloth[cu118_torch211] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu121_torch211] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu118_ampere_torch211] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu121_ampere_torch211] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu118torch211] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu121torch211] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu118amperetorch211] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu121amperetorch211] @ git+https://github.com/unslothai/unsloth.git" ``` 4. For Pytorch 2.2.0: Use the `"ampere"` path for newer RTX 30xx GPUs or higher. ```bash @@ -130,10 +130,10 @@ pip install --upgrade --force-reinstall --no-cache-dir torch==2.2.0 triton \ --index-url https://download.pytorch.org/whl/cu121 ``` ```bash -pip install "unsloth[cu118_torch220] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu121_torch220] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu118_ampere_torch220] @ git+https://github.com/unslothai/unsloth.git" -pip install "unsloth[cu121_ampere_torch220] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu118torch220] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu121torch220] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu118amperetorch220] @ git+https://github.com/unslothai/unsloth.git" +pip install "unsloth[cu121amperetorch220] @ git+https://github.com/unslothai/unsloth.git" ``` 5. If you get errors, try the below first, then go back to step 1: ```bash diff --git a/unsloth/__init__.py b/unsloth/__init__.py index d052b333f2..b290b690bd 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -64,7 +64,7 @@ try: libcuda_dirs() except: warnings.warn( - "Running `ldconfig /usr/lib64-nvidia` to link CUDA."\ + "Unsloth: Running `ldconfig /usr/lib64-nvidia` to link CUDA."\ ) os.system("ldconfig /usr/lib64-nvidia") importlib.reload(bnb) @@ -75,7 +75,7 @@ except: cdequantize_blockwise_fp32 = bnb.functional.lib.cdequantize_blockwise_fp32 libcuda_dirs() except: - raise ImportError("CUDA is not linked properly.\n"\ + raise ImportError("Unsloth: CUDA is not linked properly.\n"\ "We tried running `ldconfig /usr/lib64-nvidia` ourselves, but it didn't work.\n"\ "You need to run in your terminal `ldconfig /usr/lib64-nvidia` yourself, then import Unsloth.") pass diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 8604298734..e5b8ec984e 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -55,6 +55,7 @@ from peft import PeftModelForCausalLM from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit from peft.tuners.lora import Linear4bit as Peft_Linear4bit from ..save import patch_saving_functions +import re, os, inspect, math, sys def original_apply_qkv(self, X): @@ -782,32 +783,6 @@ pass # https://github.com/huggingface/transformers/pull/27931 # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py class LlamaRotaryEmbedding(torch.nn.Module): - # def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): - # super().__init__() - - # self.dim = dim - # self.max_position_embeddings = max_position_embeddings - # self.base = base - # inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) - # self.register_buffer("inv_freq", inv_freq, persistent=False) - - # # Build here to make `torch.jit.trace` work. - # self._set_cos_sin_cache( - # seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() - # ) - # pass - - # def _set_cos_sin_cache(self, seq_len, device, dtype): - # self.max_seq_len_cached = seq_len - # t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - - # freqs = torch.outer(t, self.inv_freq) - # # Different from paper, but it uses a different permutation in order to obtain the same calculation - # emb = torch.cat((freqs, freqs), dim=-1) - # self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - # self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - # pass - # Fixes https://github.com/huggingface/transformers/pull/28837 # https://github.com/microsoft/DeepSpeed/issues/4932 # The precision of RoPE buffers is not correct, so we cast to int64. @@ -852,24 +827,6 @@ pass class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding): """LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev""" - - # def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): - # self.scaling_factor = scaling_factor - # super().__init__(dim, max_position_embeddings, base, device) - # pass - - # def _set_cos_sin_cache(self, seq_len, device, dtype): - # self.max_seq_len_cached = seq_len - # t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - # t = t / self.scaling_factor - - # freqs = torch.outer(t, self.inv_freq) - # # Different from paper, but it uses a different permutation in order to obtain the same calculation - # emb = torch.cat((freqs, freqs), dim=-1) - # self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - # self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - # pass - # Fixes https://github.com/huggingface/transformers/pull/28837 # https://github.com/microsoft/DeepSpeed/issues/4932 # The precision of RoPE buffers is not correct, so we cast to int64. @@ -1006,6 +963,112 @@ class FastLlamaModel: layer.self_attn.apply_o = original_apply_o pass + # Patch Trainer + from transformers.trainer import Trainer + inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + + import transformers.trainer + items_in_trainer = dir(transformers.trainer) + good_items = [] + for item in items_in_trainer: + # TODO: Support Deepspeed + if item.startswith(("deepspeed", "xm", "met", "smp")): continue + if item in inner_training_loop: good_items.append(item) + pass + exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")") + + start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0] + end = inner_training_loop.find("\n\n", start) + original_debug = inner_training_loop[start:end] + spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:] + front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0) + + debug_info = """debug_info = \\ + f"==((====))== Unsloth - Free Apache OSS license | Num GPUs = {args.world_size}\\n"\\ + f" \\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\ + f"O^O/ \_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\ + f"\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\ + f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}' + logger.warning_once(debug_info)""" + + debug_info = debug_info.split('\n') + debug_info = "\n".join([debug_info[0]] + [spaces + x for x in debug_info[1:]]) + inner_training_loop = inner_training_loop.replace(original_debug, debug_info) + + debug_info = """n_total_devices = total_train_batch_size // \\ + args.gradient_accumulation_steps // self._train_batch_size + if n_total_devices > 2+2: + logger.warning_once( + "Our OSS was designed for people with few GPU resources to level the playing field.\\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + debug_info =""" + debug_info = debug_info.split('\n') + debug_info = "\n".join([debug_info[0]] + [spaces + x for x in debug_info[1:]]) + inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1) + + front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0) + inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags = re.MULTILINE) + inner_training_loop = inner_training_loop.replace( + "train_dataloader = tpu_spmd_dataloader(train_dataloader)", + "raise RuntimeError('Unsloth: TPUs are not yet supported!')" + ) + inner_training_loop = inner_training_loop.replace( + "self.accelerator.free_memory()", + "self.accelerator.free_memory()\n" + \ + front_spaces + "if self.is_deepspeed_enabled:"\ + "raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1, + ) + + check_batches = """train_dataloader = self.get_train_dataloader() + ga = args.gradient_accumulation_steps + bsz = self._train_batch_size + total_batches = bsz * ga * args.world_size + n_total_devices = total_batches // ga // bsz + if n_total_devices > 2 * 2: + logger.warning_once( + "Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n" + "The OSS currently works on four GPUs - we're a 2 person team, so please help fund\\n" + "our development costs by supporting us through Ko-fi or buying a license! Thanks!", + ) + divisor = n_total_devices / 2 / 2 + bsz = self._train_batch_size = max(int(bsz / divisor), 1) + if total_batches // ga // bsz > 2 / 2: + divisor = n_total_devices / 2 / 2 + ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)""" + check_batches = check_batches.split('\n') + check_batches = "\n".join([check_batches[0]] + [front_spaces + x for x in check_batches[1:]]) + inner_training_loop = inner_training_loop.replace( + "train_dataloader = self.get_train_dataloader()", + check_batches, 1, + ) + inner_training_loop = inner_training_loop.replace( + "_inner_training_loop", + "_fast_inner_training_loop", 1, + ) + exec(inner_training_loop, globals()) + + Trainer._inner_training_loop = _fast_inner_training_loop + inner_training_loop = inner_training_loop.replace( + "is_torch_tpu_available()", + "False", + ) + if "n_total_devices >" not in inner_training_loop: + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + inner_training_loop = inner_training_loop.replace( + "is_sagemaker_mp_enabled()", + "False", + ) + Trainer._inner_training_loop = _fast_inner_training_loop + # Save max_seq_length model.max_seq_length = max_position_embeddings internal_model = model @@ -1126,6 +1189,16 @@ class FastLlamaModel: SUPPORTS_LOFTQ = "loftq_config" in signature SUPPORTS_RSLORA = "use_rslora" in signature + from transformers.trainer import Trainer + if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + assert(max_seq_length <= model.max_seq_length) if lora_dropout != 0: @@ -1252,6 +1325,18 @@ class FastLlamaModel: model.peft_config[active_adapter].revision = f"unsloth" pass + # Fix loftq issues + # loftq_config must not = None, but rather {} + all_configs = model.peft_config + for key, current_config in all_configs.items(): + if hasattr(current_config, "loftq_config") and current_config.loftq_config is None: + new_args = current_config.__dict__ + new_args["loftq_config"] = {} + current_config = current_config.__class__(**new_args) + all_configs[key] = current_config + pass + pass + # Do patching n_mlp = 0 n_qkv = 0 diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 615e43643d..08c97998a6 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -368,6 +368,112 @@ class FastMistralModel(FastLlamaModel): layer.self_attn.apply_o = original_apply_o pass + # Patch Trainer + from transformers.trainer import Trainer + inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + + import transformers.trainer + items_in_trainer = dir(transformers.trainer) + good_items = [] + for item in items_in_trainer: + # TODO: Support Deepspeed + if item.startswith(("deepspeed", "xm", "met", "smp")): continue + if item in inner_training_loop: good_items.append(item) + pass + exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")") + + start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0] + end = inner_training_loop.find("\n\n", start) + original_debug = inner_training_loop[start:end] + spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:] + front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0) + + debug_info = """debug_info = \\ + f"==((====))== Unsloth - Free Apache OSS license | Num GPUs = {args.world_size}\\n"\\ + f" \\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\ + f"O^O/ \_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\ + f"\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\ + f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}' + logger.warning_once(debug_info)""" + + debug_info = debug_info.split('\n') + debug_info = "\n".join([debug_info[0]] + [spaces + x for x in debug_info[1:]]) + inner_training_loop = inner_training_loop.replace(original_debug, debug_info) + + debug_info = """n_total_devices = total_train_batch_size // \\ + args.gradient_accumulation_steps // self._train_batch_size + if n_total_devices > 2+2: + logger.warning_once( + "Our OSS was designed for people with few GPU resources to level the playing field.\\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + debug_info =""" + debug_info = debug_info.split('\n') + debug_info = "\n".join([debug_info[0]] + [spaces + x for x in debug_info[1:]]) + inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1) + + front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0) + inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags = re.MULTILINE) + inner_training_loop = inner_training_loop.replace( + "train_dataloader = tpu_spmd_dataloader(train_dataloader)", + "raise RuntimeError('Unsloth: TPUs are not yet supported!')" + ) + inner_training_loop = inner_training_loop.replace( + "self.accelerator.free_memory()", + "self.accelerator.free_memory()\n" + \ + front_spaces + "if self.is_deepspeed_enabled:"\ + "raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1, + ) + + check_batches = """train_dataloader = self.get_train_dataloader() + ga = args.gradient_accumulation_steps + bsz = self._train_batch_size + total_batches = bsz * ga * args.world_size + n_total_devices = total_batches // ga // bsz + if n_total_devices > 2 * 2: + logger.warning_once( + "Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n" + "The OSS currently works on four GPUs - we're a 2 person team, so please help fund\\n" + "our development costs by supporting us through Ko-fi or buying a license! Thanks!", + ) + divisor = n_total_devices / 2 / 2 + bsz = self._train_batch_size = max(int(bsz / divisor), 1) + if total_batches // ga // bsz > 2 / 2: + divisor = n_total_devices / 2 / 2 + ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)""" + check_batches = check_batches.split('\n') + check_batches = "\n".join([check_batches[0]] + [front_spaces + x for x in check_batches[1:]]) + inner_training_loop = inner_training_loop.replace( + "train_dataloader = self.get_train_dataloader()", + check_batches, 1, + ) + inner_training_loop = inner_training_loop.replace( + "_inner_training_loop", + "_fast_inner_training_loop", 1, + ) + exec(inner_training_loop, globals()) + + Trainer._inner_training_loop = _fast_inner_training_loop + inner_training_loop = inner_training_loop.replace( + "is_torch_tpu_available()", + "False", + ) + if "n_total_devices >" not in inner_training_loop: + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + inner_training_loop = inner_training_loop.replace( + "is_sagemaker_mp_enabled()", + "False", + ) + Trainer._inner_training_loop = _fast_inner_training_loop + # Save max_seq_length max_position_embeddings = max(max_seq_length, model.config.max_position_embeddings) model.max_seq_length = max_position_embeddings