From 25619645dd02a055d7855e41e402215ed0af3b0b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 29 Feb 2024 00:15:22 +1100 Subject: [PATCH] Hotfix - fix DoRA, Gemma prompt template (#202) * Update save.py * saving * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update __init__.py * Update save.py * Update save.py * Update save.py * save * trainer * spaces * original * Gemma * Update pyproject.toml * Update mapper.py * Update fast_lora.py * FastGemmaModel * model_type * Update llama.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update fast_lora.py * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * gemma * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update fast_lora.py * Update fast_lora.py * Fast CE Loss * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * CE * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update geglu.py * Update cross_entropy_loss.py * revert * Update llama.py * Update llama.py * norm * Update gemma.py * Update gemma.py * position_ids * Update gemma.py * Update gemma.py * pos * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * revert * revert * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * rope * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * llama * Update llama.py * gemma * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update save.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update gemma.py * correct_dtype * Update gemma.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Chat Templates * Update README.md * Update README.md * Update llama.py * DoRA * Update _utils.py * Update chat_templates.py --- unsloth/chat_templates.py | 3 +- unsloth/models/_utils.py | 83 ++++++++++++--------------------------- 2 files changed, 28 insertions(+), 58 deletions(-) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 6f0237c27b..5ad34d6882 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -221,6 +221,7 @@ CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token,) # Notice we must use |trim for lstrip and rstrip. maps to 106. # maps to 107. user and model are normal 1 word tokens. gemma_template = \ + "{{ bos_token }}"\ "{% for message in messages %}"\ "{% if message['role'] == 'user' %}"\ "{{'user\n' + message['content'] | trim + '\n'}}"\ @@ -238,7 +239,7 @@ CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token,) # Gemma with ChatML instead -gemma_chatml_template = chatml_template +gemma_chatml_template = "{{ bos_token }}" + chatml_template gemma_chatml_eos_token = ( {"" : "<|im_start|>", "" : "<|im_end|>"}, "<|im_end|>", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c4de7198b9..e92be5e993 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -240,61 +240,30 @@ pass # Weirdly LoraLayer.update_layer downcasts PEFT layers to float16?? # For mixed precision, we need it to be in float32 not float16. -def LoraLayer_update_layer(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights, - use_rslora = False): - # This code works for linear layers, override for other layer types - if r <= 0: - raise ValueError(f"`r` should be a positive integer value but the value passed is {r}") - - self.r[adapter_name] = r - self.lora_alpha[adapter_name] = lora_alpha - if lora_dropout > 0.0: - lora_dropout_layer = torch.nn.Dropout(p=lora_dropout) - else: - lora_dropout_layer = torch.nn.Identity() - - self.lora_dropout.update(torch.nn.ModuleDict({adapter_name: lora_dropout_layer})) - # Actual trainable parameters - self.lora_A[adapter_name] = torch.nn.Linear(self.in_features, r, bias=False) - self.lora_B[adapter_name] = torch.nn.Linear(r, self.out_features, bias=False) - if use_rslora: - self.scaling[adapter_name] = lora_alpha / math.sqrt(r) - else: - self.scaling[adapter_name] = lora_alpha / r - - if init_lora_weights == "loftq": - # We manually check for PEFT - if not hasattr(self, "loftq_init"): - import peft - raise RuntimeError( - f"Unsloth: Your PEFT version of {peft.__version__} does not support LoftQ init.\n"\ - "Please install PEFT 0.7.2 or higher.\n"\ - "You can also install from source: `pip install git+https://github.com/huggingface/peft.git" - ) - pass - self.loftq_init(adapter_name) - - elif init_lora_weights: - self.reset_lora_parameters(adapter_name, init_lora_weights) - - # check weight and qweight (for GPTQ) - for weight_name in ("weight", "qweight"): - weight = getattr(self.get_base_layer(), weight_name, None) - if weight is not None: - # [INCORRECT code] - # - # the layer is already completely initialized, this is an update - # if weight.dtype.is_floating_point or weight.dtype.is_complex: - # self.to(weight.device, dtype=weight.dtype) - # else: - # self.to(weight.device) - self.to(weight.device, non_blocking = True) - break - self.set_adapter(self.active_adapters) -pass - -# Fix up incorrect downcasting of LoRA weights from peft.tuners.lora.layer import LoraLayer -LoraLayer.update_layer = LoraLayer_update_layer -from peft.tuners.lora import LoraLayer -LoraLayer.update_layer = LoraLayer_update_layer +import inspect, re +try: + source = inspect.getsource(LoraLayer.update_layer) + text = "if weight is not None:\n" + start = source.find(text) + len(text) + end = source.find("self.to(weight.device)", start) + spaces = re.findall(r"^([ ]{1,})break", source, flags = re.MULTILINE)[0] + source = source.replace(source[start : end], spaces) + spaces = len(re.match(r"[\s]{1,}", source).group(0)) + lines = source.split("\n") + source = "\n".join(x[spaces:] for x in lines) + source = re.sub("([^\.])nn\.", r"\1torch.nn.", source) + source = source.replace("def update_layer", "def LoraLayer_update_layer") + exec(source, globals()) + + # Fix up incorrect downcasting of LoRA weights + from peft.tuners.lora.layer import LoraLayer + LoraLayer.update_layer = LoraLayer_update_layer + from peft.tuners.lora import LoraLayer + LoraLayer.update_layer = LoraLayer_update_layer +except: + logger.warning_once( + "Unsloth unsuccessfully patched LoraLayer.update_layer. Please file a bug report.\n"\ + "Luckily, your training run will still work in the meantime!" + ) +pass