From 9f9739cbace7f8d730c905319c96f9a268634aad Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sun, 28 Jan 2024 03:49:54 +1100 Subject: [PATCH] attention_mask --- unsloth/models/llama.py | 3 ++- unsloth/save.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index d5dd7833d5..852388e4f8 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -487,9 +487,10 @@ def LlamaModel_fast_forward( inputs_embeds = self.embed_tokens(input_ids) # Ignore attention_mask + print(attention_mask) if attention_mask is None: padding_mask = None - elif self.training: + elif False:#self.training: attention_mask = None padding_mask = None else: diff --git a/unsloth/save.py b/unsloth/save.py index 6c44d23cb0..471897c0ad 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -14,6 +14,7 @@ from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit from peft.tuners.lora import Linear4bit as Peft_Linear4bit +from peft.tuners.lora import Linear as Peft_Linear from typing import Optional, Callable, Union, List import torch import os @@ -72,11 +73,15 @@ pass def _merge_lora(layer, name): - if isinstance(layer, (Bnb_Linear4bit, Peft_Linear4bit)): + + if isinstance(layer, (Bnb_Linear4bit, Peft_Linear4bit, Peft_Linear)): # Is LoRA so we need to merge! W, quant_state, A, B, s = get_lora_parameters(layer) - dtype = quant_state.dtype if type(quant_state) is not list else quant_state[2] - W = fast_dequantize(W, quant_state).to(torch.float32).t() + if quant_state is not None: + dtype = quant_state.dtype if type(quant_state) is not list else quant_state[2] + W = fast_dequantize(W, quant_state) + pass + W = W.to(torch.float32).t() if A is not None: sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32))) @@ -84,7 +89,6 @@ def _merge_lora(layer, name): if not torch.isfinite(W).all(): raise ValueError(f"Unsloth: Merge failed.\n{name} has some elements = infinity.") pass - W = W.t().to(dtype) else: W = layer.weight