diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index ac3cc27068..b70e6e4fee 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -18,28 +18,27 @@ from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel def matmul_lora(X, W, W_quant, A, B, s, out = None): - with torch.cuda.amp.autocast(dtype = torch.float16): - dtype = X.dtype - W = fast_dequantize(W.t(), W_quant) + dtype = X.dtype + W = fast_dequantize(W.t(), W_quant) - if X.dim() == 3: - batch, seq_len, d = X.shape - X = X.view(-1, X.shape[-1]) - reshape = True - else: - reshape = False - pass + if X.dim() == 3: + batch, seq_len, d = X.shape + X = X.view(-1, X.shape[-1]) + reshape = True + else: + reshape = False + pass - out = torch.matmul(X, W, out = out) - if W_quant is not None: del W + out = torch.matmul(X, W, out = out) + if W_quant is not None: del W - if A is not None: - # LoRA is enabled - A, B = A.t(), B.t() - out += (X @ A) @ (s * B) - pass - - return out.view(batch, seq_len, -1) if reshape else out + if A is not None: + # LoRA is enabled + A, B = A.t(), B.t() + out += (X @ A.to(dtype)) @ (s * B.to(dtype)) + pass + + return out.view(batch, seq_len, -1) if reshape else out pass diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index da99ea4b68..a43e1e7170 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1047,66 +1047,66 @@ class FastLlamaModel: lora_dropout = model.peft_config[active_adapter].lora_dropout bias = model.peft_config[active_adapter].bias - if lora_dropout == 0 and bias == "none": - for idx, layer in enumerate(model.model.model.layers): + # if lora_dropout == 0 and bias == "none": + # for idx, layer in enumerate(model.model.model.layers): - # MLP patching - gate_proj = layer.mlp.gate_proj - up_proj = layer.mlp. up_proj - down_proj = layer.mlp.down_proj + # # MLP patching + # gate_proj = layer.mlp.gate_proj + # up_proj = layer.mlp. up_proj + # down_proj = layer.mlp.down_proj - if hasattr(gate_proj, "lora_A") and \ - hasattr( up_proj, "lora_A") and \ - hasattr(down_proj, "lora_A") and \ - (gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None and \ - ( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None and \ - (down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None: + # if hasattr(gate_proj, "lora_A") and \ + # hasattr( up_proj, "lora_A") and \ + # hasattr(down_proj, "lora_A") and \ + # (gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None and \ + # ( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None and \ + # (down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None: - # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module - layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp) - n_mlp += 1 - else: - logger.warning_once( - "Unsloth cannot patch MLP layers with our manual autograd engine since either LoRA adapters\n"\ - "are not enabled or a bias term (like in Qwen) is used." - ) - pass + # # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module + # layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp) + # n_mlp += 1 + # else: + # logger.warning_once( + # "Unsloth cannot patch MLP layers with our manual autograd engine since either LoRA adapters\n"\ + # "are not enabled or a bias term (like in Qwen) is used." + # ) + # pass - # QKV attention patching - q_proj = layer.self_attn.q_proj - k_proj = layer.self_attn.k_proj - v_proj = layer.self_attn.v_proj - if hasattr(q_proj, "lora_A") and \ - hasattr(k_proj, "lora_A") and \ - hasattr(v_proj, "lora_A") and \ - (q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None and \ - (k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None and \ - (v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None: + # # QKV attention patching + # q_proj = layer.self_attn.q_proj + # k_proj = layer.self_attn.k_proj + # v_proj = layer.self_attn.v_proj + # if hasattr(q_proj, "lora_A") and \ + # hasattr(k_proj, "lora_A") and \ + # hasattr(v_proj, "lora_A") and \ + # (q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None and \ + # (k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None and \ + # (v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None: - layer.self_attn.apply_qkv = apply_lora_qkv - n_qkv += 1 - else: - logger.warning_once( - "Unsloth cannot patch Attention layers with our manual autograd engine since either LoRA adapters\n"\ - "are not enabled or a bias term (like in Qwen) is used." - ) - pass + # layer.self_attn.apply_qkv = apply_lora_qkv + # n_qkv += 1 + # else: + # logger.warning_once( + # "Unsloth cannot patch Attention layers with our manual autograd engine since either LoRA adapters\n"\ + # "are not enabled or a bias term (like in Qwen) is used." + # ) + # pass - # O attention patching - o_proj = layer.self_attn.o_proj - if hasattr(o_proj, "lora_A") and \ - (o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None: + # # O attention patching + # o_proj = layer.self_attn.o_proj + # if hasattr(o_proj, "lora_A") and \ + # (o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None: - layer.self_attn.apply_o = apply_lora_o - n_o += 1 - else: - logger.warning_once( - "Unsloth cannot patch O projection layer with our manual autograd engine since either LoRA adapters\n"\ - "are not enabled or a bias term (like in Qwen) is used." - ) - pass - pass - pass + # layer.self_attn.apply_o = apply_lora_o + # n_o += 1 + # else: + # logger.warning_once( + # "Unsloth cannot patch O projection layer with our manual autograd engine since either LoRA adapters\n"\ + # "are not enabled or a bias term (like in Qwen) is used." + # ) + # pass + # pass + # pass logger.warning_once( f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "\