ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove patching

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-25 23:35:56 +11:00
commit 194093e375
2 changed files with 80 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

37
unsloth/kernels/fast_lora.py
View file

@ -18,28 +18,27 @@ from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel
def matmul_lora(X, W, W_quant, A, B, s, out = None):
with torch.cuda.amp.autocast(dtype = torch.float16):
dtype = X.dtype
W = fast_dequantize(W.t(), W_quant)
dtype = X.dtype
W = fast_dequantize(W.t(), W_quant)
if X.dim() == 3:
batch, seq_len, d = X.shape
X = X.view(-1, X.shape[-1])
reshape = True
else:
reshape = False
pass
if X.dim() == 3:
batch, seq_len, d = X.shape
X = X.view(-1, X.shape[-1])
reshape = True
else:
reshape = False
pass
out = torch.matmul(X, W, out = out)
if W_quant is not None: del W
out = torch.matmul(X, W, out = out)
if W_quant is not None: del W
if A is not None:
# LoRA is enabled
A, B = A.t(), B.t()
out += (X @ A) @ (s * B)
pass
return out.view(batch, seq_len, -1) if reshape else out
if A is not None:
# LoRA is enabled
A, B = A.t(), B.t()
out += (X @ A.to(dtype)) @ (s * B.to(dtype))
pass
return out.view(batch, seq_len, -1) if reshape else out
pass

106
unsloth/models/llama.py
View file

@ -1047,66 +1047,66 @@ class FastLlamaModel:
lora_dropout = model.peft_config[active_adapter].lora_dropout
bias = model.peft_config[active_adapter].bias
if lora_dropout == 0 and bias == "none":
for idx, layer in enumerate(model.model.model.layers):
# if lora_dropout == 0 and bias == "none":
# for idx, layer in enumerate(model.model.model.layers):
# MLP patching
gate_proj = layer.mlp.gate_proj
up_proj = layer.mlp. up_proj
down_proj = layer.mlp.down_proj
# # MLP patching
# gate_proj = layer.mlp.gate_proj
# up_proj = layer.mlp. up_proj
# down_proj = layer.mlp.down_proj
if hasattr(gate_proj, "lora_A") and \
hasattr( up_proj, "lora_A") and \
hasattr(down_proj, "lora_A") and \
(gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None and \
( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None and \
(down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None:
# if hasattr(gate_proj, "lora_A") and \
# hasattr( up_proj, "lora_A") and \
# hasattr(down_proj, "lora_A") and \
# (gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None and \
# ( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None and \
# (down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None:
# https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module
layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp)
n_mlp += 1
else:
logger.warning_once(
"Unsloth cannot patch MLP layers with our manual autograd engine since either LoRA adapters\n"\
"are not enabled or a bias term (like in Qwen) is used."
)
pass
# # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module
# layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp)
# n_mlp += 1
# else:
# logger.warning_once(
# "Unsloth cannot patch MLP layers with our manual autograd engine since either LoRA adapters\n"\
# "are not enabled or a bias term (like in Qwen) is used."
# )
# pass
# QKV attention patching
q_proj = layer.self_attn.q_proj
k_proj = layer.self_attn.k_proj
v_proj = layer.self_attn.v_proj
if hasattr(q_proj, "lora_A") and \
hasattr(k_proj, "lora_A") and \
hasattr(v_proj, "lora_A") and \
(q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None and \
(k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None and \
(v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None:
# # QKV attention patching
# q_proj = layer.self_attn.q_proj
# k_proj = layer.self_attn.k_proj
# v_proj = layer.self_attn.v_proj
# if hasattr(q_proj, "lora_A") and \
# hasattr(k_proj, "lora_A") and \
# hasattr(v_proj, "lora_A") and \
# (q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None and \
# (k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None and \
# (v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None:
layer.self_attn.apply_qkv = apply_lora_qkv
n_qkv += 1
else:
logger.warning_once(
"Unsloth cannot patch Attention layers with our manual autograd engine since either LoRA adapters\n"\
"are not enabled or a bias term (like in Qwen) is used."
)
pass
# layer.self_attn.apply_qkv = apply_lora_qkv
# n_qkv += 1
# else:
# logger.warning_once(
# "Unsloth cannot patch Attention layers with our manual autograd engine since either LoRA adapters\n"\
# "are not enabled or a bias term (like in Qwen) is used."
# )
# pass
# O attention patching
o_proj = layer.self_attn.o_proj
if hasattr(o_proj, "lora_A") and \
(o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None:
# # O attention patching
# o_proj = layer.self_attn.o_proj
# if hasattr(o_proj, "lora_A") and \
# (o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None:
layer.self_attn.apply_o = apply_lora_o
n_o += 1
else:
logger.warning_once(
"Unsloth cannot patch O projection layer with our manual autograd engine since either LoRA adapters\n"\
"are not enabled or a bias term (like in Qwen) is used."
)
pass
pass
pass
# layer.self_attn.apply_o = apply_lora_o
# n_o += 1
# else:
# logger.warning_once(
# "Unsloth cannot patch O projection layer with our manual autograd engine since either LoRA adapters\n"\
# "are not enabled or a bias term (like in Qwen) is used."
# )
# pass
# pass
# pass
logger.warning_once(
f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "\