ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update llama.py

Browse source
This commit is contained in:
Daniel Han 2025-02-05 20:24:33 -08:00
commit 8e2dbefbf4
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
unsloth/models/llama.py
View file

@ -384,6 +384,7 @@ def LlamaAttention_fast_forward(
assert(n_kv_heads * n_groups == n_heads)
Q, K, V = self.apply_qkv(self, hidden_states)
print("#######", Q, self.q_proj.lora_B.default.weight)
Q = Q.view(bsz, q_len, n_heads, head_dim).transpose(1, 2)
K = K.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)
V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)