From 37d0dedde5032e33216d6bc903111fbadd7ae64d Mon Sep 17 00:00:00 2001 From: Lei Zhenyuan Date: Tue, 22 Jul 2025 18:27:57 +0800 Subject: [PATCH] fix for casual mask (#3011) --- unsloth/models/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 8d985aa9d2..27b92282c3 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -342,9 +342,9 @@ def LlamaAttention_fast_forward_inference( A = torch_matmul(A, Vnn, out = Qn) else: if SDPA_HAS_GQA: - A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = False, enable_gqa = True) + A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = is_causal, enable_gqa = True) else: - A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = False) + A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = is_causal) pass A = A.transpose(1, 2) A = A.reshape(bsz, 1, attention_size)