From 01cd3c328494e540ffe11acdf5504f296be74c57 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 2 Feb 2025 02:11:33 -0800 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 23a8c0a681..3c33253911 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -254,6 +254,7 @@ def LlamaAttention_fast_forward_inference( # pass # Attention + print(attention_mask) if bsz == 1: Qn *= self.scalar # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963 # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows