From 8db7e809d0dd60fb0262b3d0c4db70d43100cce0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 30 Jul 2024 23:11:35 -0700 Subject: [PATCH] Update gemma2.py --- unsloth/models/gemma2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py index d2bfb78992..1cbaf5b169 100644 --- a/unsloth/models/gemma2.py +++ b/unsloth/models/gemma2.py @@ -141,7 +141,7 @@ def Gemma2Attention_fast_forward( # FA uses 1 / sqrt for softmax_scale! if not hasattr(self, "_flash_attention_softmax_scale"): - self._flash_attention_softmax_scale = 1.0 / self.config.query_pre_attn_scalar**0.5 + self._flash_attention_softmax_scale = 1.0 / (self.config.query_pre_attn_scalar**0.5) pass Q = Q.transpose(1, 2)