ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update gemma2.py

Browse source
This commit is contained in:
Daniel Han 2024-07-30 23:11:35 -07:00
commit 8db7e809d0
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/gemma2.py
View file

@ -141,7 +141,7 @@ def Gemma2Attention_fast_forward(
# FA uses 1 / sqrt for softmax_scale!
if not hasattr(self, "_flash_attention_softmax_scale"):
self._flash_attention_softmax_scale = 1.0 / self.config.query_pre_attn_scalar**0.5
self._flash_attention_softmax_scale = 1.0 / (self.config.query_pre_attn_scalar**0.5)
pass
Q = Q.transpose(1, 2)