ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update gemma2.py

Browse source
This commit is contained in:
Daniel Han 2024-07-30 23:04:00 -07:00
commit 04a9f4da74
1 changed files with 3 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unsloth/models/gemma2.py
View file

@ -131,6 +131,9 @@ def Gemma2Attention_fast_forward(
# Only enable if the attention_mask is True
has_sliding_window = type(attention_mask) is bool and attention_mask is True
print(HAS_FLASH_ATTENTION_SOFTCAPPING)
print(has_sliding_window)
print(attention_mask)
if HAS_FLASH_ATTENTION_SOFTCAPPING and type(attention_mask) is bool:
window = (-1, -1)
if has_sliding_window: