From 6af0ece0c19d11ca15b84a6e64d8444538aec801 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 16 Nov 2024 15:18:38 -0800 Subject: [PATCH] Update gemma2.py --- unsloth/models/gemma2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py index 872824b11a..62ecb9690f 100644 --- a/unsloth/models/gemma2.py +++ b/unsloth/models/gemma2.py @@ -60,8 +60,7 @@ if HAS_FLASH_ATTENTION_SOFTCAPPING: from flash_attn import flash_attn_func # [TODO] We must randomnly use torch.compile? -# I checked the gradients and formulas and I'm sure it's correct. -# I'm stumped :( +# Gemma 2 uses double RMS Layernorms, so the backward passes should not overwrite the gradients! @torch.compile(fullgraph = False, dynamic = True, options = torch_compile_options) def fast_rms_layernorm_gemma2_compiled(layernorm, X, gemma = True): old_dtype = X.dtype