From cb193f7e0a8a1f0709e2eb433281b0d2f04d81c1 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Mon, 4 Mar 2024 16:14:50 +1100 Subject: [PATCH] Update gemma.py --- unsloth/models/gemma.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index e9560cd172..99bd9e7865 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -357,7 +357,8 @@ class FastGemmaModel(FastLlamaModel): if isinstance(module, GemmaRMSNorm): # Must be in float32 # https://github.com/keras-team/keras-nlp/blob/v0.8.2/keras_nlp/models/gemma/rms_normalization.py#L36 - module = module.to(torch.float32) + # module = module.to(torch.float32) + # Don't convert to float32 since error analysis shows it makes it worse!! module.weight += 1.0 # return output * (1 + self.weight) if not hasattr(module, "variance_epsilon"): module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon