From 879bdd2efb52468108cf7dfa432501bcfcd11f33 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 23 Feb 2024 02:20:26 +1100 Subject: [PATCH] Update gemma.py --- unsloth/models/gemma.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 2621398614..580336f0ef 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -331,7 +331,8 @@ class FastGemmaModel(FastLlamaModel): for name, module in model.named_modules(): if isinstance(module, GemmaRMSNorm): module.weight += 1.0 # return output * (1 + self.weight) - model.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon + if not hasattr(module, "variance_epsilon"): + model.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon pass # Clear deleted GPU items