From 47c6feaf899f7224466bc2bf5c688cabd673ede6 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 24 Feb 2024 19:12:52 +1100 Subject: [PATCH] Update gemma.py --- unsloth/models/gemma.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 75af5f64e7..2383ea2bb4 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -650,12 +650,12 @@ class FastGemmaModel(FastLlamaModel): pass print("Unsloth: Patching Gemma RMS Layernorm + 1") - for name, module in model.named_modules(): - if isinstance(module, GemmaRMSNorm): - module.weight += 1.0 # return output * (1 + self.weight) - if not hasattr(module, "variance_epsilon"): - module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon - pass + # for name, module in model.named_modules(): + # if isinstance(module, GemmaRMSNorm): + # module.weight += 1.0 # return output * (1 + self.weight) + # if not hasattr(module, "variance_epsilon"): + # module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon + # pass # Clear deleted GPU items import gc