ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update gemma.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-02-24 19:12:52 +11:00
commit 47c6feaf89
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
unsloth/models/gemma.py
View file

@ -650,12 +650,12 @@ class FastGemmaModel(FastLlamaModel):
pass
print("Unsloth: Patching Gemma RMS Layernorm + 1")
for name, module in model.named_modules():
if isinstance(module, GemmaRMSNorm):
module.weight += 1.0 # return output * (1 + self.weight)
if not hasattr(module, "variance_epsilon"):
module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon
pass
# for name, module in model.named_modules():
# if isinstance(module, GemmaRMSNorm):
# module.weight += 1.0 # return output * (1 + self.weight)
# if not hasattr(module, "variance_epsilon"):
# module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon
# pass
# Clear deleted GPU items
import gc