ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sqrt

Browse source
This commit is contained in:
Daniel Han-Chen 2024-03-06 04:48:26 +11:00
commit 85c052d8fb
2 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
unsloth/models/gemma.py
View file

@ -156,8 +156,9 @@ def GemmaModel_fast_forward_inference(
hidden_states = self.embed_tokens(input_ids)
# 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
# 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
inputs_embeds *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = inputs_embeds.dtype)
# inputs_embeds *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = inputs_embeds.dtype)
inputs_embeds *= math_sqrt(self.config.hidden_size)
next_decoder_cache = []
for idx, decoder_layer in enumerate(self.layers):
# Self Attention

3
unsloth/models/llama.py
View file

@ -523,7 +523,8 @@ def LlamaModel_fast_forward(
# inputs_embeds *= math_sqrt(self.config.hidden_size)
# Ie 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
# & 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
inputs_embeds *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = inputs_embeds.dtype)
# inputs_embeds *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = inputs_embeds.dtype)
inputs_embeds *= math_sqrt(self.config.hidden_size)
if inputs_requires_grad: inputs_embeds.requires_grad_(True)
pass