ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update gemma.py

Browse source
This commit is contained in:
Daniel Han 2025-02-01 19:15:55 -08:00
commit 0dce4ffc2d
1 changed files with 2 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
unsloth/models/gemma.py
View file

@ -211,12 +211,11 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
):
super().__init__()
if config is not None:
print(config)
print(dir(config))
# [TODO] Hack to pass in config - need to remove later
base = config.rope_theta
partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
dim = int((config.hidden_size // config.num_attention_heads))
dim = getattr(config, "head_dim", None)
if dim is None: dim = int((config.hidden_size // config.num_attention_heads))
device = "cuda"
max_position_embeddings = config.max_position_embeddings
pass