From 24e9d0af283d5765da360e4b32789b85692bd671 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Mon, 26 Feb 2024 05:05:41 +1100 Subject: [PATCH] Update gemma.py --- unsloth/models/gemma.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 351df32ade..8dc7a5f3db 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -92,8 +92,7 @@ class FastGemmaRotaryEmbedding(torch.nn.Module): position_ids = torch.arange(self.max_position_embeddings, device=x.device, dtype=torch.int64).unsqueeze(0) inv_freq_expanded = self.inv_freq[None, :, None].float().expand(1, -1, 1) position_ids_expanded = position_ids[:, None, :].float() - with torch.autocast(device_type="cuda", enabled = False): - freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2) + freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2) emb = torch.cat((freqs, freqs), dim=-1) self.cos_cached = emb.cos().to(dtype=x.dtype) self.sin_cached = emb.sin().to(dtype=x.dtype) @@ -107,6 +106,10 @@ class FastGemmaRotaryEmbedding(torch.nn.Module): cos_cached = emb.cos().to(dtype=x.dtype) sin_cached = emb.sin().to(dtype=x.dtype) + print(freqs) + print(self.cos_cached2) + raise 1 + # return self.cos_cached[:,:length], self.sin_cached[:,:length] return self.cos_cached, self.sin_cached pass