From c96e8c28f85d452b07e9f6d20f30bd179abccffd Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sun, 25 Feb 2024 03:03:31 +1100 Subject: [PATCH] Update gemma.py --- unsloth/models/gemma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index d8156435a4..154049e473 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -85,7 +85,7 @@ class FastGemmaRotaryEmbedding(torch.nn.Module): emb_new = torch.cat((freqs, freqs), dim=-1) logger.warning_once(str(torch.dist(emb_old, emb_new))) - return emb_new.cos().to(dtype=x.dtype), emb_new.sin().to(dtype=x.dtype) + return emb_old.cos().to(dtype=x.dtype), emb_old.sin().to(dtype=x.dtype) pass