From 23752a7ab1911e12bc3c05ddbdca03b5367deb43 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 16 Aug 2024 23:39:44 -0700 Subject: [PATCH] Bug #930 (#931) * untrained tokens llama 3.1 base * Update tokenizer_utils.py * Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index a4f0b33be4..7316656b2a 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -834,7 +834,7 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16): # We instead check for repeated vectors lm_head_where = torch.where(indicator_untrained1)[0] lm_head_bad = lm_head_matrix[lm_head_where] - lm_head_bad = lm_head_bad.cpu().numpy().round(3) + lm_head_bad = lm_head_bad.cpu().float().numpy().round(3) from collections import Counter counter = Counter() for row in lm_head_bad: counter[hash(row.data.tobytes())] += 1