From 9cb5c2eca4c7b5ea8f2a3fb3048d0b376589296e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 16 Aug 2024 23:38:02 -0700 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index a4f0b33be4..38d5949f4b 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -834,7 +834,7 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16): # We instead check for repeated vectors lm_head_where = torch.where(indicator_untrained1)[0] lm_head_bad = lm_head_matrix[lm_head_where] - lm_head_bad = lm_head_bad.cpu().numpy().round(3) + lm_head_bad = lm_head_bad.cpu().to(torch.float32).numpy().round(3) from collections import Counter counter = Counter() for row in lm_head_bad: counter[hash(row.data.tobytes())] += 1