From ded20b2462387ecde2cdf60306c9395c95aba2d4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 19 Jul 2024 01:39:08 -0700 Subject: [PATCH] Nightly (#784) * Update __init__.py * dynamic RoPE * Update mistral.py * Update llama.py * Update tokenizer_utils.py * Update mistral.py * Update llama.py * Update __init__.py * Update flex_attention.py * Update llama.py * Update llama.py * Mistral Nemo * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 07cd874125..3f75d16863 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -682,6 +682,11 @@ def fix_untrained_tokens(model, tokenizer, train_dataset, eps = 1e-16): embedding_matrix = model.get_input_embeddings ().weight lm_head_matrix = model.get_output_embeddings().weight + # Ignore some model checks for now + if model.config._name_or_path in IGNORED_TOKENIZER_NAMES: + return + pass + # Get untrained tokens indicator_untrained = torch.amax(embedding_matrix, axis = 1) <= eps where_untrained = torch.where(indicator_untrained)[0]