From 4e525b734b95e56ab18229c4f0fd4fb97cd1f01a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Mar 2026 11:05:08 +0000 Subject: [PATCH] Fix docstring, remove artifact, restore valuable comments in tokenizer_utils.py - Fix get_tokenizer_info() docstring example: correct tokenizer_class to PreTrainedTokenizerFast, vocab_size to 128000, swap added_tokens_count (256) and special_tokens_count (3) to match actual Llama-3.2-1B-Instruct output - Remove accidentally committed "# ... (rest of file unchanged)" diff artifact - Restore fix_sentencepiece_gguf() docstring with llama.cpp upstream link - Restore 10 comments containing upstream URLs, model-specific workarounds, and non-obvious context (issue #292, sentencepiece#121, Starling hack, Kaggle /tmp limit, Deepseek slow tokenizer, twitter/danielhanchen references) --- unsloth/tokenizer_utils.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index d39737da9e..7629043483 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -297,6 +297,9 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer): return check_chat_template and check_special_tokens except: + # For eg see https://github.com/unslothai/unsloth/issues/292 + # Sometimes tokenizer has weird tokens, causing a combined tokenization to fail. + # [TODO] We temporarily disable this for CodeLlama tokenizers if slow_tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING: return check_chat_template else: @@ -309,6 +312,8 @@ def fix_sentencepiece_tokenizer( token_mapping, temporary_location = "_unsloth_sentencepiece_temp", ): + # From https://github.com/google/sentencepiece/issues/121 + # We need to manually edit the sentencepiece tokenizer! try: from transformers.convert_slow_tokenizer import import_protobuf @@ -325,6 +330,7 @@ def fix_sentencepiece_tokenizer( f"Please downgrade via `pip install --force-reinstall protobuf==3.20.3`" ) except: + # This will only work for older SentencePiece versions <= 3.20.3 from transformers.utils import sentencepiece_model_pb2 if not os.path.exists(temporary_location): @@ -351,6 +357,7 @@ def fix_sentencepiece_tokenizer( ) continue ids = ids[0] + # [TODO] Hack for Starling - try except try: tokenizer_piece = tokenizer_file.pieces[ids] except: @@ -372,6 +379,11 @@ def fix_sentencepiece_tokenizer( def fix_sentencepiece_gguf(saved_location): + """ + Fixes sentencepiece tokenizers which did not extend the vocabulary with + user defined tokens. + Inspiration from https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py + """ from copy import deepcopy from transformers.utils import sentencepiece_model_pb2 import json @@ -442,10 +454,12 @@ def _load_correct_tokenizer( if IS_COLAB_ENVIRONMENT: cache_dir = cache_dir elif IS_KAGGLE_ENVIRONMENT: + # /tmp of Kaggle seems has a 80GB limit! cache_dir = os.path.join(KAGGLE_TMP, cache_dir) else: cache_dir = None + # Mainly to solve Deepseek models with no tokenizer.model file slow_tokenizer = None try: slow_tokenizer = AutoTokenizer.from_pretrained( @@ -454,6 +468,7 @@ def _load_correct_tokenizer( padding_side = padding_side, token = token, trust_remote_code = trust_remote_code, + # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373 use_fast = False, legacy = False, from_slow = True, @@ -663,6 +678,11 @@ def check_tokenizer( token = None, _reload = True, ): + # Checks tokenizer for out of bounds ids. + # Mainly a fix for https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha + # where had token id=32002. + # See https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha/discussions/25 + # Seems like the Fast tokenizer in Rust breaks things! if tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING: return tokenizer @@ -743,12 +763,14 @@ def check_tokenizer( else: cache_dir = None + # Sometimes slow tokenizer does not work like Deepseek try: tokenizer = AutoTokenizer.from_pretrained( model_name, model_max_length = model_max_length, padding_side = padding_side, token = token, + # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373 use_fast = False, legacy = False, from_slow = True, @@ -785,10 +807,10 @@ def get_tokenizer_info(tokenizer) -> dict: { "name_or_path": "unsloth/Llama-3.2-1B-Instruct", - "tokenizer_class": "LlamaTokenizerFast", + "tokenizer_class": "PreTrainedTokenizerFast", "is_fast": True, - "vocab_size": 128256, - "added_tokens_count": 3, + "vocab_size": 128000, + "added_tokens_count": 256, "model_max_length": 131072, "padding_side": "right", "bos_token": "<|begin_of_text|>", @@ -796,7 +818,7 @@ def get_tokenizer_info(tokenizer) -> dict: "pad_token": "<|finetune_right_pad_id|>", "unk_token": None, "has_chat_template": True, - "special_tokens_count": 256, + "special_tokens_count": 3, } Args: @@ -830,6 +852,5 @@ import trl.trainer.sft_trainer from trl.trainer.sft_trainer import * from transformers.trainer import * -# ... (rest of file unchanged) # Finally patch TRL tokenizer things -> moved to RL # patch_sft_trainer_tokenizer()