From 6dad5dd9322c9310ee98bf1b6507543bb3b5d69b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 15 Apr 2024 04:18:01 +1000 Subject: [PATCH] Tokenizers fix (#336) * Update llama.py * Update llama.py * Update llama.py * Update save.py * Accuracy * Revert * Update save.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update chat_templates.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Account for DoRA * Update llama.py * Update save.py * GGUF incorrect * Update save.py * Update pyproject.toml * kaggle new * Update pyproject.toml * Update pyproject.toml * upcasting * Fix Colab * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update rope_embedding.py * Update rope_embedding.py * Fix bugs * Update fast_lora.py * Update fast_lora.py * Update README.md * Update README.md * GGUF * Update save.py * Update save.py * Update save.py * Update save.py * Update README.md * Update README.md * Bugs * Update fast_lora.py * Update pyproject.toml * Update fast_lora.py * Update __init__.py * Update fast_lora.py * dtype * Update llama.py * Update llama.py * Update llama.py * dtype * Update mistral.py * trust_remote_code * lm_head * Update llama.py * save_pretrained_settings * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * state_dict * Update save.py * whoami * Update llama.py * Update save.py * Update llama.py * Patch tokenizer * Update chat_templates.py * Heal tokenizers * Update chat_templates.py * Update mapper.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update chat_templates.py * tokenizer patching * patch_tokenizer * Update chat_templates.py * Update tokenizer_utils.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update tokenizer_utils.py * Edit * Update mistral.py * Update mistral.py * Stats * Update mistral.py * attention_mask * Update llama.py * Update llama.py * batch * Temp fix batch inference * Update llama.py * Update gemma.py * Fix inference * swiglu * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update mistral.py * Update llama.py * fast inference * model * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update llama.py * Update utils.py * inference * Update llama.py * Update llama.py * Update llama.py * overhead * Update llama.py * Update llama.py * compile * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update utils.py * lora mamtul * Update llama.py * Update llama.py * Update llama.py * offloaded checkpointing * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update llama.py * Update gemma.py * Revert "Update gemma.py" This reverts commit e3c3c5f3fa3d04a87f854056f6b547ced610d712. * Update _utils.py * Update _utils.py * Update _utils.py * Saving * sentencepiece_model_pb2 * Update llama.py * Update save.py * Update llama.py * padding side * Update tokenizer_utils.py * cache dir * Update tokenizer_utils.py * Update tokenizer_utils.py * Update pyproject.toml * Update pyproject.toml * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * Update save.py * Update save.py * checkpoint * Gemma 1.1 * more models * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * dtype * Update llama.py * CodeGemma * Fix downcasting * Some bugs * Fix Yi tokenizer * HF_TOKEN * Update llama.py * Update tokenizer_utils.py --- unsloth/chat_templates.py | 28 +++++++++-- unsloth/models/llama.py | 14 ++++-- unsloth/models/loader.py | 6 +++ unsloth/models/mistral.py | 7 +++ unsloth/save.py | 4 +- unsloth/tokenizer_utils.py | 100 ++++++++++++++++++++++++------------- 6 files changed, 113 insertions(+), 46 deletions(-) diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index ec62c9b858..6a0be38620 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -293,7 +293,7 @@ def get_chat_template( # Check fast tokenizer if not is_fast_tokenizer: - logger.warning_once( + print( f"Unsloth: Not a fast tokenizer, so can't process it as of yet :(\n"\ "Please log a Github issue if you want this as a new feature!\n"\ "Your chat template will still work, but it won't add or edit tokens." @@ -348,11 +348,31 @@ def get_chat_template( # But training the lm_head and embeddings are slow! # This is a HACK! # Idea from https://huggingface.co/cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser + + old_bos_token = getattr(tokenizer, "bos_token", None) + old_eos_token = getattr(tokenizer, "eos_token", None) + old_pad_token = getattr(tokenizer, "pad_token", None) + old_unk_token = getattr(tokenizer, "unk_token", None) + string_vocab = tokenizer._tokenizer.to_str() - old_eos_token = tokenizer.eos_token - string_vocab = string_vocab.replace(old_eos_token, stop_word) + # First check if new stop_word is in the tokenizer + if stop_word in string_vocab: + # We shall swap them around + temporary_stop_token = "<|:__TEMP//STOP//TOKEN__:|>" + string_vocab = string_vocab.replace(old_eos_token, temporary_stop_token) + string_vocab = string_vocab.replace(stop_word, old_eos_token) + string_vocab = string_vocab.replace(temporary_stop_token, stop_word) + else: + string_vocab = string_vocab.replace(old_eos_token, stop_word) + pass new_tokenizer = tokenizer._tokenizer.from_str(string_vocab) - new_tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word) + new_tokenizer = tokenizer.__class__( + tokenizer_object = new_tokenizer, + bos_token = old_bos_token, + eos_token = stop_word, + unk_token = old_unk_token, + pad_token = old_pad_token, + ) # Must fix the sentence piece tokenizer since there's no tokenizer.model file! token_mapping = { old_eos_token : stop_word, } diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 202c692c4e..6f70bc5105 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1017,6 +1017,12 @@ class FastLlamaModel: trust_remote_code = False, **kwargs, ): + if token is None and "HF_TOKEN" in os.environ: + token = os.environ["HF_TOKEN"] + + if token is None and "HUGGINGFACE_TOKEN" in os.environ: + token = os.environ["HUGGINGFACE_TOKEN"] + if model_patcher is None: model_patcher = FastLlamaModel SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported() gpu_stats = torch.cuda.get_device_properties(0) @@ -1445,8 +1451,8 @@ class FastLlamaModel: for module in target_modules: if module == "lm_head": logger.warning_once( - "Unsloth: `lm_head` should be placed in `modules_to_save` and not `target_modules`."\ - "We shall do it for you!" + "Unsloth: `lm_head` should be placed in `modules_to_save` and not `target_modules`. "\ + "Luckily, we shall do it for you!" ) train_lm_head = True if modules_to_save is None: modules_to_save = ["lm_head"] @@ -1454,8 +1460,8 @@ class FastLlamaModel: elif module == "embed_tokens": logger.warning_once( - "Unsloth: `embed_tokens` should be placed in `modules_to_save` and not `target_modules`."\ - "We shall do it for you!" + "Unsloth: `embed_tokens` should be placed in `modules_to_save` and not `target_modules`. "\ + "Luckily, we shall do it for you!" ) train_embed_tokens = True if modules_to_save is None: modules_to_save = ["embed_tokens"] diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 29d25f3c20..fa864a9a80 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -78,6 +78,12 @@ class FastLanguageModel(FastLlamaModel): use_gradient_checkpointing = True, *args, **kwargs, ): + if token is None and "HF_TOKEN" in os.environ: + token = os.environ["HF_TOKEN"] + + if token is None and "HUGGINGFACE_TOKEN" in os.environ: + token = os.environ["HUGGINGFACE_TOKEN"] + old_model_name = model_name model_name = _get_model_name(model_name, load_in_4bit) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 3034b83d2a..56108939b9 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -13,6 +13,7 @@ # limitations under the License. from .llama import * +import os from ._utils import __version__ from transformers.models.mistral.modeling_mistral import ( @@ -301,6 +302,12 @@ class FastMistralModel(FastLlamaModel): trust_remote_code = False, **kwargs, ): + if token is None and "HF_TOKEN" in os.environ: + token = os.environ["HF_TOKEN"] + + if token is None and "HUGGINGFACE_TOKEN" in os.environ: + token = os.environ["HUGGINGFACE_TOKEN"] + if model_patcher is None: model_patcher = FastMistralModel # Mistral does NOT support RoPE Scaling! if rope_scaling is not None: diff --git a/unsloth/save.py b/unsloth/save.py index d0010321b1..655d1c5107 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -327,7 +327,7 @@ def unsloth_save_model( if hasattr(model, "config"): print(f"Saved {save_method} model to https://huggingface.co/" + save_directory) pass - return save_directory + return save_directory, None pass # Tokenizer has different saving arguments @@ -402,7 +402,7 @@ def unsloth_save_model( pass print(" Done.") - return save_directory + return save_directory, None pass # If push_to_hub, we must remove the .../ part of a repo diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 2640f321d9..132591276a 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -48,7 +48,7 @@ def try_fix_tokenizer(tokenizer, prepend = True): tokenizer_string = converted_tokenizer.to_str() - # Llama does ▁apple. Sometimes this is wrong!! + # Llama does _apple. Sometimes this is wrong!! prepend_text = '{"type":"Prepend","prepend":"▁"},' if not prepend and prepend_text in tokenizer_string: tokenizer_string = tokenizer_string.replace(prepend_text, "", 1) @@ -269,15 +269,26 @@ def load_correct_tokenizer( cache_dir = None pass - slow_tokenizer = AutoTokenizer.from_pretrained( - tokenizer_name, - model_max_length = model_max_length, - padding_side = padding_side, - token = token, - trust_remote_code = trust_remote_code, - use_fast = False, - cache_dir = cache_dir, - ) + # Try loading the slow tokenizer. If it fails, then try Fast only + # Mainly to solve Deepseek models with no tokenizer.model file + slow_tokenizer = None + try: + slow_tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, + model_max_length = model_max_length, + padding_side = padding_side, + token = token, + trust_remote_code = trust_remote_code, + use_fast = False, + cache_dir = cache_dir, + ) + except: + print( + f"Unsloth: {tokenizer_name} has no tokenizer.model file.\n"\ + "Just informing you about this - this is not a critical error." + ) + pass + fast_tokenizer = AutoTokenizer.from_pretrained( tokenizer_name, model_max_length = model_max_length, @@ -286,14 +297,19 @@ def load_correct_tokenizer( trust_remote_code = trust_remote_code, cache_dir = cache_dir, ) - fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token - fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token - - # Confirm if slow and fast are equivalent! - if assert_same_tokenization(slow_tokenizer, fast_tokenizer): - return fast_tokenizer + + if slow_tokenizer is not None: + fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token + fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token + + # Confirm if slow and fast are equivalent! + if assert_same_tokenization(slow_tokenizer, fast_tokenizer): + return fast_tokenizer + else: + return convert_to_fast_tokenizer(slow_tokenizer) + pass else: - return convert_to_fast_tokenizer(slow_tokenizer) + return fast_tokenizer pass pass @@ -408,25 +424,37 @@ def check_tokenizer( cache_dir = None pass - # Try slow tokenizer which can fix things! - tokenizer = AutoTokenizer.from_pretrained( - model_name, - model_max_length = model_max_length, - padding_side = padding_side, - token = token, - use_fast = False, - cache_dir = cache_dir, - ) - return check_tokenizer( - model = model, - tokenizer = tokenizer, - model_name = model_name, - model_max_length = model_max_length, - padding_side = padding_side, - token = token, - _reload = False, - ) - break + # Sometimes slow tokenizer does not work like Deepseek + try: + # Try slow tokenizer which can fix things! + tokenizer = AutoTokenizer.from_pretrained( + model_name, + model_max_length = model_max_length, + padding_side = padding_side, + token = token, + use_fast = False, + cache_dir = cache_dir, + ) + return check_tokenizer( + model = model, + tokenizer = tokenizer, + model_name = model_name, + model_max_length = model_max_length, + padding_side = padding_side, + token = token, + _reload = False, + ) + break + except: + # Tokenizer has out of bounds issues and we can't + # load the slow tokenizer version :( + logger.warning_once( + "Unsloth: Tokenizer is most likely buggy, and Unsloth failed to repair it.\n"\ + "It will still work, but beware of out of bounds memory accesses.\n"\ + "Please file an issue on the model owner's repo about this issue." + ) + return tokenizer + pass pass pass return convert_to_fast_tokenizer(tokenizer)