Tokenizers fix (#336)
* Update llama.py * Update llama.py * Update llama.py * Update save.py * Accuracy * Revert * Update save.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update chat_templates.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Account for DoRA * Update llama.py * Update save.py * GGUF incorrect * Update save.py * Update pyproject.toml * kaggle new * Update pyproject.toml * Update pyproject.toml * upcasting * Fix Colab * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update rope_embedding.py * Update rope_embedding.py * Fix bugs * Update fast_lora.py * Update fast_lora.py * Update README.md * Update README.md * GGUF * Update save.py * Update save.py * Update save.py * Update save.py * Update README.md * Update README.md * Bugs * Update fast_lora.py * Update pyproject.toml * Update fast_lora.py * Update __init__.py * Update fast_lora.py * dtype * Update llama.py * Update llama.py * Update llama.py * dtype * Update mistral.py * trust_remote_code * lm_head * Update llama.py * save_pretrained_settings * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * state_dict * Update save.py * whoami * Update llama.py * Update save.py * Update llama.py * Patch tokenizer * Update chat_templates.py * Heal tokenizers * Update chat_templates.py * Update mapper.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update chat_templates.py * tokenizer patching * patch_tokenizer * Update chat_templates.py * Update tokenizer_utils.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update tokenizer_utils.py * Edit * Update mistral.py * Update mistral.py * Stats * Update mistral.py * attention_mask * Update llama.py * Update llama.py * batch * Temp fix batch inference * Update llama.py * Update gemma.py * Fix inference * swiglu * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update mistral.py * Update llama.py * fast inference * model * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update llama.py * Update utils.py * inference * Update llama.py * Update llama.py * Update llama.py * overhead * Update llama.py * Update llama.py * compile * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update utils.py * lora mamtul * Update llama.py * Update llama.py * Update llama.py * offloaded checkpointing * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update llama.py * Update gemma.py * Revert "Update gemma.py" This reverts commit e3c3c5f3fa3d04a87f854056f6b547ced610d712. * Update _utils.py * Update _utils.py * Update _utils.py * Saving * sentencepiece_model_pb2 * Update llama.py * Update save.py * Update llama.py * padding side * Update tokenizer_utils.py * cache dir * Update tokenizer_utils.py * Update tokenizer_utils.py * Update pyproject.toml * Update pyproject.toml * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * Update save.py * Update save.py * checkpoint * Gemma 1.1 * more models * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * dtype * Update llama.py * CodeGemma * Fix downcasting * Some bugs * Fix Yi tokenizer * HF_TOKEN * Update llama.py * Update tokenizer_utils.py
This commit is contained in:
Daniel Han
GitHub
parent
488fb2f64c
commit
b462ab8ede
6 changed files with 113 additions and 46 deletions
|
|
@ -293,7 +293,7 @@ def get_chat_template(
|
|||
|
||||
# Check fast tokenizer
|
||||
if not is_fast_tokenizer:
|
||||
logger.warning_once(
|
||||
print(
|
||||
f"Unsloth: Not a fast tokenizer, so can't process it as of yet :(\n"\
|
||||
"Please log a Github issue if you want this as a new feature!\n"\
|
||||
"Your chat template will still work, but it won't add or edit tokens."
|
||||
|
|
@ -348,11 +348,31 @@ def get_chat_template(
|
|||
# But training the lm_head and embeddings are slow!
|
||||
# This is a HACK!
|
||||
# Idea from https://huggingface.co/cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser
|
||||
|
||||
old_bos_token = getattr(tokenizer, "bos_token", None)
|
||||
old_eos_token = getattr(tokenizer, "eos_token", None)
|
||||
old_pad_token = getattr(tokenizer, "pad_token", None)
|
||||
old_unk_token = getattr(tokenizer, "unk_token", None)
|
||||
|
||||
string_vocab = tokenizer._tokenizer.to_str()
|
||||
old_eos_token = tokenizer.eos_token
|
||||
string_vocab = string_vocab.replace(old_eos_token, stop_word)
|
||||
# First check if new stop_word is in the tokenizer
|
||||
if stop_word in string_vocab:
|
||||
# We shall swap them around
|
||||
temporary_stop_token = "<|:__TEMP//STOP//TOKEN__:|>"
|
||||
string_vocab = string_vocab.replace(old_eos_token, temporary_stop_token)
|
||||
string_vocab = string_vocab.replace(stop_word, old_eos_token)
|
||||
string_vocab = string_vocab.replace(temporary_stop_token, stop_word)
|
||||
else:
|
||||
string_vocab = string_vocab.replace(old_eos_token, stop_word)
|
||||
pass
|
||||
new_tokenizer = tokenizer._tokenizer.from_str(string_vocab)
|
||||
new_tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word)
|
||||
new_tokenizer = tokenizer.__class__(
|
||||
tokenizer_object = new_tokenizer,
|
||||
bos_token = old_bos_token,
|
||||
eos_token = stop_word,
|
||||
unk_token = old_unk_token,
|
||||
pad_token = old_pad_token,
|
||||
)
|
||||
|
||||
# Must fix the sentence piece tokenizer since there's no tokenizer.model file!
|
||||
token_mapping = { old_eos_token : stop_word, }
|
||||
|
|
|
|||
|
|
@ -1017,6 +1017,12 @@ class FastLlamaModel:
|
|||
trust_remote_code = False,
|
||||
**kwargs,
|
||||
):
|
||||
if token is None and "HF_TOKEN" in os.environ:
|
||||
token = os.environ["HF_TOKEN"]
|
||||
|
||||
if token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
|
||||
if model_patcher is None: model_patcher = FastLlamaModel
|
||||
SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported()
|
||||
gpu_stats = torch.cuda.get_device_properties(0)
|
||||
|
|
@ -1445,8 +1451,8 @@ class FastLlamaModel:
|
|||
for module in target_modules:
|
||||
if module == "lm_head":
|
||||
logger.warning_once(
|
||||
"Unsloth: `lm_head` should be placed in `modules_to_save` and not `target_modules`."\
|
||||
"We shall do it for you!"
|
||||
"Unsloth: `lm_head` should be placed in `modules_to_save` and not `target_modules`. "\
|
||||
"Luckily, we shall do it for you!"
|
||||
)
|
||||
train_lm_head = True
|
||||
if modules_to_save is None: modules_to_save = ["lm_head"]
|
||||
|
|
@ -1454,8 +1460,8 @@ class FastLlamaModel:
|
|||
|
||||
elif module == "embed_tokens":
|
||||
logger.warning_once(
|
||||
"Unsloth: `embed_tokens` should be placed in `modules_to_save` and not `target_modules`."\
|
||||
"We shall do it for you!"
|
||||
"Unsloth: `embed_tokens` should be placed in `modules_to_save` and not `target_modules`. "\
|
||||
"Luckily, we shall do it for you!"
|
||||
)
|
||||
train_embed_tokens = True
|
||||
if modules_to_save is None: modules_to_save = ["embed_tokens"]
|
||||
|
|
|
|||
|
|
@ -78,6 +78,12 @@ class FastLanguageModel(FastLlamaModel):
|
|||
use_gradient_checkpointing = True,
|
||||
*args, **kwargs,
|
||||
):
|
||||
if token is None and "HF_TOKEN" in os.environ:
|
||||
token = os.environ["HF_TOKEN"]
|
||||
|
||||
if token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
|
||||
old_model_name = model_name
|
||||
model_name = _get_model_name(model_name, load_in_4bit)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
from .llama import *
|
||||
import os
|
||||
from ._utils import __version__
|
||||
|
||||
from transformers.models.mistral.modeling_mistral import (
|
||||
|
|
@ -301,6 +302,12 @@ class FastMistralModel(FastLlamaModel):
|
|||
trust_remote_code = False,
|
||||
**kwargs,
|
||||
):
|
||||
if token is None and "HF_TOKEN" in os.environ:
|
||||
token = os.environ["HF_TOKEN"]
|
||||
|
||||
if token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
|
||||
if model_patcher is None: model_patcher = FastMistralModel
|
||||
# Mistral does NOT support RoPE Scaling!
|
||||
if rope_scaling is not None:
|
||||
|
|
|
|||
|
|
@ -327,7 +327,7 @@ def unsloth_save_model(
|
|||
if hasattr(model, "config"):
|
||||
print(f"Saved {save_method} model to https://huggingface.co/" + save_directory)
|
||||
pass
|
||||
return save_directory
|
||||
return save_directory, None
|
||||
pass
|
||||
|
||||
# Tokenizer has different saving arguments
|
||||
|
|
@ -402,7 +402,7 @@ def unsloth_save_model(
|
|||
pass
|
||||
|
||||
print(" Done.")
|
||||
return save_directory
|
||||
return save_directory, None
|
||||
pass
|
||||
|
||||
# If push_to_hub, we must remove the .../ part of a repo
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def try_fix_tokenizer(tokenizer, prepend = True):
|
|||
|
||||
tokenizer_string = converted_tokenizer.to_str()
|
||||
|
||||
# Llama does ▁apple. Sometimes this is wrong!!
|
||||
# Llama does _apple. Sometimes this is wrong!!
|
||||
prepend_text = '{"type":"Prepend","prepend":"▁"},'
|
||||
if not prepend and prepend_text in tokenizer_string:
|
||||
tokenizer_string = tokenizer_string.replace(prepend_text, "", 1)
|
||||
|
|
@ -269,15 +269,26 @@ def load_correct_tokenizer(
|
|||
cache_dir = None
|
||||
pass
|
||||
|
||||
slow_tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
use_fast = False,
|
||||
cache_dir = cache_dir,
|
||||
)
|
||||
# Try loading the slow tokenizer. If it fails, then try Fast only
|
||||
# Mainly to solve Deepseek models with no tokenizer.model file
|
||||
slow_tokenizer = None
|
||||
try:
|
||||
slow_tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
use_fast = False,
|
||||
cache_dir = cache_dir,
|
||||
)
|
||||
except:
|
||||
print(
|
||||
f"Unsloth: {tokenizer_name} has no tokenizer.model file.\n"\
|
||||
"Just informing you about this - this is not a critical error."
|
||||
)
|
||||
pass
|
||||
|
||||
fast_tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name,
|
||||
model_max_length = model_max_length,
|
||||
|
|
@ -286,14 +297,19 @@ def load_correct_tokenizer(
|
|||
trust_remote_code = trust_remote_code,
|
||||
cache_dir = cache_dir,
|
||||
)
|
||||
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
|
||||
fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token
|
||||
|
||||
# Confirm if slow and fast are equivalent!
|
||||
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):
|
||||
return fast_tokenizer
|
||||
|
||||
if slow_tokenizer is not None:
|
||||
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
|
||||
fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token
|
||||
|
||||
# Confirm if slow and fast are equivalent!
|
||||
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):
|
||||
return fast_tokenizer
|
||||
else:
|
||||
return convert_to_fast_tokenizer(slow_tokenizer)
|
||||
pass
|
||||
else:
|
||||
return convert_to_fast_tokenizer(slow_tokenizer)
|
||||
return fast_tokenizer
|
||||
pass
|
||||
pass
|
||||
|
||||
|
|
@ -408,25 +424,37 @@ def check_tokenizer(
|
|||
cache_dir = None
|
||||
pass
|
||||
|
||||
# Try slow tokenizer which can fix things!
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
use_fast = False,
|
||||
cache_dir = cache_dir,
|
||||
)
|
||||
return check_tokenizer(
|
||||
model = model,
|
||||
tokenizer = tokenizer,
|
||||
model_name = model_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
_reload = False,
|
||||
)
|
||||
break
|
||||
# Sometimes slow tokenizer does not work like Deepseek
|
||||
try:
|
||||
# Try slow tokenizer which can fix things!
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
use_fast = False,
|
||||
cache_dir = cache_dir,
|
||||
)
|
||||
return check_tokenizer(
|
||||
model = model,
|
||||
tokenizer = tokenizer,
|
||||
model_name = model_name,
|
||||
model_max_length = model_max_length,
|
||||
padding_side = padding_side,
|
||||
token = token,
|
||||
_reload = False,
|
||||
)
|
||||
break
|
||||
except:
|
||||
# Tokenizer has out of bounds issues and we can't
|
||||
# load the slow tokenizer version :(
|
||||
logger.warning_once(
|
||||
"Unsloth: Tokenizer is most likely buggy, and Unsloth failed to repair it.\n"\
|
||||
"It will still work, but beware of out of bounds memory accesses.\n"\
|
||||
"Please file an issue on the model owner's repo about this issue."
|
||||
)
|
||||
return tokenizer
|
||||
pass
|
||||
pass
|
||||
pass
|
||||
return convert_to_fast_tokenizer(tokenizer)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue