ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Nightly (#783)

Browse source 
* Update __init__.py

* dynamic RoPE

* Update mistral.py

* Update llama.py

* Update tokenizer_utils.py

* Update mistral.py

* Update llama.py

* Update __init__.py

* Update flex_attention.py

* Update llama.py

* Update llama.py

* Mistral Nemo

* Update tokenizer_utils.py

* Update tokenizer_utils.py
This commit is contained in:
Daniel Han 2024-07-19 01:24:46 -07:00 • committed by GitHub
commit 6786fa425f
1 changed files with 7 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

10
unsloth/tokenizer_utils.py
View file

@ -41,14 +41,17 @@ IGNORED_TOKENIZER_CHECKING = frozenset((
))
IGNORED_TOKENIZER_NAMES = frozenset((
IGNORED_TOKENIZER_NAMES = [
"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit",
"unsloth/Mistral-Nemo-Instruct-2407",
"mistralai/Mistral-Nemo-Instruct-2407",
"unsloth/Mistral-Nemo-Base-2407-bnb-4bit",
"unsloth/Mistral-Nemo-Base-2407",
"mistralai/Mistral-Nemo-Base-2407",
))
]
IGNORED_TOKENIZER_NAMES = frozenset(
[x.lower() for x in IGNORED_TOKENIZER_NAMES]
)
# Check environments
keynames = "\n" + "\n".join(os.environ.keys())
@ -498,7 +501,8 @@ def load_correct_tokenizer(
cache_dir = cache_dir,
)
if tokenizer_name in IGNORED_TOKENIZER_NAMES: pass
if tokenizer_name in IGNORED_TOKENIZER_NAMES:
return fast_tokenizer
elif slow_tokenizer is not None:
if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token