ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Llama-3

Browse source
This commit is contained in:
Daniel Han-Chen 2024-04-19 02:55:20 +10:00
commit be14d980f5
2 changed files with 9 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
unsloth/models/mapper.py
View file

@ -126,6 +126,14 @@ __INT_TO_FLOAT_MAPPER = \
"unsloth/codegemma-7b-it",
"google/codegemma-7b-it",
),
"unsloth/llama-3-8b-bnb-4bit" : (
"unsloth/llama-3-8b",
"meta-llama/Meta-Llama-3-8B",
),
"unsloth/llama-3-8b-Instruct-bnb-4bit" : (
"unsloth/llama-3-8b-Instruct",
"meta-llama/Meta-Llama-3-8B-Instruct",
),
}
INT_TO_FLOAT_MAPPER = {}

2
unsloth/tokenizer_utils.py
View file

@ -187,7 +187,7 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens))
try:
string = "\n".join(all_special_tokens) + \
"A quick brown fox jumps over the lazy dog!!\n\n" + \
"A quick brown fox jumps over the lazy dog!!\n\nHi</s>\n\n" + \
"".join(all_special_tokens)
return slow_tokenizer(string).input_ids == fast_tokenizer(string).input_ids
except: