diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index b1d2faedb3..b6aab83069 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -126,6 +126,14 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/codegemma-7b-it", "google/codegemma-7b-it", ), + "unsloth/llama-3-8b-bnb-4bit" : ( + "unsloth/llama-3-8b", + "meta-llama/Meta-Llama-3-8B", + ), + "unsloth/llama-3-8b-Instruct-bnb-4bit" : ( + "unsloth/llama-3-8b-Instruct", + "meta-llama/Meta-Llama-3-8B-Instruct", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 132591276a..01ad2bff7f 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -187,7 +187,7 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer): all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens)) try: string = "\n".join(all_special_tokens) + \ - "A quick brown fox jumps over the lazy dog!!\n\n" + \ + "A quick brown fox jumps over the lazy dog!!\n\nHi\n\n" + \ "".join(all_special_tokens) return slow_tokenizer(string).input_ids == fast_tokenizer(string).input_ids except: