diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index a93f18cd41..ab752f9123 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.1.2" +__version__ = "2025.1.3" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 384b4bbca5..67a4c663b5 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -265,6 +265,10 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer): ))) all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens)) + # Remove replacement char for false positive + replacement_char = b"\xc3\xaf\xc2\xbf\xc2\xbd".decode("utf-8") + all_special_tokens = [x for x in all_special_tokens if x != replacement_char] + # Check if chat template is enabled! check_chat_template1 = True check_chat_template2 = True