From 0a445a417a1e681f172a9c485ce03d349cb90d55 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 27 Oct 2024 17:34:33 -0700 Subject: [PATCH] Update _utils.py --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c0a15592e5..d92a938eee 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -507,7 +507,7 @@ def patch_tokenizer(model, tokenizer): "\u00BF", # Inverted question mark ] for replacement_char in possible_replacements: - char = tokenizer(replacement_char, add_special_tokens = False) + char = tokenizer(replacement_char, add_special_tokens = False).input_ids if len(char) == 1: # Get actual token representation try: char = tokenizer.convert_ids_to_tokens(char[0])