From ba92dbb4af325615fcb96a826f89bdcc0023a7ac Mon Sep 17 00:00:00 2001
From: Daniel Han-Chen <danielhanchen@gmail.com>
Date: Mon, 25 Mar 2024 03:52:20 +1100
Subject: [PATCH] Update tokenizer_utils.py

---
 unsloth/tokenizer_utils.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 34f2b21301..75f99c1239 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -19,8 +19,8 @@ import re
 import os
 
 __all__ = [
-	"load_correct_tokenizer",
-	"fix_sentencepiece_tokenizer",
+    "load_correct_tokenizer",
+    "fix_sentencepiece_tokenizer",
 ]
 
 
@@ -120,11 +120,9 @@ def convert_to_fast_tokenizer(
     args2 = [x for x in args2 if not x.endswith("_file")]
     args = list(set(args + args2))
 
-    kwargs = {
-        "tokenizer_object" : try_fix_tokenizer(slow_tokenizer, prepend = True),
-        "tokenizer_file"   : slow_tokenizer.vocab_file,
-    }
+    kwargs = {}
     for arg in args: kwargs[arg] = getattr(slow_tokenizer, arg, None)
+    kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = True)
     fast_tokenizer = FastTokenizer( **kwargs )
 
     # Check if they're similar!
@@ -248,7 +246,7 @@ pass
 
 def load_correct_tokenizer(
     tokenizer_name,
-    model_max_length,
+    model_max_length = None,
     padding_side = "right",
     token = None,
     trust_remote_code = False,
@@ -278,3 +276,4 @@ def load_correct_tokenizer(
         return convert_to_fast_tokenizer(slow_tokenizer)
     pass
 pass
+