From faa66d209d85dfe0b91a696c2874878d71987e45 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 19 Apr 2024 03:33:12 +1000 Subject: [PATCH] Update _utils.py --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 69e570dc9a..8864ebab4b 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -148,7 +148,7 @@ def patch_tokenizer(model, tokenizer): model.config.update({"unsloth_version" : __version__}) if not hasattr(tokenizer, "pad_token") or tokenizer.pad_token is None: # Fixes https://github.com/unslothai/unsloth/issues/5 - if hasattr(tokenizer, "unk_token"): + if hasattr(tokenizer, "unk_token") and tokenizer.unk_token is not None: tokenizer.add_special_tokens({"pad_token" : tokenizer.unk_token}) tokenizer.pad_token = tokenizer.unk_token else: