ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Patch tokenizer

Browse source
This commit is contained in:
Daniel Han-Chen 2024-03-21 20:13:54 +11:00
commit 8d1490a4c5
2 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/chat_templates.py
View file

@ -22,6 +22,7 @@ from transformers import StoppingCriteria, StoppingCriteriaList
from torch import LongTensor, FloatTensor
from transformers.models.llama.modeling_llama import logger
from .models._utils import patch_tokenizer
from .save import patch_saving_functions
import os
import shutil
@ -433,8 +434,11 @@ def get_chat_template(
if old_bos_token != new_bos_token: tokenizer.bos_token = old_bos_token
if old_unk_token != new_unk_token: tokenizer.unk_token = old_unk_token
#stopping_criteria = create_stopping_criteria(tokenizer, stop_word)
# stopping_criteria = create_stopping_criteria(tokenizer, stop_word)
# Patch saving functions
tokenizer = patch_saving_functions(tokenizer)
return tokenizer#, stopping_criteria
pass

6
unsloth/save.py
View file

@ -276,7 +276,8 @@ def unsloth_save_model(
old_username = None, private = private,
)
model.original_push_to_hub(
getattr(model, "original_push_to_hub", tokenizer.push_to_hub)\
(
repo_id = save_directory,
use_temp_dir = use_temp_dir,
commit_message = commit_message,
@ -290,7 +291,8 @@ def unsloth_save_model(
tags = tags,
)
if tokenizer is not None:
tokenizer.original_push_to_hub(
getattr(tokenizer, "original_push_to_hub", tokenizer.push_to_hub)\
(
repo_id = save_directory,
use_temp_dir = use_temp_dir,
commit_message = commit_message,