ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2024-08-19 16:03:24 -07:00
commit 4f51fe0a8c
1 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

16
unsloth/tokenizer_utils.py
View file

@ -1176,15 +1176,15 @@ def patch_sft_trainer_tokenizer():
# Add NEFTune since it doesn't seem to work?? We need to manually inject it
check_text += \
"\n\n"\
"print(1)\n"\
"if getattr(self.model.get_input_embeddings(), 'neftune_noise_alpha', None) is not None:\n"\
" print(2)\n"\
" if hasattr(self, 'neftune_hook_handle'):\n"\
" self.neftune_hook_handle.remove()\n"\
" if hasattr(self, 'neftune_hook_handle'): del self.neftune_hook_handle\n"\
"\n"\
" self.neftune_hook_handle = self.model.get_input_embeddings().register_forward_hook(neftune_post_forward_hook)\n\n"\
"if hasattr(self, 'neftune_hook_handle'):\n"\
" self.neftune_hook_handle.remove()\n"\
" if hasattr(self, 'neftune_hook_handle'): del self.neftune_hook_handle\n"\
"\n"\
"if getattr(self, 'neftune_noise_alpha', None) is not None:\n"\
" self.model.get_input_embeddings().neftune_noise_alpha = self.neftune_noise_alpha\n"\
" self.neftune_hook_handle = self.model.get_input_embeddings().register_forward_hook(neftune_post_forward_hook)\n"\
"pass\n"\
"\n"
check_text = check_text.split("\n")