ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Phi-4 bug fix

Browse source
This commit is contained in:
Daniel Han 2025-01-08 15:40:27 -08:00
commit 3b4364985f
2 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/_utils.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "2025.1.2"
__version__ = "2025.1.3"
__all__ = [
"SUPPORTS_BFLOAT16",

4
unsloth/tokenizer_utils.py
View file

@ -265,6 +265,10 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
)))
all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens))
# Remove replacement char for false positive
replacement_char = b"\xc3\xaf\xc2\xbf\xc2\xbd".decode("utf-8")
all_special_tokens = [x for x in all_special_tokens if x != replacement_char]
# Check if chat template is enabled!
check_chat_template1 = True
check_chat_template2 = True