ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into feature/chat-api

Browse source
This commit is contained in:
Daniel Han 2026-05-11 03:02:22 -07:00 • committed by GitHub
commit 003f65f916
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 20 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

23
tests/sh/test_torch_constraint.sh
View file

@ -109,11 +109,28 @@ assert_eq "hardcoded torch>=2.4 appears exactly once" "1" "$_hardcoded"
echo ""
echo "=== Structural: tokenizers in no-torch-runtime.txt ==="
_has_tokenizers=$(grep -c '^tokenizers$' "$NO_TORCH_RT" || true)
assert_eq "tokenizers present as standalone line" "1" "$_has_tokenizers"
# Package-name boundary is anything not valid in a PEP 508 name, or EOL.
# Covers `tokenizers`, `tokenizers<=0.23.0`, `tokenizers[extra]`,
# `tokenizers; python_version<"3.13"`, etc., but NOT `tokenizers-foo`.
_TOK_RE='^tokenizers([^a-zA-Z0-9._-]|$)'
_has_tokenizers=$(grep -cE "$_TOK_RE" "$NO_TORCH_RT" || true)
assert_eq "tokenizers package listed" "1" "$_has_tokenizers"
# Regression guard for #5359: the tokenizers line must carry an upper
# bound that excludes 0.23.1+. transformers in the allowed 4.56..5.3
# window rejects 0.23.1 at import time with
# `tokenizers<=0.23.0,>=0.22.0 is required, but found 0.23.1`.
# Accept both `<=0.23.0` and the functionally equivalent `<0.23.1`.
# Two-stage grep: pick lines that start with the tokenizers package
# name (PEP 508 name boundary), then require a safe upper bound.
_has_safe_pin=$(grep -E "$_TOK_RE" "$NO_TORCH_RT" \
| grep -cE '(<=[[:space:]]*0\.23\.0|<[[:space:]]*0\.23\.1)' \
|| true)
assert_eq "tokenizers pinned with upper bound excluding 0.23.1+" "1" "$_has_safe_pin"
# tokenizers before transformers
_tok_line=$(grep -n '^tokenizers$' "$NO_TORCH_RT" | head -1 | cut -d: -f1)
_tok_line=$(grep -nE "$_TOK_RE" "$NO_TORCH_RT" | head -1 | cut -d: -f1)
_tf_line=$(grep -n '^transformers' "$NO_TORCH_RT" | head -1 | cut -d: -f1)
_tok_first=$([ "$_tok_line" -lt "$_tf_line" ] && echo "yes" || echo "no")
assert_eq "tokenizers before transformers" "yes" "$_tok_first"