ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: use exact variant matching and shard-prefix discovery for split GGUFs

Browse source 
Substring matching (e.g. "Q8_0" in filename) could match superset
variants like "IQ8_0", causing wrong quantizations to be downloaded.
Now uses word-boundary regex for variant matching and discovers split
shards by shared filename prefix rather than treating all variant
matches as shards.
This commit is contained in:
Roland Tannous 2026-03-10 19:13:03 +00:00
commit d635846b8d
1 changed files with 21 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

28
studio/backend/core/inference/llama_cpp.py
View file

@ -255,16 +255,32 @@ class LlamaCppBackend:
if hf_variant:
# Try common naming patterns
try:
import re
from huggingface_hub import list_repo_files
files = list_repo_files(hf_repo, token=hf_token)
variant_lower = hf_variant.lower()
matching = sorted(
f for f in files
if f.endswith(".gguf") and variant_lower in f.lower()
# Use word-boundary matching so "Q8_0" doesn't also
# match "IQ8_0" or other superset variant names.
boundary = re.compile(
r'(?<![a-zA-Z0-9])' + re.escape(variant_lower) + r'(?![a-zA-Z0-9])'
)
if matching:
gguf_filename = matching[0] # first shard (or single file)
gguf_extra_shards = matching[1:] # remaining shards if split
gguf_files = sorted(
f for f in files
if f.endswith(".gguf") and boundary.search(f.lower())
)
if gguf_files:
gguf_filename = gguf_files[0]
# For split GGUFs (e.g. model-Q8_0-00001-of-00003.gguf)
# discover siblings by shared prefix instead of
# trusting all variant matches to be shards.
shard_pat = re.compile(r'^(.*)-\d{5}-of-\d{5}\.gguf$')
m = shard_pat.match(gguf_filename)
if m:
prefix = m.group(1)
gguf_extra_shards = [
f for f in gguf_files[1:]
if f.startswith(prefix + "-") and shard_pat.match(f)
]
except Exception as e:
logger.warning(f"Could not list repo files: {e}")