Fix GGUF variant file selection (#6342)
* Fix GGUF variant resolution * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Address GGUF variant review feedback * Harden GGUF endian filtering * Address GGUF endian review comments * Mirror GGUF endian filter in local resolver * Fix GGUF route import test stub * Apply GGUF endian filtering across load paths * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Wasim Yousef Said
GitHub
parent
0ac1fb5d9e
commit
048f34e8f2
12 changed files with 669 additions and 79 deletions
|
|
@ -247,8 +247,8 @@ def _should_suppress_forced_no_tool_output(text: str) -> bool:
|
|||
|
||||
|
||||
# ── Pre-compiled patterns for GGUF shard detection ───────────
|
||||
_SHARD_FULL_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$")
|
||||
_SHARD_RE = re.compile(r"^(.*)-\d{5}-of-\d{5}\.gguf$")
|
||||
_SHARD_FULL_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE)
|
||||
_SHARD_RE = re.compile(r"^(.*)-\d{5}-of-\d{5}\.gguf$", re.IGNORECASE)
|
||||
|
||||
|
||||
# ── Sliding-window-pattern resolver ───────────────────────────
|
||||
|
|
@ -626,6 +626,99 @@ def _is_companion_gguf_path(path: str) -> bool:
|
|||
return name.startswith("mtp-") or "/mtp/" in f"/{p}"
|
||||
|
||||
|
||||
_BIG_ENDIAN_GGUF_FILENAME_RE = re.compile(r"(^|[-_])be(?:[._-]|$)", re.IGNORECASE)
|
||||
_GGUF_KNOWN_QUANT_RE = re.compile(
|
||||
r"(UD-)?"
|
||||
r"(MXFP[0-9]+(?:_[A-Z0-9]+)*"
|
||||
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
|
||||
r"|TQ[0-9]+_[0-9]+"
|
||||
r"|Q[0-9]+_K_[A-Z]+"
|
||||
r"|Q[0-9]+_[0-9]+"
|
||||
r"|Q[0-9]+_K"
|
||||
r"|BF16|F16|F32)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_big_endian_gguf_path(path: str, variant_key: str = "") -> bool:
|
||||
normalized = path.replace("\\", "/")
|
||||
name = normalized.rsplit("/", 1)[-1]
|
||||
stem = name.rsplit(".", 1)[0].lower()
|
||||
variant_key = variant_key.strip().lower()
|
||||
variant_index = stem.find(variant_key) if variant_key else -1
|
||||
parent = normalized.rsplit("/", 1)[0].lower() if "/" in normalized else ""
|
||||
variant_in_parent_only = (
|
||||
bool(parent)
|
||||
and variant_index < 0
|
||||
and (
|
||||
(variant_key and variant_key in parent)
|
||||
or (not variant_key and _GGUF_KNOWN_QUANT_RE.search(parent) is not None)
|
||||
)
|
||||
)
|
||||
for match in _BIG_ENDIAN_GGUF_FILENAME_RE.finditer(stem):
|
||||
if variant_index >= 0 and variant_index < match.start():
|
||||
return True
|
||||
tail = stem[match.end() :].lstrip("._-")
|
||||
if not tail or _GGUF_KNOWN_QUANT_RE.search(tail) is None:
|
||||
return not variant_in_parent_only
|
||||
return False
|
||||
|
||||
|
||||
def _gguf_snapshot_files(snapshot: Path) -> list[str]:
|
||||
return [
|
||||
p.relative_to(snapshot).as_posix()
|
||||
for p in snapshot.rglob("*")
|
||||
if p.is_file() and p.name.lower().endswith(".gguf")
|
||||
]
|
||||
|
||||
|
||||
def _gguf_extra_shards(files: Iterable[str], first_shard: str) -> list[str]:
|
||||
m = _SHARD_FULL_RE.match(first_shard)
|
||||
if not m:
|
||||
return []
|
||||
prefix = m.group(1)
|
||||
total = m.group(3)
|
||||
sibling_pat = re.compile(
|
||||
r"^" + re.escape(prefix) + r"-\d{5}-of-" + re.escape(total) + r"\.gguf$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
return sorted(f for f in files if f != first_shard and sibling_pat.match(f))
|
||||
|
||||
|
||||
def _gguf_files_for_variant(files: Iterable[str], variant: str) -> list[str]:
|
||||
"""Return main GGUF files matching a requested variant.
|
||||
|
||||
Prefer exact quant-label matches over loose substring matches so a request
|
||||
for ``stories260K`` does not resolve to ``stories260K-be.gguf``.
|
||||
"""
|
||||
variant_key = variant.strip().lower()
|
||||
main_files = [
|
||||
f
|
||||
for f in files
|
||||
if f.lower().endswith(".gguf")
|
||||
and not _is_companion_gguf_path(f)
|
||||
and not _is_big_endian_gguf_path(f, variant_key)
|
||||
]
|
||||
if not variant_key:
|
||||
return sorted(main_files)
|
||||
|
||||
try:
|
||||
from utils.models.model_config import _extract_quant_label
|
||||
except Exception:
|
||||
_extract_quant_label = None
|
||||
|
||||
if _extract_quant_label is not None:
|
||||
try:
|
||||
exact = sorted(f for f in main_files if _extract_quant_label(f).lower() == variant_key)
|
||||
if exact:
|
||||
return exact
|
||||
except Exception as e:
|
||||
logger.warning("Failed to extract GGUF quant labels: %s", e)
|
||||
|
||||
boundary = re.compile(r"(?<![a-zA-Z0-9])" + re.escape(variant_key) + r"(?![a-zA-Z0-9])")
|
||||
return sorted(f for f in main_files if boundary.search(f.lower()))
|
||||
|
||||
|
||||
# Below this many B params, draft-mtp regresses vs spec-off (bench in
|
||||
# _build_speculative_flags); auto mode drops MTP under it.
|
||||
_MTP_MIN_SIZE_B = 3.0
|
||||
|
|
@ -1046,12 +1139,13 @@ class LlamaCppBackend:
|
|||
m = _SHARD_RE.match(stem)
|
||||
prefix = m.group(1) if m else None
|
||||
if prefix and parent.is_dir():
|
||||
prefix_lower = prefix.lower()
|
||||
for sibling in parent.iterdir():
|
||||
if (
|
||||
sibling.is_file()
|
||||
and sibling.name.startswith(prefix)
|
||||
and sibling.name.lower().startswith(prefix_lower)
|
||||
and sibling.name != stem
|
||||
and sibling.suffix == ".gguf"
|
||||
and sibling.suffix.lower() == ".gguf"
|
||||
):
|
||||
try:
|
||||
bytes_total += sibling.stat().st_size
|
||||
|
|
@ -1500,7 +1594,8 @@ class LlamaCppBackend:
|
|||
if m:
|
||||
prefix, _, num_total = m.group(1), m.group(2), m.group(3)
|
||||
sibling_pat = re.compile(
|
||||
r"^" + re.escape(prefix) + r"-\d{5}-of-" + re.escape(num_total) + r"\.gguf$"
|
||||
r"^" + re.escape(prefix) + r"-\d{5}-of-" + re.escape(num_total) + r"\.gguf$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
for sibling in main.parent.iterdir():
|
||||
if sibling != main and sibling_pat.match(sibling.name):
|
||||
|
|
@ -2319,7 +2414,11 @@ class LlamaCppBackend:
|
|||
|
||||
files = list_repo_files(hf_repo, token = hf_token)
|
||||
gguf_files = [
|
||||
f for f in files if f.endswith(".gguf") and not _is_companion_gguf_path(f)
|
||||
f
|
||||
for f in files
|
||||
if f.lower().endswith(".gguf")
|
||||
and not _is_companion_gguf_path(f)
|
||||
and not _is_big_endian_gguf_path(f)
|
||||
]
|
||||
if not gguf_files:
|
||||
return None
|
||||
|
|
@ -2912,27 +3011,10 @@ class LlamaCppBackend:
|
|||
from huggingface_hub import list_repo_files
|
||||
|
||||
files = list_repo_files(hf_repo, token = hf_token)
|
||||
variant_lower = hf_variant.lower()
|
||||
boundary = re.compile(
|
||||
r"(?<![a-zA-Z0-9])" + re.escape(variant_lower) + r"(?![a-zA-Z0-9])"
|
||||
)
|
||||
gguf_files = sorted(
|
||||
f
|
||||
for f in files
|
||||
if f.endswith(".gguf")
|
||||
and boundary.search(f.lower())
|
||||
and not _is_companion_gguf_path(f)
|
||||
)
|
||||
gguf_files = _gguf_files_for_variant(files, hf_variant)
|
||||
if gguf_files:
|
||||
gguf_filename = gguf_files[0]
|
||||
m = _SHARD_FULL_RE.match(gguf_filename)
|
||||
if m:
|
||||
prefix = m.group(1)
|
||||
total = m.group(3)
|
||||
sibling_pat = re.compile(
|
||||
r"^" + re.escape(prefix) + r"-\d{5}-of-" + re.escape(total) + r"\.gguf$"
|
||||
)
|
||||
gguf_extra_shards = [f for f in gguf_files[1:] if sibling_pat.match(f)]
|
||||
gguf_extra_shards = _gguf_extra_shards(gguf_files, gguf_filename)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not list repo files: {e}")
|
||||
|
||||
|
|
@ -2944,33 +3026,13 @@ class LlamaCppBackend:
|
|||
if not gguf_filename:
|
||||
try:
|
||||
from utils.models.model_config import _iter_hf_cache_snapshots
|
||||
boundary = re.compile(
|
||||
r"(?<![a-zA-Z0-9])" + re.escape(hf_variant.lower()) + r"(?![a-zA-Z0-9])"
|
||||
)
|
||||
for snap in _iter_hf_cache_snapshots(hf_repo):
|
||||
matches = sorted(
|
||||
p.relative_to(snap).as_posix()
|
||||
for p in snap.rglob("*.gguf")
|
||||
if not _is_companion_gguf_path(p.relative_to(snap).as_posix())
|
||||
and boundary.search(p.relative_to(snap).as_posix().lower())
|
||||
)
|
||||
cached_files = _gguf_snapshot_files(snap)
|
||||
matches = _gguf_files_for_variant(cached_files, hf_variant)
|
||||
if not matches:
|
||||
continue
|
||||
gguf_filename = matches[0]
|
||||
m = _SHARD_FULL_RE.match(Path(gguf_filename).name)
|
||||
if m:
|
||||
prefix = m.group(1)
|
||||
total = m.group(3)
|
||||
sibling_pat = re.compile(
|
||||
r"^"
|
||||
+ re.escape(prefix)
|
||||
+ r"-\d{5}-of-"
|
||||
+ re.escape(total)
|
||||
+ r"\.gguf$"
|
||||
)
|
||||
gguf_extra_shards = [
|
||||
f for f in matches[1:] if sibling_pat.match(Path(f).name)
|
||||
]
|
||||
gguf_extra_shards = _gguf_extra_shards(matches, gguf_filename)
|
||||
logger.info(
|
||||
"Resolved variant %s -> %s from local HF cache",
|
||||
hf_variant,
|
||||
|
|
@ -3050,10 +3112,11 @@ class LlamaCppBackend:
|
|||
_m = _SHARD_RE.match(gguf_filename)
|
||||
_prefix = _m.group(1) if _m else None
|
||||
if _prefix:
|
||||
prefix_lower = _prefix.lower()
|
||||
gguf_extra_shards = sorted(
|
||||
f
|
||||
for f in all_gguf_files
|
||||
if f.startswith(_prefix)
|
||||
if f.lower().startswith(prefix_lower)
|
||||
and f != gguf_filename
|
||||
and not _is_companion_gguf_path(f)
|
||||
)
|
||||
|
|
@ -3138,7 +3201,7 @@ class LlamaCppBackend:
|
|||
try:
|
||||
from utils.models.model_config import _iter_hf_cache_snapshots
|
||||
for snap in _iter_hf_cache_snapshots(hf_repo):
|
||||
rel_files = [p.relative_to(snap).as_posix() for p in snap.rglob("*.gguf")]
|
||||
rel_files = _gguf_snapshot_files(snap)
|
||||
target = pick(rel_files)
|
||||
if target is not None:
|
||||
logger.info("Resolved %s %s from local HF cache", label, target)
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from hub.utils.hf_cache_state import (
|
|||
from hub.utils.gguf import (
|
||||
extract_quant_label,
|
||||
iter_hf_cache_snapshots,
|
||||
is_big_endian_gguf_path,
|
||||
list_gguf_variants,
|
||||
list_gguf_variants_from_hf_cache,
|
||||
list_local_gguf_variants,
|
||||
|
|
@ -482,7 +483,10 @@ async def get_gguf_variants_response(
|
|||
by_filename[key] = max(by_filename.get(key, 0), size)
|
||||
if _is_mmproj_filename(f.name) or _is_mtp_drafter_path(rel):
|
||||
continue
|
||||
q = extract_quant_label(rel).lower()
|
||||
q = extract_quant_label(rel)
|
||||
if is_big_endian_gguf_path(rel, q):
|
||||
continue
|
||||
q = q.lower()
|
||||
by_quant[q] = by_quant.get(q, 0) + size
|
||||
if by_filename:
|
||||
cached_filenames_by_snapshot.append(by_filename)
|
||||
|
|
|
|||
|
|
@ -78,6 +78,28 @@ class TestExtractQuantToken:
|
|||
assert labels == {"Q4_K_M", "Q8_0"}
|
||||
|
||||
|
||||
def test_big_endian_detection_ignores_model_name_be_token():
|
||||
assert gguf.is_big_endian_gguf_path("model-Q4_K_M-be.gguf", "Q4_K_M")
|
||||
assert gguf.is_big_endian_gguf_path("model-Q4_K_M_be_infill.gguf", "Q4_K_M")
|
||||
assert not gguf.is_big_endian_gguf_path("foo-be-Q4_K_M.gguf", "Q4_K_M")
|
||||
assert not gguf.is_big_endian_gguf_path("Q4_K_M/foo-be.gguf", "Q4_K_M")
|
||||
assert gguf.pick_best_gguf(["model-Q4_K_M-be.gguf", "model-Q4_K_M.gguf"]) == (
|
||||
"model-Q4_K_M.gguf"
|
||||
)
|
||||
|
||||
|
||||
def test_list_local_gguf_variants_skips_big_endian_sibling(tmp_path):
|
||||
(tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"x" * 100)
|
||||
(tmp_path / "model-Q4_K_M.gguf").write_bytes(b"y" * 10)
|
||||
|
||||
variants, has_vision = gguf.list_local_gguf_variants(str(tmp_path))
|
||||
|
||||
assert has_vision is False
|
||||
assert [(v.quant, v.filename, v.size_bytes) for v in variants] == [
|
||||
("Q4_K_M", "model-Q4_K_M.gguf", 10)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("repo_id", ["bert-base-uncased", "owner/repo"])
|
||||
def test_repo_id_validation_accepts_hf_repo_id_contract(repo_id):
|
||||
assert paths.is_valid_repo_id(repo_id)
|
||||
|
|
@ -304,6 +326,22 @@ def test_gguf_variant_requirements_include_split_files_and_preferred_mmproj():
|
|||
)
|
||||
|
||||
|
||||
def test_gguf_variant_requirements_skip_big_endian_sibling():
|
||||
requirements = gguf_variants._build_gguf_variant_requirements(
|
||||
[
|
||||
_sibling("model-Q4_K_M-be.gguf", 100, "main-be"),
|
||||
_sibling("model-Q4_K_M.gguf", 10, "main-le"),
|
||||
]
|
||||
)
|
||||
|
||||
req = requirements["q4_k_m"]
|
||||
|
||||
assert req.main_size_bytes == 10
|
||||
assert req.main_hashes == frozenset({"main-le"})
|
||||
assert req.main_filenames == frozenset({"model-Q4_K_M.gguf"})
|
||||
assert req.target_filenames == ("model-Q4_K_M.gguf",)
|
||||
|
||||
|
||||
def test_worker_gguf_variant_plan_matches_service_requirement(monkeypatch):
|
||||
siblings = [
|
||||
_sibling("model-Q4_K_M-00001-of-00002.gguf", 10, "main-a"),
|
||||
|
|
|
|||
|
|
@ -109,6 +109,33 @@ def is_gguf_filename(filename: str) -> bool:
|
|||
return filename.lower().endswith(".gguf")
|
||||
|
||||
|
||||
_BIG_ENDIAN_GGUF_FILENAME_RE = re.compile(r"(^|[-_])be(?:[._-]|$)", re.IGNORECASE)
|
||||
|
||||
|
||||
def is_big_endian_gguf_path(path: str, quant: str = "") -> bool:
|
||||
normalized = path.replace("\\", "/")
|
||||
name = normalized.rsplit("/", 1)[-1]
|
||||
stem = name.rsplit(".", 1)[0].lower()
|
||||
quant_key = quant.strip().lower()
|
||||
quant_index = stem.find(quant_key) if quant_key else -1
|
||||
parent = normalized.rsplit("/", 1)[0].lower() if "/" in normalized else ""
|
||||
quant_in_parent_only = (
|
||||
bool(parent)
|
||||
and quant_index < 0
|
||||
and (
|
||||
(quant_key and quant_key in parent)
|
||||
or (not quant_key and _GGUF_QUANT_RE.search(parent) is not None)
|
||||
)
|
||||
)
|
||||
for match in _BIG_ENDIAN_GGUF_FILENAME_RE.finditer(stem):
|
||||
if quant_index >= 0 and quant_index < match.start():
|
||||
return True
|
||||
tail = stem[match.end() :].lstrip("._-")
|
||||
if not tail or _GGUF_QUANT_RE.search(tail) is None:
|
||||
return not quant_in_parent_only
|
||||
return False
|
||||
|
||||
|
||||
# Cap recursive walks so a huge or system path cannot run unbounded.
|
||||
_MAX_LOCAL_SCAN_ENTRIES = 100_000
|
||||
|
||||
|
|
@ -143,7 +170,10 @@ def pick_best_gguf(filenames: list[str]) -> Optional[str]:
|
|||
gguf_files = [
|
||||
name
|
||||
for name in filenames
|
||||
if is_gguf_filename(name) and not is_mmproj_filename(name) and not is_mtp_drafter_path(name)
|
||||
if is_gguf_filename(name)
|
||||
and not is_mmproj_filename(name)
|
||||
and not is_mtp_drafter_path(name)
|
||||
and not is_big_endian_gguf_path(name, extract_quant_label(name))
|
||||
]
|
||||
if not gguf_files:
|
||||
return None
|
||||
|
|
@ -372,6 +402,8 @@ def list_gguf_variants(
|
|||
has_vision = True
|
||||
continue
|
||||
quant = extract_quant_label(filename)
|
||||
if is_big_endian_gguf_path(filename, quant):
|
||||
continue
|
||||
quant_totals[quant] = quant_totals.get(quant, 0) + int(getattr(sibling, "size", 0) or 0)
|
||||
quant_first_file.setdefault(quant, filename)
|
||||
|
||||
|
|
@ -424,6 +456,8 @@ def list_local_gguf_variants(directory: str) -> tuple[list[GgufVariantInfo], boo
|
|||
if is_mtp_drafter_path(rel):
|
||||
continue
|
||||
quant = extract_quant_label(rel)
|
||||
if is_big_endian_gguf_path(rel, quant):
|
||||
continue
|
||||
quant_totals[quant] = quant_totals.get(quant, 0) + size
|
||||
quant_first_file.setdefault(quant, rel)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import Optional, Sequence
|
|||
from hub.utils.download_manifest import ExpectedFile
|
||||
from hub.utils.gguf import (
|
||||
extract_quant_label,
|
||||
is_big_endian_gguf_path,
|
||||
is_gguf_filename,
|
||||
is_mmproj_filename,
|
||||
is_mtp_drafter_path,
|
||||
|
|
@ -68,6 +69,7 @@ def is_main_gguf_variant_path(path: str, variant: str) -> bool:
|
|||
is_gguf_filename(path)
|
||||
and not is_mmproj_filename(path)
|
||||
and not is_mtp_drafter_path(path)
|
||||
and not is_big_endian_gguf_path(path, variant)
|
||||
and extract_quant_label(path).lower() == variant.lower()
|
||||
)
|
||||
|
||||
|
|
@ -140,6 +142,8 @@ def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:
|
|||
if is_mmproj_filename(name) or is_mtp_drafter_path(name):
|
||||
continue
|
||||
quant = extract_quant_label(name).lower()
|
||||
if is_big_endian_gguf_path(name, quant):
|
||||
continue
|
||||
main.setdefault(quant, []).append(sibling)
|
||||
|
||||
plans: dict[str, GgufVariantPlan] = {}
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ try:
|
|||
from utils.models.model_config import (
|
||||
_pick_best_gguf,
|
||||
_extract_quant_label,
|
||||
_is_big_endian_gguf_path,
|
||||
is_audio_input_type,
|
||||
)
|
||||
from core.inference import get_inference_backend
|
||||
|
|
@ -112,6 +113,7 @@ except ImportError:
|
|||
from utils.models.model_config import (
|
||||
_pick_best_gguf,
|
||||
_extract_quant_label,
|
||||
_is_big_endian_gguf_path,
|
||||
is_audio_input_type,
|
||||
)
|
||||
from core.inference import get_inference_backend
|
||||
|
|
@ -2106,7 +2108,11 @@ async def get_gguf_variants(
|
|||
size = f.stat().st_size
|
||||
except OSError:
|
||||
continue # broken symlink / unreadable: skip
|
||||
q = _extract_quant_label(f.name).lower()
|
||||
rel = f.relative_to(snap).as_posix()
|
||||
q = _extract_quant_label(rel)
|
||||
if _is_big_endian_gguf_path(rel, q):
|
||||
continue
|
||||
q = q.lower()
|
||||
by_quant[q] = by_quant.get(q, 0) + size
|
||||
if by_quant:
|
||||
cached_bytes_by_quant_per_snapshot.append(by_quant)
|
||||
|
|
@ -2182,8 +2188,12 @@ async def get_gguf_download_progress(
|
|||
for f in _iter_gguf_paths(entry):
|
||||
if _is_mmproj_filename(f.name):
|
||||
continue
|
||||
fname = f.name.lower().replace("-", "").replace("_", "")
|
||||
if not variant_lower or variant_lower in fname:
|
||||
rel = f.relative_to(entry).as_posix()
|
||||
quant = _extract_quant_label(rel)
|
||||
if _is_big_endian_gguf_path(rel, quant):
|
||||
continue
|
||||
rel_key = rel.lower().replace("-", "").replace("_", "")
|
||||
if not variant_lower or variant_lower in rel_key:
|
||||
try:
|
||||
downloaded_bytes += f.stat().st_size
|
||||
except OSError:
|
||||
|
|
|
|||
|
|
@ -503,6 +503,58 @@ def test_gguf_variants_mmproj_does_not_mark_quant_downloaded(monkeypatch, tmp_pa
|
|||
assert flags["F16"] is False
|
||||
|
||||
|
||||
def test_gguf_variants_ignore_big_endian_siblings(monkeypatch, tmp_path):
|
||||
import huggingface_hub.constants as hf_constants
|
||||
|
||||
siblings = [
|
||||
SimpleNamespace(rfilename = "model-Q4_K_M-be.gguf", size = 100),
|
||||
SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"huggingface_hub.model_info",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(siblings = siblings),
|
||||
)
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
|
||||
|
||||
snap = tmp_path / "models--org--repo" / "snapshots" / "rev"
|
||||
snap.mkdir(parents = True)
|
||||
(snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 10)
|
||||
|
||||
result = asyncio.run(
|
||||
models_route.get_gguf_variants(
|
||||
repo_id = "org/repo", hf_token = None, current_subject = "test-user"
|
||||
)
|
||||
)
|
||||
|
||||
assert [(v.quant, v.filename, v.size_bytes, v.downloaded) for v in result.variants] == [
|
||||
("Q4_K_M", "model-Q4_K_M.gguf", 10, True)
|
||||
]
|
||||
|
||||
|
||||
def test_gguf_variants_cached_big_endian_does_not_satisfy_variant(monkeypatch, tmp_path):
|
||||
import huggingface_hub.constants as hf_constants
|
||||
|
||||
variants = [
|
||||
SimpleNamespace(filename = "model-Q4_K_M.gguf", quant = "Q4_K_M", size_bytes = 10),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
models_route, "list_gguf_variants", lambda repo_id, hf_token = None: (variants, False)
|
||||
)
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
|
||||
|
||||
snap = tmp_path / "models--org--repo" / "snapshots" / "rev"
|
||||
snap.mkdir(parents = True)
|
||||
(snap / "model-Q4_K_M-be.gguf").write_bytes(b"x" * 10)
|
||||
|
||||
result = asyncio.run(
|
||||
models_route.get_gguf_variants(
|
||||
repo_id = "org/repo", hf_token = None, current_subject = "test-user"
|
||||
)
|
||||
)
|
||||
|
||||
assert result.variants[0].downloaded is False
|
||||
|
||||
|
||||
def test_gguf_download_progress_excludes_mmproj(monkeypatch, tmp_path):
|
||||
"""A cached mmproj adapter must not count toward a same-label main
|
||||
variant's download progress (mmproj-F16 vs an F16 weight)."""
|
||||
|
|
@ -524,3 +576,45 @@ def test_gguf_download_progress_excludes_mmproj(monkeypatch, tmp_path):
|
|||
|
||||
assert result["downloaded_bytes"] == 0
|
||||
assert result["progress"] == 0
|
||||
|
||||
|
||||
def test_gguf_download_progress_excludes_big_endian_sibling(monkeypatch, tmp_path):
|
||||
import huggingface_hub.constants as hf_constants
|
||||
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
|
||||
snap = tmp_path / "models--org--repo" / "snapshots" / "rev"
|
||||
snap.mkdir(parents = True)
|
||||
(snap / "model-Q4_K_M-be.gguf").write_bytes(b"y" * 20_000)
|
||||
|
||||
result = asyncio.run(
|
||||
models_route.get_gguf_download_progress(
|
||||
repo_id = "org/repo",
|
||||
variant = "Q4_K_M",
|
||||
expected_bytes = 20_000,
|
||||
current_subject = "test-user",
|
||||
)
|
||||
)
|
||||
|
||||
assert result["downloaded_bytes"] == 0
|
||||
assert result["progress"] == 0
|
||||
|
||||
|
||||
def test_gguf_download_progress_counts_quant_subdir(monkeypatch, tmp_path):
|
||||
import huggingface_hub.constants as hf_constants
|
||||
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
|
||||
snap = tmp_path / "models--org--repo" / "snapshots" / "rev" / "Q4_K_M"
|
||||
snap.mkdir(parents = True)
|
||||
(snap / "foo.gguf").write_bytes(b"x" * 20_000)
|
||||
|
||||
result = asyncio.run(
|
||||
models_route.get_gguf_download_progress(
|
||||
repo_id = "org/repo",
|
||||
variant = "Q4_K_M",
|
||||
expected_bytes = 20_000,
|
||||
current_subject = "test-user",
|
||||
)
|
||||
)
|
||||
|
||||
assert result["downloaded_bytes"] == 20_000
|
||||
assert result["progress"] == 1.0
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ def _install_lightweight_backend_stubs(monkeypatch):
|
|||
utils_model_config = types.ModuleType("utils.models.model_config")
|
||||
utils_model_config._pick_best_gguf = lambda variants: variants[0] if variants else None
|
||||
utils_model_config._extract_quant_label = lambda value: value
|
||||
utils_model_config._is_big_endian_gguf_path = lambda *args, **kwargs: False
|
||||
utils_model_config.is_audio_input_type = lambda *args, **kwargs: None
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
|
|
|
|||
|
|
@ -83,6 +83,23 @@ def test_detects_gguf_in_directory(tmp_path):
|
|||
assert result.endswith("model-Q4_K_M.gguf")
|
||||
|
||||
|
||||
def test_directory_auto_detect_ignores_big_endian_sibling(tmp_path):
|
||||
be = tmp_path / "model-Q4_K_M-be.gguf"
|
||||
be.write_bytes(b"x" * 100)
|
||||
target = tmp_path / "model-Q4_K_M.gguf"
|
||||
target.write_bytes(b"y" * 10)
|
||||
|
||||
result = detect_gguf_model(str(tmp_path))
|
||||
assert result == str(target.resolve())
|
||||
|
||||
|
||||
def test_direct_big_endian_file_is_not_detected(tmp_path):
|
||||
gguf = tmp_path / "model-Q4_K_M-be.gguf"
|
||||
gguf.write_bytes(b"")
|
||||
|
||||
assert detect_gguf_model(str(gguf)) is None
|
||||
|
||||
|
||||
def test_directory_named_like_gguf_scans_inside(tmp_path):
|
||||
"""A directory named *.gguf resolves the real .gguf inside, not itself."""
|
||||
gguf_dir = tmp_path / "mymodel.gguf"
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ from huggingface_hub import constants as hf_constants
|
|||
|
||||
from core.inference.llama_cpp import (
|
||||
LlamaCppBackend,
|
||||
_gguf_files_for_variant,
|
||||
_hf_offline_if_dns_dead,
|
||||
_probe_dns_dead,
|
||||
)
|
||||
|
|
@ -130,6 +131,155 @@ def clean_offline_env(monkeypatch):
|
|||
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
|
||||
|
||||
|
||||
class TestGgufVariantFileResolution:
|
||||
def test_prefers_exact_unknown_variant_over_big_endian_sibling(self):
|
||||
files = [
|
||||
"tinyllamas/stories260K-be.gguf",
|
||||
"tinyllamas/stories260K-infill.gguf",
|
||||
"tinyllamas/stories260K.gguf",
|
||||
]
|
||||
|
||||
assert _gguf_files_for_variant(files, "stories260K") == ["tinyllamas/stories260K.gguf"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"big_endian_path",
|
||||
[
|
||||
"model-Q4_K_M-be.gguf",
|
||||
"model-Q4_K_M_be.gguf",
|
||||
"model-Q4_K_M_be_infill.gguf",
|
||||
r"nested\model-Q4_K_M_be.gguf",
|
||||
],
|
||||
)
|
||||
def test_filters_big_endian_known_quant_before_exact_match(self, big_endian_path):
|
||||
files = [
|
||||
big_endian_path,
|
||||
"model-Q4_K_M.gguf",
|
||||
]
|
||||
|
||||
assert _gguf_files_for_variant(files, "Q4_K_M") == ["model-Q4_K_M.gguf"]
|
||||
|
||||
def test_keeps_model_name_be_token_before_quant(self):
|
||||
files = [
|
||||
"foo-be-Q4_K_M.gguf",
|
||||
]
|
||||
|
||||
assert _gguf_files_for_variant(files, "Q4_K_M") == ["foo-be-Q4_K_M.gguf"]
|
||||
|
||||
def test_keeps_model_name_be_token_with_quant_subdir(self):
|
||||
files = [
|
||||
"Q4_K_M/foo-be.gguf",
|
||||
]
|
||||
|
||||
assert _gguf_files_for_variant(files, "Q4_K_M") == ["Q4_K_M/foo-be.gguf"]
|
||||
|
||||
def test_empty_variant_filters_big_endian_files(self):
|
||||
files = [
|
||||
"model-Q4_K_M-be.gguf",
|
||||
"model-Q4_K_M.gguf",
|
||||
]
|
||||
|
||||
assert _gguf_files_for_variant(files, "") == ["model-Q4_K_M.gguf"]
|
||||
|
||||
def test_remote_listing_skips_big_endian_quant_sibling(self, monkeypatch, clean_offline_env):
|
||||
siblings = [
|
||||
_types.SimpleNamespace(rfilename = "model-Q4_K_M-be.gguf", size = 100),
|
||||
_types.SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"huggingface_hub.model_info",
|
||||
lambda *_args, **_kwargs: _types.SimpleNamespace(siblings = siblings),
|
||||
)
|
||||
|
||||
variants, has_vision = list_gguf_variants("org/repo")
|
||||
|
||||
assert has_vision is False
|
||||
assert [(v.quant, v.filename, v.size_bytes) for v in variants] == [
|
||||
("Q4_K_M", "model-Q4_K_M.gguf", 10)
|
||||
]
|
||||
|
||||
def test_download_uses_exact_variant_label(self, monkeypatch, tmp_path):
|
||||
backend = LlamaCppBackend()
|
||||
downloaded: list[str] = []
|
||||
|
||||
def fake_get_paths_info(
|
||||
_repo_id,
|
||||
paths,
|
||||
token = None,
|
||||
):
|
||||
return [_types.SimpleNamespace(path = path, size = 1) for path in paths if path is not None]
|
||||
|
||||
def fake_download(
|
||||
*,
|
||||
repo_id,
|
||||
filename,
|
||||
token = None,
|
||||
):
|
||||
downloaded.append(filename)
|
||||
return f"/fake/{repo_id}/{filename}"
|
||||
|
||||
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
|
||||
with (
|
||||
patch(
|
||||
"huggingface_hub.list_repo_files",
|
||||
lambda *_a, **_k: [
|
||||
"tinyllamas/stories260K-be.gguf",
|
||||
"tinyllamas/stories260K-infill.gguf",
|
||||
"tinyllamas/stories260K.gguf",
|
||||
],
|
||||
),
|
||||
patch("huggingface_hub.get_paths_info", fake_get_paths_info),
|
||||
patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None),
|
||||
patch("huggingface_hub.hf_hub_download", fake_download),
|
||||
):
|
||||
out = backend._download_gguf(
|
||||
hf_repo = "ggml-org/models",
|
||||
hf_variant = "stories260K",
|
||||
)
|
||||
|
||||
assert downloaded == ["tinyllamas/stories260K.gguf"]
|
||||
assert out == "/fake/ggml-org/models/tinyllamas/stories260K.gguf"
|
||||
|
||||
def test_download_includes_uppercase_split_gguf_shards(self, monkeypatch, tmp_path):
|
||||
backend = LlamaCppBackend()
|
||||
downloaded: list[str] = []
|
||||
|
||||
files = [
|
||||
"model-Q4_K_M-00001-of-00002.GGUF",
|
||||
"model-Q4_K_M-00002-of-00002.GGUF",
|
||||
]
|
||||
|
||||
def fake_get_paths_info(
|
||||
_repo_id,
|
||||
paths,
|
||||
token = None,
|
||||
):
|
||||
return [_types.SimpleNamespace(path = path, size = 1) for path in paths if path is not None]
|
||||
|
||||
def fake_download(
|
||||
*,
|
||||
repo_id,
|
||||
filename,
|
||||
token = None,
|
||||
):
|
||||
downloaded.append(filename)
|
||||
return f"/fake/{repo_id}/{filename}"
|
||||
|
||||
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
|
||||
with (
|
||||
patch("huggingface_hub.list_repo_files", lambda *_a, **_k: files),
|
||||
patch("huggingface_hub.get_paths_info", fake_get_paths_info),
|
||||
patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None),
|
||||
patch("huggingface_hub.hf_hub_download", fake_download),
|
||||
):
|
||||
out = backend._download_gguf(
|
||||
hf_repo = "org/repo",
|
||||
hf_variant = "Q4_K_M",
|
||||
)
|
||||
|
||||
assert downloaded == files
|
||||
assert out == "/fake/org/repo/model-Q4_K_M-00001-of-00002.GGUF"
|
||||
|
||||
|
||||
def _siblings(items: dict[str, int]):
|
||||
"""Mock ``hf_model_info(...).siblings`` payload."""
|
||||
return _types.SimpleNamespace(
|
||||
|
|
@ -268,6 +418,22 @@ class TestDetectGgufFromCache:
|
|||
out == "BF16/foo.gguf"
|
||||
), f"subdir-only layout must resolve to relative path, got {out}"
|
||||
|
||||
def test_subdir_quant_keeps_be_model_name_token(self, hf_cache):
|
||||
_build_cache(
|
||||
hf_cache,
|
||||
"unsloth/a",
|
||||
{"Q4_K_M/foo-be.gguf": 1},
|
||||
)
|
||||
assert _detect_gguf_from_hf_cache("unsloth/a") == "Q4_K_M/foo-be.gguf"
|
||||
|
||||
def test_big_endian_only_cache_is_not_detected(self, hf_cache):
|
||||
_build_cache(
|
||||
hf_cache,
|
||||
"unsloth/a",
|
||||
{"model-Q4_K_M-be.gguf": 1},
|
||||
)
|
||||
assert _detect_gguf_from_hf_cache("unsloth/a") is None
|
||||
|
||||
def test_returns_none_when_no_gguf(self, hf_cache):
|
||||
_build_cache(hf_cache, "unsloth/a", {"README.md": 10})
|
||||
assert _detect_gguf_from_hf_cache("unsloth/a") is None
|
||||
|
|
@ -298,6 +464,17 @@ class TestDetectGgufModelRemoteOffline:
|
|||
out = detect_gguf_model_remote("unsloth/a")
|
||||
assert out == "a-Q4_K_M.gguf"
|
||||
|
||||
def test_remote_big_endian_only_repo_is_not_detected(self, clean_offline_env, monkeypatch):
|
||||
siblings = [
|
||||
_types.SimpleNamespace(rfilename = "model-Q4_K_M-be.gguf"),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"huggingface_hub.model_info",
|
||||
lambda *_args, **_kwargs: _types.SimpleNamespace(siblings = siblings),
|
||||
)
|
||||
|
||||
assert detect_gguf_model_remote("unsloth/a") is None
|
||||
|
||||
def test_repository_not_found_does_not_consult_cache(self, hf_cache, clean_offline_env):
|
||||
# Cache has a file but the API says the repo is gone.
|
||||
_build_cache(hf_cache, "unsloth/a", {"a-Q4_K_M.gguf": 1})
|
||||
|
|
@ -566,6 +743,49 @@ class TestListLocalGgufVariantsSubdir:
|
|||
assert out is not None
|
||||
assert Path(out).name == "foo.gguf"
|
||||
|
||||
def test_find_local_gguf_by_variant_ignores_big_endian_sibling(self, tmp_path):
|
||||
from utils.models.model_config import _find_local_gguf_by_variant
|
||||
|
||||
(tmp_path / "config.json").write_text("{}")
|
||||
(tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"\0" * 10)
|
||||
target = tmp_path / "model-Q4_K_M.gguf"
|
||||
target.write_bytes(b"\0" * 20)
|
||||
|
||||
out = _find_local_gguf_by_variant(str(tmp_path), "Q4_K_M")
|
||||
assert out == str(target.resolve())
|
||||
|
||||
def test_find_local_gguf_by_variant_skips_big_endian_only_match(self, tmp_path):
|
||||
from utils.models.model_config import _find_local_gguf_by_variant
|
||||
|
||||
(tmp_path / "config.json").write_text("{}")
|
||||
(tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"\0" * 10)
|
||||
|
||||
assert _find_local_gguf_by_variant(str(tmp_path), "Q4_K_M") is None
|
||||
|
||||
def test_model_config_variant_ignores_big_endian_sibling(self, tmp_path):
|
||||
from utils.models.model_config import ModelConfig
|
||||
|
||||
(tmp_path / "config.json").write_text("{}")
|
||||
(tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"\0" * 10)
|
||||
target = tmp_path / "model-Q4_K_M.gguf"
|
||||
target.write_bytes(b"\0" * 20)
|
||||
|
||||
config = ModelConfig.from_identifier(str(tmp_path), gguf_variant = "Q4_K_M")
|
||||
assert config is not None
|
||||
assert config.gguf_file == str(target.resolve())
|
||||
|
||||
def test_local_variant_listing_keeps_subdir_be_model_name_token(self, tmp_path):
|
||||
from utils.models.model_config import list_local_gguf_variants
|
||||
|
||||
(tmp_path / "config.json").write_text("{}")
|
||||
(tmp_path / "Q4_K_M").mkdir()
|
||||
(tmp_path / "Q4_K_M" / "foo-be.gguf").write_bytes(b"\0" * 10)
|
||||
|
||||
variants, _ = list_local_gguf_variants(str(tmp_path))
|
||||
assert [(v.quant, v.filename, v.size_bytes) for v in variants] == [
|
||||
("Q4_K_M", "Q4_K_M/foo-be.gguf", 10)
|
||||
]
|
||||
|
||||
|
||||
class TestListGgufVariantsPermanentErrors:
|
||||
"""Permanent HF errors must surface; cache fallback only on transient."""
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from utils.models.gguf_metadata import (
|
|||
import structlog
|
||||
from loggers import get_logger
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
|
@ -1235,7 +1236,9 @@ def detect_gguf_model(path: str) -> Optional[str]:
|
|||
# (-m drafter --model-draft drafter). Include the immediate parent
|
||||
# dir so the MTP/ subdir copies are caught -- the basename alone
|
||||
# (...-MTP.gguf) doesn't match the predicate's mtp- prefix.
|
||||
if _is_mmproj(p.name) or _is_mtp_drafter(f"{p.parent.name}/{p.name}"):
|
||||
rel = f"{p.parent.name}/{p.name}"
|
||||
quant = _extract_quant_label(rel)
|
||||
if _is_mmproj(p.name) or _is_mtp_drafter(rel) or _is_big_endian_gguf_path(rel, quant):
|
||||
return None
|
||||
# Extension is authoritative: don't gate on is_file()/exists(), which
|
||||
# can fail in the Windows lock window after llama-server is killed.
|
||||
|
|
@ -1249,15 +1252,18 @@ def detect_gguf_model(path: str) -> Optional[str]:
|
|||
|
||||
# Case 2: directory containing .gguf files (skip mmproj / MTP drafter)
|
||||
if p.is_dir():
|
||||
gguf_files = sorted(
|
||||
(
|
||||
f
|
||||
for f in _iter_gguf_files(p)
|
||||
if not _is_mmproj(f.name) and not _is_mtp_drafter(f"{f.parent.name}/{f.name}")
|
||||
),
|
||||
key = lambda f: f.stat().st_size,
|
||||
reverse = True,
|
||||
)
|
||||
gguf_files = []
|
||||
for f in _iter_gguf_files(p):
|
||||
context_rel = f"{f.parent.name}/{f.name}"
|
||||
quant = _extract_quant_label(context_rel)
|
||||
if (
|
||||
_is_mmproj(f.name)
|
||||
or _is_mtp_drafter(context_rel)
|
||||
or _is_big_endian_gguf_path(context_rel, quant)
|
||||
):
|
||||
continue
|
||||
gguf_files.append(f)
|
||||
gguf_files.sort(key = lambda f: f.stat().st_size, reverse = True)
|
||||
if gguf_files:
|
||||
return str(gguf_files[0].resolve())
|
||||
|
||||
|
|
@ -1383,6 +1389,44 @@ def _extract_quant_label(filename: str) -> str:
|
|||
return stem.split("-")[-1]
|
||||
|
||||
|
||||
_BIG_ENDIAN_GGUF_FILENAME_RE = re.compile(r"(^|[-_])be(?:[._-]|$)", re.IGNORECASE)
|
||||
_GGUF_KNOWN_QUANT_RE = re.compile(
|
||||
r"(UD-)?"
|
||||
r"(MXFP[0-9]+(?:_[A-Z0-9]+)*"
|
||||
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
|
||||
r"|TQ[0-9]+_[0-9]+"
|
||||
r"|Q[0-9]+_K_[A-Z]+"
|
||||
r"|Q[0-9]+_[0-9]+"
|
||||
r"|Q[0-9]+_K"
|
||||
r"|BF16|F16|F32)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_big_endian_gguf_path(path: str, quant: str = "") -> bool:
|
||||
normalized = path.replace("\\", "/")
|
||||
name = normalized.rsplit("/", 1)[-1]
|
||||
stem = name.rsplit(".", 1)[0].lower()
|
||||
quant_key = quant.strip().lower()
|
||||
quant_index = stem.find(quant_key) if quant_key else -1
|
||||
parent = normalized.rsplit("/", 1)[0].lower() if "/" in normalized else ""
|
||||
quant_in_parent_only = (
|
||||
bool(parent)
|
||||
and quant_index < 0
|
||||
and (
|
||||
(quant_key and quant_key in parent)
|
||||
or (not quant_key and _GGUF_KNOWN_QUANT_RE.search(parent) is not None)
|
||||
)
|
||||
)
|
||||
for match in _BIG_ENDIAN_GGUF_FILENAME_RE.finditer(stem):
|
||||
if quant_index >= 0 and quant_index < match.start():
|
||||
return True
|
||||
tail = stem[match.end() :].lstrip("._-")
|
||||
if not tail or _GGUF_KNOWN_QUANT_RE.search(tail) is None:
|
||||
return not quant_in_parent_only
|
||||
return False
|
||||
|
||||
|
||||
def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str:
|
||||
"""Directory to scan upward from for local GGUF companion files."""
|
||||
import re
|
||||
|
|
@ -1522,6 +1566,8 @@ def list_gguf_variants(
|
|||
continue
|
||||
|
||||
quant = _extract_quant_label(fname)
|
||||
if _is_big_endian_gguf_path(fname, quant):
|
||||
continue
|
||||
quant_totals[quant] = quant_totals.get(quant, 0) + size
|
||||
if quant not in quant_first_file:
|
||||
quant_first_file[quant] = fname
|
||||
|
|
@ -1598,6 +1644,8 @@ def list_local_gguf_variants(directory: str) -> tuple[list[GgufVariantInfo], boo
|
|||
if _is_mtp_drafter(rel):
|
||||
continue
|
||||
quant = _extract_quant_label(rel)
|
||||
if _is_big_endian_gguf_path(rel, quant):
|
||||
continue
|
||||
quant_totals[quant] = quant_totals.get(quant, 0) + size
|
||||
if quant not in quant_first_file:
|
||||
quant_first_file[quant] = rel
|
||||
|
|
@ -1630,13 +1678,16 @@ def _find_local_gguf_by_variant(directory: str, variant: str) -> Optional[str]:
|
|||
# ``BF16/foo-BF16-00001-of-00002.gguf``) are found. Match the relative
|
||||
# path so the quant label can come from the dir name when the basename
|
||||
# omits it.
|
||||
matches = sorted(
|
||||
f
|
||||
for f in _iter_gguf_files(p, recursive = True)
|
||||
if not _is_mmproj(f.name)
|
||||
and not _is_mtp_drafter(f.relative_to(p).as_posix())
|
||||
and _extract_quant_label(f.relative_to(p).as_posix()) == variant
|
||||
)
|
||||
matches = []
|
||||
for f in _iter_gguf_files(p, recursive = True):
|
||||
rel = f.relative_to(p).as_posix()
|
||||
if _is_mmproj(f.name) or _is_mtp_drafter(rel):
|
||||
continue
|
||||
quant = _extract_quant_label(rel)
|
||||
if quant != variant or _is_big_endian_gguf_path(rel, quant):
|
||||
continue
|
||||
matches.append(f)
|
||||
matches.sort()
|
||||
if matches:
|
||||
return str(matches[0].resolve())
|
||||
return None
|
||||
|
|
@ -1649,11 +1700,13 @@ def _detect_gguf_from_hf_cache(repo_id: str) -> Optional[str]:
|
|||
the projector cannot route it as the main model.
|
||||
"""
|
||||
for snap in _iter_hf_cache_snapshots(repo_id):
|
||||
rel_files = [
|
||||
rel
|
||||
for f in _iter_gguf_files(snap, recursive = True)
|
||||
if not _is_mtp_drafter(rel := f.relative_to(snap).as_posix()) and not _is_mmproj(f.name)
|
||||
]
|
||||
rel_files = []
|
||||
for f in _iter_gguf_files(snap, recursive = True):
|
||||
rel = f.relative_to(snap).as_posix()
|
||||
quant = _extract_quant_label(rel)
|
||||
if _is_mmproj(f.name) or _is_mtp_drafter(rel) or _is_big_endian_gguf_path(rel, quant):
|
||||
continue
|
||||
rel_files.append(rel)
|
||||
if rel_files:
|
||||
return _pick_best_gguf(rel_files)
|
||||
return None
|
||||
|
|
@ -1678,7 +1731,19 @@ def detect_gguf_model_remote(repo_id: str, hf_token: Optional[str] = None) -> Op
|
|||
for attempt in range(3):
|
||||
try:
|
||||
info = hf_model_info(repo_id, token = hf_token)
|
||||
repo_files = [s.rfilename for s in info.siblings]
|
||||
repo_files = []
|
||||
for sibling in info.siblings:
|
||||
fname = sibling.rfilename
|
||||
if not fname.lower().endswith(".gguf"):
|
||||
continue
|
||||
quant = _extract_quant_label(fname)
|
||||
if (
|
||||
_is_mmproj(fname)
|
||||
or _is_mtp_drafter(fname)
|
||||
or _is_big_endian_gguf_path(fname, quant)
|
||||
):
|
||||
continue
|
||||
repo_files.append(fname)
|
||||
return _pick_best_gguf(repo_files)
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import {
|
|||
import { useExternalProvidersStore } from "../stores/external-providers-store";
|
||||
import { isMultimodalResponse } from "../types/api";
|
||||
import type {
|
||||
GgufVariantDetail,
|
||||
OpenAIChatCompletionsRequest,
|
||||
OpenAIChatMessage,
|
||||
OpenAIMessageContent,
|
||||
|
|
@ -1141,6 +1142,45 @@ function waitForModelReady(abortSignal?: AbortSignal): Promise<void> {
|
|||
*/
|
||||
// Cap cascade so broken cached repos can't spam /api/inference/load.
|
||||
const MAX_AUTO_LOAD_ATTEMPTS = 3;
|
||||
const BIG_ENDIAN_GGUF_FILENAME_RE = /(^|[-_])be(?:[._-]|$)/gi;
|
||||
const GGUF_KNOWN_QUANT_RE =
|
||||
/(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|Q[0-9]+_[0-9]+|Q[0-9]+_K|BF16|F16|F32)/i;
|
||||
|
||||
function hasBigEndianGgufMarker(filename: string, quant?: string | null): boolean {
|
||||
const normalized = filename.replace(/\\/g, "/").toLowerCase();
|
||||
const separatorIndex = normalized.lastIndexOf("/");
|
||||
const basename = separatorIndex >= 0 ? normalized.slice(separatorIndex + 1) : normalized;
|
||||
const parent = separatorIndex >= 0 ? normalized.slice(0, separatorIndex) : "";
|
||||
const stem = basename.replace(/\.[^.]*$/, "");
|
||||
const quantKey = quant?.trim().toLowerCase() || "";
|
||||
const quantIndex = quantKey ? stem.indexOf(quantKey) : -1;
|
||||
const quantInParentOnly =
|
||||
!!parent &&
|
||||
quantIndex < 0 &&
|
||||
((!!quantKey && parent.includes(quantKey)) ||
|
||||
(!quantKey && GGUF_KNOWN_QUANT_RE.test(parent)));
|
||||
for (const match of stem.matchAll(BIG_ENDIAN_GGUF_FILENAME_RE)) {
|
||||
if (quantIndex >= 0 && quantIndex < (match.index ?? 0)) {
|
||||
return true;
|
||||
}
|
||||
const tail = stem.slice((match.index ?? 0) + match[0].length).replace(/^[._-]+/, "");
|
||||
if (!tail || !GGUF_KNOWN_QUANT_RE.test(tail)) {
|
||||
return !quantInParentOnly;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isAutoLoadableGgufVariant(variant: GgufVariantDetail | null): boolean {
|
||||
if (!variant?.filename) {
|
||||
return false;
|
||||
}
|
||||
const filename = variant.filename.trim().toLowerCase();
|
||||
if (!filename) {
|
||||
return false;
|
||||
}
|
||||
return !hasBigEndianGgufMarker(filename, variant.quant);
|
||||
}
|
||||
|
||||
async function autoLoadSmallestModel(): Promise<{
|
||||
loaded: boolean;
|
||||
|
|
@ -1195,7 +1235,7 @@ async function autoLoadSmallestModel(): Promise<{
|
|||
try {
|
||||
const variants = await listGgufVariants(repo.repo_id);
|
||||
const downloaded = variants.variants
|
||||
.filter((v) => v.downloaded)
|
||||
.filter((v) => v.downloaded && isAutoLoadableGgufVariant(v))
|
||||
.sort((a, b) => a.size_bytes - b.size_bytes);
|
||||
if (downloaded.length > 0) {
|
||||
const variant = downloaded[0];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue