diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index bd6bdad8c6..0f4abe04f5 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -247,8 +247,8 @@ def _should_suppress_forced_no_tool_output(text: str) -> bool: # ── Pre-compiled patterns for GGUF shard detection ─────────── -_SHARD_FULL_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$") -_SHARD_RE = re.compile(r"^(.*)-\d{5}-of-\d{5}\.gguf$") +_SHARD_FULL_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE) +_SHARD_RE = re.compile(r"^(.*)-\d{5}-of-\d{5}\.gguf$", re.IGNORECASE) # ── Sliding-window-pattern resolver ─────────────────────────── @@ -626,6 +626,99 @@ def _is_companion_gguf_path(path: str) -> bool: return name.startswith("mtp-") or "/mtp/" in f"/{p}" +_BIG_ENDIAN_GGUF_FILENAME_RE = re.compile(r"(^|[-_])be(?:[._-]|$)", re.IGNORECASE) +_GGUF_KNOWN_QUANT_RE = re.compile( + r"(UD-)?" + r"(MXFP[0-9]+(?:_[A-Z0-9]+)*" + r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?" + r"|TQ[0-9]+_[0-9]+" + r"|Q[0-9]+_K_[A-Z]+" + r"|Q[0-9]+_[0-9]+" + r"|Q[0-9]+_K" + r"|BF16|F16|F32)", + re.IGNORECASE, +) + + +def _is_big_endian_gguf_path(path: str, variant_key: str = "") -> bool: + normalized = path.replace("\\", "/") + name = normalized.rsplit("/", 1)[-1] + stem = name.rsplit(".", 1)[0].lower() + variant_key = variant_key.strip().lower() + variant_index = stem.find(variant_key) if variant_key else -1 + parent = normalized.rsplit("/", 1)[0].lower() if "/" in normalized else "" + variant_in_parent_only = ( + bool(parent) + and variant_index < 0 + and ( + (variant_key and variant_key in parent) + or (not variant_key and _GGUF_KNOWN_QUANT_RE.search(parent) is not None) + ) + ) + for match in _BIG_ENDIAN_GGUF_FILENAME_RE.finditer(stem): + if variant_index >= 0 and variant_index < match.start(): + return True + tail = stem[match.end() :].lstrip("._-") + if not tail or _GGUF_KNOWN_QUANT_RE.search(tail) is None: + return not variant_in_parent_only + return False + + +def _gguf_snapshot_files(snapshot: Path) -> list[str]: + return [ + p.relative_to(snapshot).as_posix() + for p in snapshot.rglob("*") + if p.is_file() and p.name.lower().endswith(".gguf") + ] + + +def _gguf_extra_shards(files: Iterable[str], first_shard: str) -> list[str]: + m = _SHARD_FULL_RE.match(first_shard) + if not m: + return [] + prefix = m.group(1) + total = m.group(3) + sibling_pat = re.compile( + r"^" + re.escape(prefix) + r"-\d{5}-of-" + re.escape(total) + r"\.gguf$", + re.IGNORECASE, + ) + return sorted(f for f in files if f != first_shard and sibling_pat.match(f)) + + +def _gguf_files_for_variant(files: Iterable[str], variant: str) -> list[str]: + """Return main GGUF files matching a requested variant. + + Prefer exact quant-label matches over loose substring matches so a request + for ``stories260K`` does not resolve to ``stories260K-be.gguf``. + """ + variant_key = variant.strip().lower() + main_files = [ + f + for f in files + if f.lower().endswith(".gguf") + and not _is_companion_gguf_path(f) + and not _is_big_endian_gguf_path(f, variant_key) + ] + if not variant_key: + return sorted(main_files) + + try: + from utils.models.model_config import _extract_quant_label + except Exception: + _extract_quant_label = None + + if _extract_quant_label is not None: + try: + exact = sorted(f for f in main_files if _extract_quant_label(f).lower() == variant_key) + if exact: + return exact + except Exception as e: + logger.warning("Failed to extract GGUF quant labels: %s", e) + + boundary = re.compile(r"(? %s from local HF cache", hf_variant, @@ -3050,10 +3112,11 @@ class LlamaCppBackend: _m = _SHARD_RE.match(gguf_filename) _prefix = _m.group(1) if _m else None if _prefix: + prefix_lower = _prefix.lower() gguf_extra_shards = sorted( f for f in all_gguf_files - if f.startswith(_prefix) + if f.lower().startswith(prefix_lower) and f != gguf_filename and not _is_companion_gguf_path(f) ) @@ -3138,7 +3201,7 @@ class LlamaCppBackend: try: from utils.models.model_config import _iter_hf_cache_snapshots for snap in _iter_hf_cache_snapshots(hf_repo): - rel_files = [p.relative_to(snap).as_posix() for p in snap.rglob("*.gguf")] + rel_files = _gguf_snapshot_files(snap) target = pick(rel_files) if target is not None: logger.info("Resolved %s %s from local HF cache", label, target) diff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.py index 0b4671c6d2..095e3de641 100644 --- a/studio/backend/hub/services/models/gguf_variants.py +++ b/studio/backend/hub/services/models/gguf_variants.py @@ -26,6 +26,7 @@ from hub.utils.hf_cache_state import ( from hub.utils.gguf import ( extract_quant_label, iter_hf_cache_snapshots, + is_big_endian_gguf_path, list_gguf_variants, list_gguf_variants_from_hf_cache, list_local_gguf_variants, @@ -482,7 +483,10 @@ async def get_gguf_variants_response( by_filename[key] = max(by_filename.get(key, 0), size) if _is_mmproj_filename(f.name) or _is_mtp_drafter_path(rel): continue - q = extract_quant_label(rel).lower() + q = extract_quant_label(rel) + if is_big_endian_gguf_path(rel, q): + continue + q = q.lower() by_quant[q] = by_quant.get(q, 0) + size if by_filename: cached_filenames_by_snapshot.append(by_filename) diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.py index 44ab4b80d0..df97b9cf97 100644 --- a/studio/backend/hub/tests/test_model_services.py +++ b/studio/backend/hub/tests/test_model_services.py @@ -78,6 +78,28 @@ class TestExtractQuantToken: assert labels == {"Q4_K_M", "Q8_0"} +def test_big_endian_detection_ignores_model_name_be_token(): + assert gguf.is_big_endian_gguf_path("model-Q4_K_M-be.gguf", "Q4_K_M") + assert gguf.is_big_endian_gguf_path("model-Q4_K_M_be_infill.gguf", "Q4_K_M") + assert not gguf.is_big_endian_gguf_path("foo-be-Q4_K_M.gguf", "Q4_K_M") + assert not gguf.is_big_endian_gguf_path("Q4_K_M/foo-be.gguf", "Q4_K_M") + assert gguf.pick_best_gguf(["model-Q4_K_M-be.gguf", "model-Q4_K_M.gguf"]) == ( + "model-Q4_K_M.gguf" + ) + + +def test_list_local_gguf_variants_skips_big_endian_sibling(tmp_path): + (tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"x" * 100) + (tmp_path / "model-Q4_K_M.gguf").write_bytes(b"y" * 10) + + variants, has_vision = gguf.list_local_gguf_variants(str(tmp_path)) + + assert has_vision is False + assert [(v.quant, v.filename, v.size_bytes) for v in variants] == [ + ("Q4_K_M", "model-Q4_K_M.gguf", 10) + ] + + @pytest.mark.parametrize("repo_id", ["bert-base-uncased", "owner/repo"]) def test_repo_id_validation_accepts_hf_repo_id_contract(repo_id): assert paths.is_valid_repo_id(repo_id) @@ -304,6 +326,22 @@ def test_gguf_variant_requirements_include_split_files_and_preferred_mmproj(): ) +def test_gguf_variant_requirements_skip_big_endian_sibling(): + requirements = gguf_variants._build_gguf_variant_requirements( + [ + _sibling("model-Q4_K_M-be.gguf", 100, "main-be"), + _sibling("model-Q4_K_M.gguf", 10, "main-le"), + ] + ) + + req = requirements["q4_k_m"] + + assert req.main_size_bytes == 10 + assert req.main_hashes == frozenset({"main-le"}) + assert req.main_filenames == frozenset({"model-Q4_K_M.gguf"}) + assert req.target_filenames == ("model-Q4_K_M.gguf",) + + def test_worker_gguf_variant_plan_matches_service_requirement(monkeypatch): siblings = [ _sibling("model-Q4_K_M-00001-of-00002.gguf", 10, "main-a"), diff --git a/studio/backend/hub/utils/gguf.py b/studio/backend/hub/utils/gguf.py index c9b3ce97e3..4eb8180628 100644 --- a/studio/backend/hub/utils/gguf.py +++ b/studio/backend/hub/utils/gguf.py @@ -109,6 +109,33 @@ def is_gguf_filename(filename: str) -> bool: return filename.lower().endswith(".gguf") +_BIG_ENDIAN_GGUF_FILENAME_RE = re.compile(r"(^|[-_])be(?:[._-]|$)", re.IGNORECASE) + + +def is_big_endian_gguf_path(path: str, quant: str = "") -> bool: + normalized = path.replace("\\", "/") + name = normalized.rsplit("/", 1)[-1] + stem = name.rsplit(".", 1)[0].lower() + quant_key = quant.strip().lower() + quant_index = stem.find(quant_key) if quant_key else -1 + parent = normalized.rsplit("/", 1)[0].lower() if "/" in normalized else "" + quant_in_parent_only = ( + bool(parent) + and quant_index < 0 + and ( + (quant_key and quant_key in parent) + or (not quant_key and _GGUF_QUANT_RE.search(parent) is not None) + ) + ) + for match in _BIG_ENDIAN_GGUF_FILENAME_RE.finditer(stem): + if quant_index >= 0 and quant_index < match.start(): + return True + tail = stem[match.end() :].lstrip("._-") + if not tail or _GGUF_QUANT_RE.search(tail) is None: + return not quant_in_parent_only + return False + + # Cap recursive walks so a huge or system path cannot run unbounded. _MAX_LOCAL_SCAN_ENTRIES = 100_000 @@ -143,7 +170,10 @@ def pick_best_gguf(filenames: list[str]) -> Optional[str]: gguf_files = [ name for name in filenames - if is_gguf_filename(name) and not is_mmproj_filename(name) and not is_mtp_drafter_path(name) + if is_gguf_filename(name) + and not is_mmproj_filename(name) + and not is_mtp_drafter_path(name) + and not is_big_endian_gguf_path(name, extract_quant_label(name)) ] if not gguf_files: return None @@ -372,6 +402,8 @@ def list_gguf_variants( has_vision = True continue quant = extract_quant_label(filename) + if is_big_endian_gguf_path(filename, quant): + continue quant_totals[quant] = quant_totals.get(quant, 0) + int(getattr(sibling, "size", 0) or 0) quant_first_file.setdefault(quant, filename) @@ -424,6 +456,8 @@ def list_local_gguf_variants(directory: str) -> tuple[list[GgufVariantInfo], boo if is_mtp_drafter_path(rel): continue quant = extract_quant_label(rel) + if is_big_endian_gguf_path(rel, quant): + continue quant_totals[quant] = quant_totals.get(quant, 0) + size quant_first_file.setdefault(quant, rel) diff --git a/studio/backend/hub/utils/gguf_plan.py b/studio/backend/hub/utils/gguf_plan.py index 60782fa109..40d5a32549 100644 --- a/studio/backend/hub/utils/gguf_plan.py +++ b/studio/backend/hub/utils/gguf_plan.py @@ -9,6 +9,7 @@ from typing import Optional, Sequence from hub.utils.download_manifest import ExpectedFile from hub.utils.gguf import ( extract_quant_label, + is_big_endian_gguf_path, is_gguf_filename, is_mmproj_filename, is_mtp_drafter_path, @@ -68,6 +69,7 @@ def is_main_gguf_variant_path(path: str, variant: str) -> bool: is_gguf_filename(path) and not is_mmproj_filename(path) and not is_mtp_drafter_path(path) + and not is_big_endian_gguf_path(path, variant) and extract_quant_label(path).lower() == variant.lower() ) @@ -140,6 +142,8 @@ def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]: if is_mmproj_filename(name) or is_mtp_drafter_path(name): continue quant = extract_quant_label(name).lower() + if is_big_endian_gguf_path(name, quant): + continue main.setdefault(quant, []).append(sibling) plans: dict[str, GgufVariantPlan] = {} diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index a2f2eca81b..6835126e62 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -81,6 +81,7 @@ try: from utils.models.model_config import ( _pick_best_gguf, _extract_quant_label, + _is_big_endian_gguf_path, is_audio_input_type, ) from core.inference import get_inference_backend @@ -112,6 +113,7 @@ except ImportError: from utils.models.model_config import ( _pick_best_gguf, _extract_quant_label, + _is_big_endian_gguf_path, is_audio_input_type, ) from core.inference import get_inference_backend @@ -2106,7 +2108,11 @@ async def get_gguf_variants( size = f.stat().st_size except OSError: continue # broken symlink / unreadable: skip - q = _extract_quant_label(f.name).lower() + rel = f.relative_to(snap).as_posix() + q = _extract_quant_label(rel) + if _is_big_endian_gguf_path(rel, q): + continue + q = q.lower() by_quant[q] = by_quant.get(q, 0) + size if by_quant: cached_bytes_by_quant_per_snapshot.append(by_quant) @@ -2182,8 +2188,12 @@ async def get_gguf_download_progress( for f in _iter_gguf_paths(entry): if _is_mmproj_filename(f.name): continue - fname = f.name.lower().replace("-", "").replace("_", "") - if not variant_lower or variant_lower in fname: + rel = f.relative_to(entry).as_posix() + quant = _extract_quant_label(rel) + if _is_big_endian_gguf_path(rel, quant): + continue + rel_key = rel.lower().replace("-", "").replace("_", "") + if not variant_lower or variant_lower in rel_key: try: downloaded_bytes += f.stat().st_size except OSError: diff --git a/studio/backend/tests/test_cached_gguf_routes.py b/studio/backend/tests/test_cached_gguf_routes.py index 5a4ca68ab4..6409aef11e 100644 --- a/studio/backend/tests/test_cached_gguf_routes.py +++ b/studio/backend/tests/test_cached_gguf_routes.py @@ -503,6 +503,58 @@ def test_gguf_variants_mmproj_does_not_mark_quant_downloaded(monkeypatch, tmp_pa assert flags["F16"] is False +def test_gguf_variants_ignore_big_endian_siblings(monkeypatch, tmp_path): + import huggingface_hub.constants as hf_constants + + siblings = [ + SimpleNamespace(rfilename = "model-Q4_K_M-be.gguf", size = 100), + SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10), + ] + monkeypatch.setattr( + "huggingface_hub.model_info", + lambda *_args, **_kwargs: SimpleNamespace(siblings = siblings), + ) + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + + snap = tmp_path / "models--org--repo" / "snapshots" / "rev" + snap.mkdir(parents = True) + (snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 10) + + result = asyncio.run( + models_route.get_gguf_variants( + repo_id = "org/repo", hf_token = None, current_subject = "test-user" + ) + ) + + assert [(v.quant, v.filename, v.size_bytes, v.downloaded) for v in result.variants] == [ + ("Q4_K_M", "model-Q4_K_M.gguf", 10, True) + ] + + +def test_gguf_variants_cached_big_endian_does_not_satisfy_variant(monkeypatch, tmp_path): + import huggingface_hub.constants as hf_constants + + variants = [ + SimpleNamespace(filename = "model-Q4_K_M.gguf", quant = "Q4_K_M", size_bytes = 10), + ] + monkeypatch.setattr( + models_route, "list_gguf_variants", lambda repo_id, hf_token = None: (variants, False) + ) + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + + snap = tmp_path / "models--org--repo" / "snapshots" / "rev" + snap.mkdir(parents = True) + (snap / "model-Q4_K_M-be.gguf").write_bytes(b"x" * 10) + + result = asyncio.run( + models_route.get_gguf_variants( + repo_id = "org/repo", hf_token = None, current_subject = "test-user" + ) + ) + + assert result.variants[0].downloaded is False + + def test_gguf_download_progress_excludes_mmproj(monkeypatch, tmp_path): """A cached mmproj adapter must not count toward a same-label main variant's download progress (mmproj-F16 vs an F16 weight).""" @@ -524,3 +576,45 @@ def test_gguf_download_progress_excludes_mmproj(monkeypatch, tmp_path): assert result["downloaded_bytes"] == 0 assert result["progress"] == 0 + + +def test_gguf_download_progress_excludes_big_endian_sibling(monkeypatch, tmp_path): + import huggingface_hub.constants as hf_constants + + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + snap = tmp_path / "models--org--repo" / "snapshots" / "rev" + snap.mkdir(parents = True) + (snap / "model-Q4_K_M-be.gguf").write_bytes(b"y" * 20_000) + + result = asyncio.run( + models_route.get_gguf_download_progress( + repo_id = "org/repo", + variant = "Q4_K_M", + expected_bytes = 20_000, + current_subject = "test-user", + ) + ) + + assert result["downloaded_bytes"] == 0 + assert result["progress"] == 0 + + +def test_gguf_download_progress_counts_quant_subdir(monkeypatch, tmp_path): + import huggingface_hub.constants as hf_constants + + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + snap = tmp_path / "models--org--repo" / "snapshots" / "rev" / "Q4_K_M" + snap.mkdir(parents = True) + (snap / "foo.gguf").write_bytes(b"x" * 20_000) + + result = asyncio.run( + models_route.get_gguf_download_progress( + repo_id = "org/repo", + variant = "Q4_K_M", + expected_bytes = 20_000, + current_subject = "test-user", + ) + ) + + assert result["downloaded_bytes"] == 20_000 + assert result["progress"] == 1.0 diff --git a/studio/backend/tests/test_export_absolute_paths.py b/studio/backend/tests/test_export_absolute_paths.py index f333a590c2..d62c2a5e61 100644 --- a/studio/backend/tests/test_export_absolute_paths.py +++ b/studio/backend/tests/test_export_absolute_paths.py @@ -156,6 +156,7 @@ def _install_lightweight_backend_stubs(monkeypatch): utils_model_config = types.ModuleType("utils.models.model_config") utils_model_config._pick_best_gguf = lambda variants: variants[0] if variants else None utils_model_config._extract_quant_label = lambda value: value + utils_model_config._is_big_endian_gguf_path = lambda *args, **kwargs: False utils_model_config.is_audio_input_type = lambda *args, **kwargs: None monkeypatch.setitem( sys.modules, diff --git a/studio/backend/tests/test_gguf_routing.py b/studio/backend/tests/test_gguf_routing.py index 8a8bff69aa..9fd83da209 100644 --- a/studio/backend/tests/test_gguf_routing.py +++ b/studio/backend/tests/test_gguf_routing.py @@ -83,6 +83,23 @@ def test_detects_gguf_in_directory(tmp_path): assert result.endswith("model-Q4_K_M.gguf") +def test_directory_auto_detect_ignores_big_endian_sibling(tmp_path): + be = tmp_path / "model-Q4_K_M-be.gguf" + be.write_bytes(b"x" * 100) + target = tmp_path / "model-Q4_K_M.gguf" + target.write_bytes(b"y" * 10) + + result = detect_gguf_model(str(tmp_path)) + assert result == str(target.resolve()) + + +def test_direct_big_endian_file_is_not_detected(tmp_path): + gguf = tmp_path / "model-Q4_K_M-be.gguf" + gguf.write_bytes(b"") + + assert detect_gguf_model(str(gguf)) is None + + def test_directory_named_like_gguf_scans_inside(tmp_path): """A directory named *.gguf resolves the real .gguf inside, not itself.""" gguf_dir = tmp_path / "mymodel.gguf" diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index aab58adfff..a9e0b98247 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -79,6 +79,7 @@ from huggingface_hub import constants as hf_constants from core.inference.llama_cpp import ( LlamaCppBackend, + _gguf_files_for_variant, _hf_offline_if_dns_dead, _probe_dns_dead, ) @@ -130,6 +131,155 @@ def clean_offline_env(monkeypatch): monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) +class TestGgufVariantFileResolution: + def test_prefers_exact_unknown_variant_over_big_endian_sibling(self): + files = [ + "tinyllamas/stories260K-be.gguf", + "tinyllamas/stories260K-infill.gguf", + "tinyllamas/stories260K.gguf", + ] + + assert _gguf_files_for_variant(files, "stories260K") == ["tinyllamas/stories260K.gguf"] + + @pytest.mark.parametrize( + "big_endian_path", + [ + "model-Q4_K_M-be.gguf", + "model-Q4_K_M_be.gguf", + "model-Q4_K_M_be_infill.gguf", + r"nested\model-Q4_K_M_be.gguf", + ], + ) + def test_filters_big_endian_known_quant_before_exact_match(self, big_endian_path): + files = [ + big_endian_path, + "model-Q4_K_M.gguf", + ] + + assert _gguf_files_for_variant(files, "Q4_K_M") == ["model-Q4_K_M.gguf"] + + def test_keeps_model_name_be_token_before_quant(self): + files = [ + "foo-be-Q4_K_M.gguf", + ] + + assert _gguf_files_for_variant(files, "Q4_K_M") == ["foo-be-Q4_K_M.gguf"] + + def test_keeps_model_name_be_token_with_quant_subdir(self): + files = [ + "Q4_K_M/foo-be.gguf", + ] + + assert _gguf_files_for_variant(files, "Q4_K_M") == ["Q4_K_M/foo-be.gguf"] + + def test_empty_variant_filters_big_endian_files(self): + files = [ + "model-Q4_K_M-be.gguf", + "model-Q4_K_M.gguf", + ] + + assert _gguf_files_for_variant(files, "") == ["model-Q4_K_M.gguf"] + + def test_remote_listing_skips_big_endian_quant_sibling(self, monkeypatch, clean_offline_env): + siblings = [ + _types.SimpleNamespace(rfilename = "model-Q4_K_M-be.gguf", size = 100), + _types.SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10), + ] + monkeypatch.setattr( + "huggingface_hub.model_info", + lambda *_args, **_kwargs: _types.SimpleNamespace(siblings = siblings), + ) + + variants, has_vision = list_gguf_variants("org/repo") + + assert has_vision is False + assert [(v.quant, v.filename, v.size_bytes) for v in variants] == [ + ("Q4_K_M", "model-Q4_K_M.gguf", 10) + ] + + def test_download_uses_exact_variant_label(self, monkeypatch, tmp_path): + backend = LlamaCppBackend() + downloaded: list[str] = [] + + def fake_get_paths_info( + _repo_id, + paths, + token = None, + ): + return [_types.SimpleNamespace(path = path, size = 1) for path in paths if path is not None] + + def fake_download( + *, + repo_id, + filename, + token = None, + ): + downloaded.append(filename) + return f"/fake/{repo_id}/{filename}" + + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) + with ( + patch( + "huggingface_hub.list_repo_files", + lambda *_a, **_k: [ + "tinyllamas/stories260K-be.gguf", + "tinyllamas/stories260K-infill.gguf", + "tinyllamas/stories260K.gguf", + ], + ), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("huggingface_hub.hf_hub_download", fake_download), + ): + out = backend._download_gguf( + hf_repo = "ggml-org/models", + hf_variant = "stories260K", + ) + + assert downloaded == ["tinyllamas/stories260K.gguf"] + assert out == "/fake/ggml-org/models/tinyllamas/stories260K.gguf" + + def test_download_includes_uppercase_split_gguf_shards(self, monkeypatch, tmp_path): + backend = LlamaCppBackend() + downloaded: list[str] = [] + + files = [ + "model-Q4_K_M-00001-of-00002.GGUF", + "model-Q4_K_M-00002-of-00002.GGUF", + ] + + def fake_get_paths_info( + _repo_id, + paths, + token = None, + ): + return [_types.SimpleNamespace(path = path, size = 1) for path in paths if path is not None] + + def fake_download( + *, + repo_id, + filename, + token = None, + ): + downloaded.append(filename) + return f"/fake/{repo_id}/{filename}" + + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: files), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("huggingface_hub.hf_hub_download", fake_download), + ): + out = backend._download_gguf( + hf_repo = "org/repo", + hf_variant = "Q4_K_M", + ) + + assert downloaded == files + assert out == "/fake/org/repo/model-Q4_K_M-00001-of-00002.GGUF" + + def _siblings(items: dict[str, int]): """Mock ``hf_model_info(...).siblings`` payload.""" return _types.SimpleNamespace( @@ -268,6 +418,22 @@ class TestDetectGgufFromCache: out == "BF16/foo.gguf" ), f"subdir-only layout must resolve to relative path, got {out}" + def test_subdir_quant_keeps_be_model_name_token(self, hf_cache): + _build_cache( + hf_cache, + "unsloth/a", + {"Q4_K_M/foo-be.gguf": 1}, + ) + assert _detect_gguf_from_hf_cache("unsloth/a") == "Q4_K_M/foo-be.gguf" + + def test_big_endian_only_cache_is_not_detected(self, hf_cache): + _build_cache( + hf_cache, + "unsloth/a", + {"model-Q4_K_M-be.gguf": 1}, + ) + assert _detect_gguf_from_hf_cache("unsloth/a") is None + def test_returns_none_when_no_gguf(self, hf_cache): _build_cache(hf_cache, "unsloth/a", {"README.md": 10}) assert _detect_gguf_from_hf_cache("unsloth/a") is None @@ -298,6 +464,17 @@ class TestDetectGgufModelRemoteOffline: out = detect_gguf_model_remote("unsloth/a") assert out == "a-Q4_K_M.gguf" + def test_remote_big_endian_only_repo_is_not_detected(self, clean_offline_env, monkeypatch): + siblings = [ + _types.SimpleNamespace(rfilename = "model-Q4_K_M-be.gguf"), + ] + monkeypatch.setattr( + "huggingface_hub.model_info", + lambda *_args, **_kwargs: _types.SimpleNamespace(siblings = siblings), + ) + + assert detect_gguf_model_remote("unsloth/a") is None + def test_repository_not_found_does_not_consult_cache(self, hf_cache, clean_offline_env): # Cache has a file but the API says the repo is gone. _build_cache(hf_cache, "unsloth/a", {"a-Q4_K_M.gguf": 1}) @@ -566,6 +743,49 @@ class TestListLocalGgufVariantsSubdir: assert out is not None assert Path(out).name == "foo.gguf" + def test_find_local_gguf_by_variant_ignores_big_endian_sibling(self, tmp_path): + from utils.models.model_config import _find_local_gguf_by_variant + + (tmp_path / "config.json").write_text("{}") + (tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"\0" * 10) + target = tmp_path / "model-Q4_K_M.gguf" + target.write_bytes(b"\0" * 20) + + out = _find_local_gguf_by_variant(str(tmp_path), "Q4_K_M") + assert out == str(target.resolve()) + + def test_find_local_gguf_by_variant_skips_big_endian_only_match(self, tmp_path): + from utils.models.model_config import _find_local_gguf_by_variant + + (tmp_path / "config.json").write_text("{}") + (tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"\0" * 10) + + assert _find_local_gguf_by_variant(str(tmp_path), "Q4_K_M") is None + + def test_model_config_variant_ignores_big_endian_sibling(self, tmp_path): + from utils.models.model_config import ModelConfig + + (tmp_path / "config.json").write_text("{}") + (tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"\0" * 10) + target = tmp_path / "model-Q4_K_M.gguf" + target.write_bytes(b"\0" * 20) + + config = ModelConfig.from_identifier(str(tmp_path), gguf_variant = "Q4_K_M") + assert config is not None + assert config.gguf_file == str(target.resolve()) + + def test_local_variant_listing_keeps_subdir_be_model_name_token(self, tmp_path): + from utils.models.model_config import list_local_gguf_variants + + (tmp_path / "config.json").write_text("{}") + (tmp_path / "Q4_K_M").mkdir() + (tmp_path / "Q4_K_M" / "foo-be.gguf").write_bytes(b"\0" * 10) + + variants, _ = list_local_gguf_variants(str(tmp_path)) + assert [(v.quant, v.filename, v.size_bytes) for v in variants] == [ + ("Q4_K_M", "Q4_K_M/foo-be.gguf", 10) + ] + class TestListGgufVariantsPermanentErrors: """Permanent HF errors must surface; cache fallback only on transient.""" diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 7245343eeb..83cd281c2e 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -25,6 +25,7 @@ from utils.models.gguf_metadata import ( import structlog from loggers import get_logger import os +import re import subprocess import sys from pathlib import Path @@ -1235,7 +1236,9 @@ def detect_gguf_model(path: str) -> Optional[str]: # (-m drafter --model-draft drafter). Include the immediate parent # dir so the MTP/ subdir copies are caught -- the basename alone # (...-MTP.gguf) doesn't match the predicate's mtp- prefix. - if _is_mmproj(p.name) or _is_mtp_drafter(f"{p.parent.name}/{p.name}"): + rel = f"{p.parent.name}/{p.name}" + quant = _extract_quant_label(rel) + if _is_mmproj(p.name) or _is_mtp_drafter(rel) or _is_big_endian_gguf_path(rel, quant): return None # Extension is authoritative: don't gate on is_file()/exists(), which # can fail in the Windows lock window after llama-server is killed. @@ -1249,15 +1252,18 @@ def detect_gguf_model(path: str) -> Optional[str]: # Case 2: directory containing .gguf files (skip mmproj / MTP drafter) if p.is_dir(): - gguf_files = sorted( - ( - f - for f in _iter_gguf_files(p) - if not _is_mmproj(f.name) and not _is_mtp_drafter(f"{f.parent.name}/{f.name}") - ), - key = lambda f: f.stat().st_size, - reverse = True, - ) + gguf_files = [] + for f in _iter_gguf_files(p): + context_rel = f"{f.parent.name}/{f.name}" + quant = _extract_quant_label(context_rel) + if ( + _is_mmproj(f.name) + or _is_mtp_drafter(context_rel) + or _is_big_endian_gguf_path(context_rel, quant) + ): + continue + gguf_files.append(f) + gguf_files.sort(key = lambda f: f.stat().st_size, reverse = True) if gguf_files: return str(gguf_files[0].resolve()) @@ -1383,6 +1389,44 @@ def _extract_quant_label(filename: str) -> str: return stem.split("-")[-1] +_BIG_ENDIAN_GGUF_FILENAME_RE = re.compile(r"(^|[-_])be(?:[._-]|$)", re.IGNORECASE) +_GGUF_KNOWN_QUANT_RE = re.compile( + r"(UD-)?" + r"(MXFP[0-9]+(?:_[A-Z0-9]+)*" + r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?" + r"|TQ[0-9]+_[0-9]+" + r"|Q[0-9]+_K_[A-Z]+" + r"|Q[0-9]+_[0-9]+" + r"|Q[0-9]+_K" + r"|BF16|F16|F32)", + re.IGNORECASE, +) + + +def _is_big_endian_gguf_path(path: str, quant: str = "") -> bool: + normalized = path.replace("\\", "/") + name = normalized.rsplit("/", 1)[-1] + stem = name.rsplit(".", 1)[0].lower() + quant_key = quant.strip().lower() + quant_index = stem.find(quant_key) if quant_key else -1 + parent = normalized.rsplit("/", 1)[0].lower() if "/" in normalized else "" + quant_in_parent_only = ( + bool(parent) + and quant_index < 0 + and ( + (quant_key and quant_key in parent) + or (not quant_key and _GGUF_KNOWN_QUANT_RE.search(parent) is not None) + ) + ) + for match in _BIG_ENDIAN_GGUF_FILENAME_RE.finditer(stem): + if quant_index >= 0 and quant_index < match.start(): + return True + tail = stem[match.end() :].lstrip("._-") + if not tail or _GGUF_KNOWN_QUANT_RE.search(tail) is None: + return not quant_in_parent_only + return False + + def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str: """Directory to scan upward from for local GGUF companion files.""" import re @@ -1522,6 +1566,8 @@ def list_gguf_variants( continue quant = _extract_quant_label(fname) + if _is_big_endian_gguf_path(fname, quant): + continue quant_totals[quant] = quant_totals.get(quant, 0) + size if quant not in quant_first_file: quant_first_file[quant] = fname @@ -1598,6 +1644,8 @@ def list_local_gguf_variants(directory: str) -> tuple[list[GgufVariantInfo], boo if _is_mtp_drafter(rel): continue quant = _extract_quant_label(rel) + if _is_big_endian_gguf_path(rel, quant): + continue quant_totals[quant] = quant_totals.get(quant, 0) + size if quant not in quant_first_file: quant_first_file[quant] = rel @@ -1630,13 +1678,16 @@ def _find_local_gguf_by_variant(directory: str, variant: str) -> Optional[str]: # ``BF16/foo-BF16-00001-of-00002.gguf``) are found. Match the relative # path so the quant label can come from the dir name when the basename # omits it. - matches = sorted( - f - for f in _iter_gguf_files(p, recursive = True) - if not _is_mmproj(f.name) - and not _is_mtp_drafter(f.relative_to(p).as_posix()) - and _extract_quant_label(f.relative_to(p).as_posix()) == variant - ) + matches = [] + for f in _iter_gguf_files(p, recursive = True): + rel = f.relative_to(p).as_posix() + if _is_mmproj(f.name) or _is_mtp_drafter(rel): + continue + quant = _extract_quant_label(rel) + if quant != variant or _is_big_endian_gguf_path(rel, quant): + continue + matches.append(f) + matches.sort() if matches: return str(matches[0].resolve()) return None @@ -1649,11 +1700,13 @@ def _detect_gguf_from_hf_cache(repo_id: str) -> Optional[str]: the projector cannot route it as the main model. """ for snap in _iter_hf_cache_snapshots(repo_id): - rel_files = [ - rel - for f in _iter_gguf_files(snap, recursive = True) - if not _is_mtp_drafter(rel := f.relative_to(snap).as_posix()) and not _is_mmproj(f.name) - ] + rel_files = [] + for f in _iter_gguf_files(snap, recursive = True): + rel = f.relative_to(snap).as_posix() + quant = _extract_quant_label(rel) + if _is_mmproj(f.name) or _is_mtp_drafter(rel) or _is_big_endian_gguf_path(rel, quant): + continue + rel_files.append(rel) if rel_files: return _pick_best_gguf(rel_files) return None @@ -1678,7 +1731,19 @@ def detect_gguf_model_remote(repo_id: str, hf_token: Optional[str] = None) -> Op for attempt in range(3): try: info = hf_model_info(repo_id, token = hf_token) - repo_files = [s.rfilename for s in info.siblings] + repo_files = [] + for sibling in info.siblings: + fname = sibling.rfilename + if not fname.lower().endswith(".gguf"): + continue + quant = _extract_quant_label(fname) + if ( + _is_mmproj(fname) + or _is_mtp_drafter(fname) + or _is_big_endian_gguf_path(fname, quant) + ): + continue + repo_files.append(fname) return _pick_best_gguf(repo_files) except Exception as e: last_err = e diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index ae4d31b27c..96cb1473bd 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -46,6 +46,7 @@ import { import { useExternalProvidersStore } from "../stores/external-providers-store"; import { isMultimodalResponse } from "../types/api"; import type { + GgufVariantDetail, OpenAIChatCompletionsRequest, OpenAIChatMessage, OpenAIMessageContent, @@ -1141,6 +1142,45 @@ function waitForModelReady(abortSignal?: AbortSignal): Promise { */ // Cap cascade so broken cached repos can't spam /api/inference/load. const MAX_AUTO_LOAD_ATTEMPTS = 3; +const BIG_ENDIAN_GGUF_FILENAME_RE = /(^|[-_])be(?:[._-]|$)/gi; +const GGUF_KNOWN_QUANT_RE = + /(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|Q[0-9]+_[0-9]+|Q[0-9]+_K|BF16|F16|F32)/i; + +function hasBigEndianGgufMarker(filename: string, quant?: string | null): boolean { + const normalized = filename.replace(/\\/g, "/").toLowerCase(); + const separatorIndex = normalized.lastIndexOf("/"); + const basename = separatorIndex >= 0 ? normalized.slice(separatorIndex + 1) : normalized; + const parent = separatorIndex >= 0 ? normalized.slice(0, separatorIndex) : ""; + const stem = basename.replace(/\.[^.]*$/, ""); + const quantKey = quant?.trim().toLowerCase() || ""; + const quantIndex = quantKey ? stem.indexOf(quantKey) : -1; + const quantInParentOnly = + !!parent && + quantIndex < 0 && + ((!!quantKey && parent.includes(quantKey)) || + (!quantKey && GGUF_KNOWN_QUANT_RE.test(parent))); + for (const match of stem.matchAll(BIG_ENDIAN_GGUF_FILENAME_RE)) { + if (quantIndex >= 0 && quantIndex < (match.index ?? 0)) { + return true; + } + const tail = stem.slice((match.index ?? 0) + match[0].length).replace(/^[._-]+/, ""); + if (!tail || !GGUF_KNOWN_QUANT_RE.test(tail)) { + return !quantInParentOnly; + } + } + return false; +} + +function isAutoLoadableGgufVariant(variant: GgufVariantDetail | null): boolean { + if (!variant?.filename) { + return false; + } + const filename = variant.filename.trim().toLowerCase(); + if (!filename) { + return false; + } + return !hasBigEndianGgufMarker(filename, variant.quant); +} async function autoLoadSmallestModel(): Promise<{ loaded: boolean; @@ -1195,7 +1235,7 @@ async function autoLoadSmallestModel(): Promise<{ try { const variants = await listGgufVariants(repo.repo_id); const downloaded = variants.variants - .filter((v) => v.downloaded) + .filter((v) => v.downloaded && isAutoLoadableGgufVariant(v)) .sort((a, b) => a.size_bytes - b.size_bytes); if (downloaded.length > 0) { const variant = downloaded[0];