diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index f30a8acf89..19d02d36fe 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2996,6 +2996,16 @@ class LlamaCppBackend: return str(mmproj) + def _mmproj_vram_bytes(self, launch_mmproj_path: Optional[str]) -> int: + """Return resolved mmproj VRAM bytes, or 0 when absent/unreadable.""" + if not launch_mmproj_path: + return 0 + try: + return self._get_gguf_size_bytes(launch_mmproj_path) + except OSError as e: + logger.debug(f"Could not size mmproj {launch_mmproj_path}: {e}") + return 0 + def _resolve_launch_mtp_path(self, *, mtp_draft_path: Optional[str]) -> Optional[str]: """Return mtp_draft_path iff it exists on disk, else None. @@ -3561,8 +3571,29 @@ class LlamaCppBackend: effective_ctx = requested_ctx if requested_ctx > 0 else (self._context_length or 0) max_available_ctx = self._context_length or effective_ctx gpus: list[tuple[int, int]] = [] + # Keep fit-budget and launch-flag mmproj resolution in sync. + launch_mmproj_path = None + if not extra_args_disable_mmproj(extra_args): + launch_mmproj_path = self._resolve_launch_mmproj_path( + model_path = model_path, + mmproj_path = mmproj_path, + ) + # Need both a resolved mmproj AND the config vision flag; a stray + # mmproj passing the family-name heuristic must not flip a non-VLM + # GGUF into vision mode. + effective_is_vision = bool(launch_mmproj_path) and bool(is_vision) + if is_vision and not effective_is_vision: + logger.warning( + "Vision-capable GGUF loaded without a usable mmproj; " + "image input will be disabled for this session" + ) try: - model_size = self._get_gguf_size_bytes(model_path) + gguf_size = self._get_gguf_size_bytes(model_path) + # Include GPU-loaded mmproj in the fit budget (#5825). + mmproj_size = ( + self._mmproj_vram_bytes(launch_mmproj_path) if effective_is_vision else 0 + ) + model_size = gguf_size + mmproj_size gpus = self._get_gpu_free_memory() # Resolve effective context: 0 means let llama-server use @@ -3802,8 +3833,12 @@ class LlamaCppBackend: kv_cache_bytes = self._estimate_kv_cache_bytes( effective_ctx, cache_type_kv, n_parallel = n_parallel ) + mmproj_note = ( + f"mmproj: {mmproj_size / (1024**3):.1f} GB, " if mmproj_size else "" + ) logger.info( - f"GGUF size: {model_size / (1024**3):.1f} GB, " + f"GGUF size: {gguf_size / (1024**3):.1f} GB, " + f"{mmproj_note}" f"est. KV cache: {kv_cache_bytes / (1024**3):.1f} GB, " f"context: {effective_ctx}, " f"GPUs free: {gpus}, selected: {gpu_indices}, fit: {use_fit}" @@ -3814,22 +3849,6 @@ class LlamaCppBackend: tp_tensor_split = None effective_ctx = requested_ctx # fall back to original - launch_mmproj_path = None - if not extra_args_disable_mmproj(extra_args): - launch_mmproj_path = self._resolve_launch_mmproj_path( - model_path = model_path, - mmproj_path = mmproj_path, - ) - # Need both a resolved mmproj AND the config vision flag; a stray - # mmproj passing the family-name heuristic must not flip a non-VLM - # GGUF into vision mode. - effective_is_vision = bool(launch_mmproj_path) and bool(is_vision) - if is_vision and not effective_is_vision: - logger.warning( - "Vision-capable GGUF loaded without a usable mmproj; " - "image input will be disabled for this session" - ) - # Audio input straight from the mmproj (clip.has_audio_encoder), # independent of token names. self._mmproj_has_audio = False diff --git a/studio/backend/tests/test_mmproj_vram_accounting.py b/studio/backend/tests/test_mmproj_vram_accounting.py new file mode 100644 index 0000000000..bee289f183 --- /dev/null +++ b/studio/backend/tests/test_mmproj_vram_accounting.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for mmproj VRAM accounting in GGUF fit budgeting (#5825).""" + +from __future__ import annotations + +from pathlib import Path + +from core.inference.llama_cpp import LlamaCppBackend + + +def _write(path: Path, n_bytes: int) -> Path: + path.parent.mkdir(parents = True, exist_ok = True) + path.write_bytes(b"\x00" * n_bytes) + return path + + +def _backend() -> LlamaCppBackend: + return LlamaCppBackend.__new__(LlamaCppBackend) + + +def test_counts_resolved_projector_size(tmp_path: Path): + mmproj = _write(tmp_path / "Qwen3.5-9B-BF16-mmproj.gguf", 1024) + + got = _backend()._mmproj_vram_bytes(str(mmproj)) + + assert got == 1024 + + +def test_zero_when_no_projector_resolved(tmp_path: Path): + assert _backend()._mmproj_vram_bytes(None) == 0 + + +def test_zero_when_projector_missing_on_disk(tmp_path: Path): + missing = tmp_path / "Qwen3.5-9B-BF16-mmproj.gguf" # never created + + got = _backend()._mmproj_vram_bytes(str(missing)) + + assert got == 0