ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: support mmproj for local vision GGUF models + fix Windows pipe deadlock

Browse source
This commit is contained in:
Roland Tannous 2026-03-01 12:55:53 +00:00
commit ff93c97024
3 changed files with 115 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

48
studio/backend/core/inference/llama_cpp.py
View file

@ -41,6 +41,8 @@ class LlamaCppBackend:
self._is_vision: bool = False
self._healthy = False
self._lock = threading.Lock()
self._stdout_lines: list[str] = []
self._stdout_thread: Optional[threading.Thread] = None
atexit.register(self._cleanup)
@ -115,6 +117,26 @@ class LlamaCppBackend:
s.bind(("", 0))
return s.getsockname()[1]
# ── Stdout drain (prevents pipe deadlock on Windows) ─────────
def _drain_stdout(self):
"""
Read lines from the subprocess stdout in a background thread.
This prevents a pipe-buffer deadlock on Windows where the default
pipe buffer is only ~4 KB. Without draining, llama-server blocks
on writes and never becomes healthy.
"""
try:
for line in self._process.stdout:
line = line.rstrip()
if line:
self._stdout_lines.append(line)
logger.info(f"[llama-server] {line}")
except (ValueError, OSError):
# Pipe closed — process is terminating
pass
# ── Lifecycle ─────────────────────────────────────────────────
def load_model(
@ -122,6 +144,8 @@ class LlamaCppBackend:
*,
# Local mode: pass a path to a .gguf file
gguf_path: Optional[str] = None,
# Vision projection (mmproj) for local vision models
mmproj_path: Optional[str] = None,
# HF mode: let llama-server download via -hf "repo:quant"
hf_repo: Optional[str] = None,
hf_variant: Optional[str] = None,
@ -186,6 +210,14 @@ class LlamaCppBackend:
if n_threads is not None:
cmd.extend(["--threads", str(n_threads)])
# Append mmproj for local vision models
if mmproj_path:
if not Path(mmproj_path).is_file():
logger.warning(f"mmproj file not found: {mmproj_path}")
else:
cmd.extend(["--mmproj", mmproj_path])
logger.info(f"Using mmproj for vision: {mmproj_path}")
logger.info(f"Starting llama-server: {' '.join(cmd)}")
# Set LD_LIBRARY_PATH so llama-server can find its shared libs
@ -196,6 +228,7 @@ class LlamaCppBackend:
existing_ld = env.get("LD_LIBRARY_PATH", "")
env["LD_LIBRARY_PATH"] = f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir
self._stdout_lines = []
self._process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
@ -204,6 +237,12 @@ class LlamaCppBackend:
env=env,
)
# Start background thread to drain stdout and prevent pipe deadlock
self._stdout_thread = threading.Thread(
target=self._drain_stdout, daemon=True, name="llama-stdout"
)
self._stdout_thread.start()
self._gguf_path = gguf_path
self._hf_repo = hf_repo
self._hf_variant = hf_variant
@ -256,6 +295,9 @@ class LlamaCppBackend:
logger.warning(f"Error killing llama-server process: {e}")
finally:
self._process = None
if self._stdout_thread is not None:
self._stdout_thread.join(timeout=2)
self._stdout_thread = None
def _cleanup(self):
"""atexit handler to ensure llama-server is terminated."""
@ -273,8 +315,10 @@ class LlamaCppBackend:
while time.monotonic() < deadline:
# Check if process crashed
if self._process.poll() is not None:
# Read remaining output for error info
output = self._process.stdout.read() if self._process.stdout else ""
# Give the drain thread a moment to collect final output
if self._stdout_thread is not None:
self._stdout_thread.join(timeout=2)
output = "\n".join(self._stdout_lines[-50:])
logger.error(
f"llama-server exited with code {self._process.returncode}. "
f"Output: {output[:2000]}"

1
studio/backend/routes/inference.py
View file

@ -125,6 +125,7 @@ async def load_model(
# Local mode: llama-server loads via -m <path>
success = llama_backend.load_model(
gguf_path=config.gguf_file,
mmproj_path=config.gguf_mmproj_file,
model_identifier=config.identifier,
is_vision=config.is_vision,
n_ctx=request.max_seq_length,

72
studio/backend/utils/models/model_config.py
View file

@ -422,6 +422,32 @@ def is_vision_model(model_name: str, hf_token: Optional[str] = None) -> bool:
pass
def _is_mmproj(filename: str) -> bool:
"""Check if a GGUF filename is a vision projection (mmproj) file."""
return "mmproj" in filename.lower()
def detect_mmproj_file(path: str) -> Optional[str]:
"""
Find the mmproj (vision projection) GGUF file in a directory.
Args:
path: Directory to search — or a .gguf file (uses its parent dir).
Returns:
Full path to the mmproj .gguf file, or None if not found.
"""
p = Path(path)
search_dir = p.parent if p.is_file() else p
if not search_dir.is_dir():
return None
for f in search_dir.glob("*.gguf"):
if _is_mmproj(f.name):
return str(f.resolve())
return None
def detect_gguf_model(path: str) -> Optional[str]:
"""
Check if the given local path is or contains a GGUF model file.
@ -430,6 +456,9 @@ def detect_gguf_model(path: str) -> Optional[str]:
1. path is a direct .gguf file path
2. path is a directory containing .gguf files
Skips mmproj (vision projection) files — those must be passed via
``--mmproj``, not ``-m``. Use :func:`detect_mmproj_file` instead.
Returns the full path to the .gguf file if found, None otherwise.
For HuggingFace repo detection, use detect_gguf_model_remote() instead.
"""
@ -437,11 +466,16 @@ def detect_gguf_model(path: str) -> Optional[str]:
# Case 1: direct .gguf file
if p.suffix == ".gguf" and p.is_file():
if _is_mmproj(p.name):
return None
return str(p.resolve())
# Case 2: directory containing .gguf files
# Case 2: directory containing .gguf files (skip mmproj)
if p.is_dir():
gguf_files = sorted(p.glob("*.gguf"), key=lambda f: f.stat().st_size, reverse=True)
gguf_files = sorted(
(f for f in p.glob("*.gguf") if not _is_mmproj(f.name)),
key=lambda f: f.stat().st_size, reverse=True,
)
if gguf_files:
return str(gguf_files[0].resolve())
@ -677,7 +711,8 @@ def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str,
continue
# Check for flat GGUF export (e.g. exports/gemma-3-4b-it-finetune-gguf/)
gguf_files = list(run_dir.glob("*.gguf"))
# Filter out mmproj (vision projection) files — they aren't loadable as main models
gguf_files = [f for f in run_dir.glob("*.gguf") if not _is_mmproj(f.name)]
if gguf_files:
base_model = None
export_meta = run_dir / "export_metadata.json"
@ -907,6 +942,7 @@ class ModelConfig:
is_lora: bool # Is this a lora adapter?
is_gguf: bool = False # Is this a GGUF model?
gguf_file: Optional[str] = None # Full path to the .gguf file (local mode)
gguf_mmproj_file: Optional[str] = None # Full path to the mmproj .gguf file (vision projection)
gguf_hf_repo: Optional[str] = None # HF repo ID for -hf mode (e.g. "unsloth/gemma-3-4b-it-GGUF")
gguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M")
base_model: Optional[str] = None # Base model (for LoRAs)
@ -1005,16 +1041,44 @@ class ModelConfig:
if gguf_file:
display_name = Path(gguf_file).stem
logger.info(f"Detected local GGUF model: {gguf_file}")
# Detect vision: check if base model is vision, then look for mmproj
mmproj_file = None
gguf_is_vision = False
gguf_dir = Path(gguf_file).parent
# Determine if this is a vision model from export metadata
base_is_vision = False
meta_path = gguf_dir / "export_metadata.json"
if meta_path.exists():
try:
meta = json.loads(meta_path.read_text())
base = meta.get("base_model")
if base and is_vision_model(base, hf_token=hf_token):
base_is_vision = True
logger.info(f"GGUF base model '{base}' is a vision model")
except Exception as e:
logger.debug(f"Could not read export metadata: {e}")
# If vision (or mmproj happens to exist), find the mmproj file
mmproj_file = detect_mmproj_file(gguf_file)
if mmproj_file:
gguf_is_vision = True
logger.info(f"Detected mmproj for vision: {mmproj_file}")
elif base_is_vision:
logger.warning(f"Base model is vision but no mmproj file found in {gguf_dir}")
return cls(
identifier=identifier,
display_name=display_name,
path=path,
is_local=True,
is_cached=True,
is_vision=False,
is_vision=gguf_is_vision,
is_lora=False,
is_gguf=True,
gguf_file=gguf_file,
gguf_mmproj_file=mmproj_file,
)
else:
# Check if the HF repo contains GGUF files