ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: show an actionable message when the GGUF runtime is missing (#6327)

Browse source 
* Studio: show an actionable message when the GGUF runtime is missing

Selecting a GGUF model with no llama-server installed surfaced a generic
"Invalid model" in the UI, because validate_model's catch-all discarded the
real cause. Add LlamaServerNotFoundError (a RuntimeError subclass) raised by the
GGUF preflight in ModelConfig.from_identifier, and catch it in the validate
route so users get an actionable message: run `unsloth studio setup` to
download the prebuilt llama.cpp runtime. Other validation failures keep the safe
generic message. Adds a regression test.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Studio: also map missing GGUF runtime to a 400 in load_model

validate_model already surfaces the actionable 'install the runtime'
message for LlamaServerNotFoundError; load_model fell through to the
generic 500 'Failed to load model'. Catch it there too so a GGUF load
without llama-server gives the same install hint instead of a 500.

* Trim comments for PR #6327

* Studio: fix stale validate test after #6398 and surface missing GGUF runtime on /load

- test_other_runtime_errors_do_not_get_gguf_message: after merging #6398,
  validate_model surfaces a RuntimeError's own message, so a plain RuntimeError
  no longer returns "Invalid model". Assert it does not receive the GGUF
  install message instead (the prior assertion was stale after the main merge).
- Raise LlamaServerNotFoundError (not a plain RuntimeError) at the backend
  load-time missing-binary branch, after diffusion routing, so /load returns the
  actionable 400 like remote validation, instead of a generic 500.
- Share LLAMA_SERVER_NOT_FOUND_DETAIL between the from_identifier preflight and
  the load-time raise so the message stays in sync.
- Add a propagation regression test for the non-tensor load path.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Daniel Han 2026-06-18 06:00:00 -07:00 • committed by GitHub
commit 8d804c9413
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 158 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

23
studio/backend/core/inference/llama_cpp.py
View file

@ -78,6 +78,19 @@ from state.tool_approvals import (
logger = get_logger(__name__)
class LlamaServerNotFoundError(RuntimeError):
"""GGUF model needs the llama.cpp runtime but no llama-server is installed.
Subclasses RuntimeError so existing handlers still catch it."""
# Shared so the from_identifier preflight and the load-time raise stay in sync.
LLAMA_SERVER_NOT_FOUND_DETAIL = (
"This is a GGUF model, but the llama.cpp runtime (llama-server) is not "
"installed. Run `unsloth studio setup` to download the prebuilt runtime, "
"then try again. (Advanced: set LLAMA_SERVER_PATH to an existing binary.)"
)
# llama-server can serve HTTP 200 while running a model entirely on CPU when a
# GPU backend fails to init (#5807 / #5106 / #5830). Classify the startup log so
# Studio can warn. Priority: explicit "offloaded N/M layers to GPU" counts
@ -4343,11 +4356,11 @@ class LlamaCppBackend:
"(access-denied; antivirus or an in-flight install). "
"Retry the load once it is released."
)
raise RuntimeError(
"llama-server binary not found. "
"Run setup.sh to build it, install llama.cpp, "
"or set LLAMA_SERVER_PATH environment variable."
)
# Reached only after the diffusion early-return above, so this is a
# genuine llama-server-backed GGUF with no runtime. Raise the typed
# error so /load returns the actionable 400 (not a generic 500), the
# same message remote validation already shows.
raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)
# Outside ``self._lock`` so /unload, /cancel, /status aren't
# blocked. ``unload_model`` also records the kill, so the

12
studio/backend/routes/inference.py
View file

@ -1951,6 +1951,8 @@ async def load_model(
GGUF models load via llama-server (llama.cpp) instead of Unsloth.
"""
from core.inference.llama_cpp import LlamaServerNotFoundError
native_grant_backed = False
model_log_label = request.model_path
try:
@ -2511,6 +2513,10 @@ async def load_model(
logger.warning("Rejected inference GPU selection: %s", e)
# User-facing validation (e.g. "Invalid gpu_ids [99]"): redact paths, keep detail.
raise HTTPException(status_code = 400, detail = redact_native_paths(str(e)))
except LlamaServerNotFoundError as e:
# Missing GGUF runtime: 400 with the install message, not a generic 500.
logger.warning("GGUF runtime missing while loading '%s': %s", model_log_label, e)
raise HTTPException(status_code = 400, detail = str(e))
except Exception as e:
# Friendlier message for models Unsloth cannot load.
not_supported_hints = [
@ -2620,6 +2626,8 @@ async def validate_model(
Checks that ModelConfig.from_identifier() can resolve model_path, but does
NOT load model weights into GPU memory.
"""
from core.inference.llama_cpp import LlamaServerNotFoundError
native_grant_backed = False
model_log_label = request.model_path
try:
@ -2709,6 +2717,10 @@ async def validate_model(
except HTTPException:
raise
except LlamaServerNotFoundError as e:
# Missing GGUF runtime: 400 with the install message, not a generic "Invalid model".
logger.warning("GGUF runtime missing while validating '%s': %s", request.model_path, e)
raise HTTPException(status_code = 400, detail = str(e))
except Exception as e:
not_supported_hints = [
"No config file found",

122
studio/backend/tests/test_validate_gguf_runtime_message.py Normal file
View file

@ -0,0 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""/api/inference/validate and /load must surface an actionable "install the runtime"
message when a GGUF model's llama-server is missing, not a generic error."""
import asyncio
import importlib.util
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from fastapi import HTTPException
from core.inference.llama_cpp import LlamaServerNotFoundError
from models.inference import LoadRequest, ValidateModelRequest
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
def _load_route_module(name: str, relative_path: str):
# Load routes/inference.py under a standalone name (mirrors test_gpu_selection).
spec = importlib.util.spec_from_file_location(name, _BACKEND_ROOT / relative_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
_GGUF_MSG = (
"This is a GGUF model, but the llama.cpp runtime (llama-server) is not "
"installed. Run `unsloth studio setup` to download the prebuilt runtime, "
"then try again. (Advanced: set LLAMA_SERVER_PATH to an existing binary.)"
)
class TestValidateGgufRuntimeMessage(unittest.TestCase):
def _validate(self, route, model_path, side_effect):
request = ValidateModelRequest(model_path = model_path)
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = (model_path, model_path, False),
),
patch.object(route.ModelConfig, "from_identifier", side_effect = side_effect),
):
with self.assertRaises(HTTPException) as exc:
asyncio.run(route.validate_model(request, current_subject = "test-user"))
return exc.exception
def test_missing_llama_server_returns_actionable_message(self):
route = _load_route_module("inf_route_runtime_msg_1", "routes/inference.py")
err = self._validate(route, "unsloth/Qwen3-1.7B-GGUF", LlamaServerNotFoundError(_GGUF_MSG))
self.assertEqual(err.status_code, 400)
self.assertIn("unsloth studio setup", err.detail)
self.assertIn("llama.cpp runtime", err.detail)
self.assertNotEqual(err.detail, "Invalid model")
def test_other_runtime_errors_do_not_get_gguf_message(self):
# LlamaServerNotFoundError subclasses RuntimeError, so a plain RuntimeError must not be
# routed to the GGUF "install the runtime" message. validate_model surfaces a RuntimeError's
# own message (#6398), so assert the GGUF install text is absent and the message is intact.
route = _load_route_module("inf_route_runtime_msg_2", "routes/inference.py")
err = self._validate(route, "not/a-real-model", RuntimeError("totally different failure"))
self.assertEqual(err.status_code, 400)
self.assertNotIn("unsloth studio setup", err.detail)
self.assertNotIn("llama.cpp runtime", err.detail)
self.assertEqual(err.detail, "totally different failure")
class TestLoadGgufRuntimeMessage(unittest.TestCase):
"""/api/inference/load surfaces the same message (not a 500) when the runtime is missing."""
def _load(self, route, model_path, side_effect):
request = LoadRequest(model_path = model_path)
backend = MagicMock(active_model_name = None) # no resident model -> reach from_identifier
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = (model_path, model_path, False),
),
patch.object(route, "resolve_effective_chat_template_override", return_value = None),
patch.object(route, "get_inference_backend", return_value = backend),
patch.object(route, "get_llama_cpp_backend", return_value = MagicMock()),
patch.object(route.ModelConfig, "from_identifier", side_effect = side_effect),
):
with self.assertRaises(HTTPException) as exc:
asyncio.run(route.load_model(request, MagicMock(), current_subject = "test-user"))
return exc.exception
def test_missing_llama_server_returns_actionable_message(self):
route = _load_route_module("inf_route_load_runtime_msg_1", "routes/inference.py")
err = self._load(route, "unsloth/Qwen3-1.7B-GGUF", LlamaServerNotFoundError(_GGUF_MSG))
self.assertEqual(err.status_code, 400)
self.assertIn("unsloth studio setup", err.detail)
self.assertIn("llama.cpp runtime", err.detail)
def test_other_load_errors_still_500(self):
route = _load_route_module("inf_route_load_runtime_msg_2", "routes/inference.py")
err = self._load(route, "unsloth/some-model", RuntimeError("totally different failure"))
self.assertEqual(err.status_code, 500)
class TestLoadPathPropagatesRuntimeError(unittest.TestCase):
"""The backend GGUF load now raises LlamaServerNotFoundError when the runtime is
missing (after diffusion routing). The default (non-tensor) load must propagate it
to load_model's 400 arm, not swallow it into a generic 500."""
def test_tensor_fallback_propagates_missing_runtime(self):
from core.inference.tensor_fallback import load_with_tensor_fallback
async def _attempt(_tensor, _extra):
raise LlamaServerNotFoundError(_GGUF_MSG)
with self.assertRaises(LlamaServerNotFoundError):
asyncio.run(
load_with_tensor_fallback(_attempt, requested_tensor = False, extra_args = None)
)
if __name__ == "__main__":
unittest.main()

11
studio/backend/utils/models/model_config.py
View file

@ -2604,13 +2604,14 @@ class ModelConfig:
# download. include_denied: a transiently locked binary still
# exists (the lock clears long before the download finishes; the
# load itself reports a still-locked binary distinctly).
from core.inference.llama_cpp import LlamaCppBackend
from core.inference.llama_cpp import (
LLAMA_SERVER_NOT_FOUND_DETAIL,
LlamaCppBackend,
LlamaServerNotFoundError,
)
if not LlamaCppBackend._find_llama_server_binary(include_denied = True):
raise RuntimeError(
"llama-server binary not found — cannot load GGUF models. "
"Run setup.sh to build it, or set LLAMA_SERVER_PATH."
)
raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)
# list_gguf_variants() detects vision & resolves the variant
variants, has_vision = list_gguf_variants(identifier, hf_token = hf_token)