diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 3c77487efe..197ecf1a9f 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -78,6 +78,19 @@ from state.tool_approvals import ( logger = get_logger(__name__) +class LlamaServerNotFoundError(RuntimeError): + """GGUF model needs the llama.cpp runtime but no llama-server is installed. + Subclasses RuntimeError so existing handlers still catch it.""" + + +# Shared so the from_identifier preflight and the load-time raise stay in sync. +LLAMA_SERVER_NOT_FOUND_DETAIL = ( + "This is a GGUF model, but the llama.cpp runtime (llama-server) is not " + "installed. Run `unsloth studio setup` to download the prebuilt runtime, " + "then try again. (Advanced: set LLAMA_SERVER_PATH to an existing binary.)" +) + + # llama-server can serve HTTP 200 while running a model entirely on CPU when a # GPU backend fails to init (#5807 / #5106 / #5830). Classify the startup log so # Studio can warn. Priority: explicit "offloaded N/M layers to GPU" counts @@ -4343,11 +4356,11 @@ class LlamaCppBackend: "(access-denied; antivirus or an in-flight install). " "Retry the load once it is released." ) - raise RuntimeError( - "llama-server binary not found. " - "Run setup.sh to build it, install llama.cpp, " - "or set LLAMA_SERVER_PATH environment variable." - ) + # Reached only after the diffusion early-return above, so this is a + # genuine llama-server-backed GGUF with no runtime. Raise the typed + # error so /load returns the actionable 400 (not a generic 500), the + # same message remote validation already shows. + raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL) # Outside ``self._lock`` so /unload, /cancel, /status aren't # blocked. ``unload_model`` also records the kill, so the diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index be8bf77eb6..52540a5a72 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1951,6 +1951,8 @@ async def load_model( GGUF models load via llama-server (llama.cpp) instead of Unsloth. """ + from core.inference.llama_cpp import LlamaServerNotFoundError + native_grant_backed = False model_log_label = request.model_path try: @@ -2511,6 +2513,10 @@ async def load_model( logger.warning("Rejected inference GPU selection: %s", e) # User-facing validation (e.g. "Invalid gpu_ids [99]"): redact paths, keep detail. raise HTTPException(status_code = 400, detail = redact_native_paths(str(e))) + except LlamaServerNotFoundError as e: + # Missing GGUF runtime: 400 with the install message, not a generic 500. + logger.warning("GGUF runtime missing while loading '%s': %s", model_log_label, e) + raise HTTPException(status_code = 400, detail = str(e)) except Exception as e: # Friendlier message for models Unsloth cannot load. not_supported_hints = [ @@ -2620,6 +2626,8 @@ async def validate_model( Checks that ModelConfig.from_identifier() can resolve model_path, but does NOT load model weights into GPU memory. """ + from core.inference.llama_cpp import LlamaServerNotFoundError + native_grant_backed = False model_log_label = request.model_path try: @@ -2709,6 +2717,10 @@ async def validate_model( except HTTPException: raise + except LlamaServerNotFoundError as e: + # Missing GGUF runtime: 400 with the install message, not a generic "Invalid model". + logger.warning("GGUF runtime missing while validating '%s': %s", request.model_path, e) + raise HTTPException(status_code = 400, detail = str(e)) except Exception as e: not_supported_hints = [ "No config file found", diff --git a/studio/backend/tests/test_validate_gguf_runtime_message.py b/studio/backend/tests/test_validate_gguf_runtime_message.py new file mode 100644 index 0000000000..f612cc4a03 --- /dev/null +++ b/studio/backend/tests/test_validate_gguf_runtime_message.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""/api/inference/validate and /load must surface an actionable "install the runtime" +message when a GGUF model's llama-server is missing, not a generic error.""" + +import asyncio +import importlib.util +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from fastapi import HTTPException + +from core.inference.llama_cpp import LlamaServerNotFoundError +from models.inference import LoadRequest, ValidateModelRequest + +_BACKEND_ROOT = Path(__file__).resolve().parent.parent + + +def _load_route_module(name: str, relative_path: str): + # Load routes/inference.py under a standalone name (mirrors test_gpu_selection). + spec = importlib.util.spec_from_file_location(name, _BACKEND_ROOT / relative_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_GGUF_MSG = ( + "This is a GGUF model, but the llama.cpp runtime (llama-server) is not " + "installed. Run `unsloth studio setup` to download the prebuilt runtime, " + "then try again. (Advanced: set LLAMA_SERVER_PATH to an existing binary.)" +) + + +class TestValidateGgufRuntimeMessage(unittest.TestCase): + def _validate(self, route, model_path, side_effect): + request = ValidateModelRequest(model_path = model_path) + with ( + patch.object( + route, + "_resolve_model_identifier_for_request", + return_value = (model_path, model_path, False), + ), + patch.object(route.ModelConfig, "from_identifier", side_effect = side_effect), + ): + with self.assertRaises(HTTPException) as exc: + asyncio.run(route.validate_model(request, current_subject = "test-user")) + return exc.exception + + def test_missing_llama_server_returns_actionable_message(self): + route = _load_route_module("inf_route_runtime_msg_1", "routes/inference.py") + err = self._validate(route, "unsloth/Qwen3-1.7B-GGUF", LlamaServerNotFoundError(_GGUF_MSG)) + self.assertEqual(err.status_code, 400) + self.assertIn("unsloth studio setup", err.detail) + self.assertIn("llama.cpp runtime", err.detail) + self.assertNotEqual(err.detail, "Invalid model") + + def test_other_runtime_errors_do_not_get_gguf_message(self): + # LlamaServerNotFoundError subclasses RuntimeError, so a plain RuntimeError must not be + # routed to the GGUF "install the runtime" message. validate_model surfaces a RuntimeError's + # own message (#6398), so assert the GGUF install text is absent and the message is intact. + route = _load_route_module("inf_route_runtime_msg_2", "routes/inference.py") + err = self._validate(route, "not/a-real-model", RuntimeError("totally different failure")) + self.assertEqual(err.status_code, 400) + self.assertNotIn("unsloth studio setup", err.detail) + self.assertNotIn("llama.cpp runtime", err.detail) + self.assertEqual(err.detail, "totally different failure") + + +class TestLoadGgufRuntimeMessage(unittest.TestCase): + """/api/inference/load surfaces the same message (not a 500) when the runtime is missing.""" + + def _load(self, route, model_path, side_effect): + request = LoadRequest(model_path = model_path) + backend = MagicMock(active_model_name = None) # no resident model -> reach from_identifier + with ( + patch.object( + route, + "_resolve_model_identifier_for_request", + return_value = (model_path, model_path, False), + ), + patch.object(route, "resolve_effective_chat_template_override", return_value = None), + patch.object(route, "get_inference_backend", return_value = backend), + patch.object(route, "get_llama_cpp_backend", return_value = MagicMock()), + patch.object(route.ModelConfig, "from_identifier", side_effect = side_effect), + ): + with self.assertRaises(HTTPException) as exc: + asyncio.run(route.load_model(request, MagicMock(), current_subject = "test-user")) + return exc.exception + + def test_missing_llama_server_returns_actionable_message(self): + route = _load_route_module("inf_route_load_runtime_msg_1", "routes/inference.py") + err = self._load(route, "unsloth/Qwen3-1.7B-GGUF", LlamaServerNotFoundError(_GGUF_MSG)) + self.assertEqual(err.status_code, 400) + self.assertIn("unsloth studio setup", err.detail) + self.assertIn("llama.cpp runtime", err.detail) + + def test_other_load_errors_still_500(self): + route = _load_route_module("inf_route_load_runtime_msg_2", "routes/inference.py") + err = self._load(route, "unsloth/some-model", RuntimeError("totally different failure")) + self.assertEqual(err.status_code, 500) + + +class TestLoadPathPropagatesRuntimeError(unittest.TestCase): + """The backend GGUF load now raises LlamaServerNotFoundError when the runtime is + missing (after diffusion routing). The default (non-tensor) load must propagate it + to load_model's 400 arm, not swallow it into a generic 500.""" + + def test_tensor_fallback_propagates_missing_runtime(self): + from core.inference.tensor_fallback import load_with_tensor_fallback + async def _attempt(_tensor, _extra): + raise LlamaServerNotFoundError(_GGUF_MSG) + + with self.assertRaises(LlamaServerNotFoundError): + asyncio.run( + load_with_tensor_fallback(_attempt, requested_tensor = False, extra_args = None) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 45474389c8..11fe58e6c3 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -2604,13 +2604,14 @@ class ModelConfig: # download. include_denied: a transiently locked binary still # exists (the lock clears long before the download finishes; the # load itself reports a still-locked binary distinctly). - from core.inference.llama_cpp import LlamaCppBackend + from core.inference.llama_cpp import ( + LLAMA_SERVER_NOT_FOUND_DETAIL, + LlamaCppBackend, + LlamaServerNotFoundError, + ) if not LlamaCppBackend._find_llama_server_binary(include_denied = True): - raise RuntimeError( - "llama-server binary not found — cannot load GGUF models. " - "Run setup.sh to build it, or set LLAMA_SERVER_PATH." - ) + raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL) # list_gguf_variants() detects vision & resolves the variant variants, has_vision = list_gguf_variants(identifier, hf_token = hf_token)