diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 3b438c8612..0682739e7e 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -29,6 +29,14 @@ from utils.models import extract_model_size_b as _extract_model_size_b def _friendly_error(exc: Exception) -> str: """Extract a user-friendly message from known llama-server errors.""" + # httpx transport-layer failures reaching the managed llama-server — + # raised by the async pass-through helpers that talk to llama-server + # directly. Treat any RequestError subclass (ConnectError, ReadError, + # RemoteProtocolError, WriteError, PoolTimeout, ...) as "the upstream + # subprocess is unreachable", which for Studio always means the + # llama-server subprocess crashed or is still coming up. + if isinstance(exc, httpx.RequestError): + return "Lost connection to the model server. It may have crashed -- try reloading the model." msg = str(exc) m = _re.search( r"request \((\d+) tokens?\) exceeds the available context size \((\d+) tokens?\)", @@ -3199,8 +3207,18 @@ async def _openai_passthrough_non_streaming( target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body(payload) - async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json = body, timeout = 600) + try: + async with httpx.AsyncClient() as client: + resp = await client.post(target_url, json = body, timeout = 600) + except httpx.RequestError as e: + # llama-server subprocess crashed / still starting / unreachable. + # Surface the same friendly message the sync chat path emits so + # operators don't see a bare 500 with no diagnostic. + logger.error("openai passthrough non-streaming: upstream unreachable: %s", e) + raise HTTPException( + status_code = 502, + detail = _friendly_error(e), + ) if resp.status_code != 200: raise HTTPException( diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index e3f4d6069e..4254dd9e89 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -12,6 +12,8 @@ Covers: - anthropic_tool_choice_to_openai() covers all four Anthropic shapes. - _build_passthrough_payload() honors a caller-supplied tool_choice and defaults to "auto" when unset. +- _friendly_error() maps httpx transport errors to a "Lost connection" + message so passthrough failures are legible instead of bare 500s. No running server or GPU required. """ @@ -22,6 +24,7 @@ import sys _backend = os.path.join(os.path.dirname(__file__), "..") sys.path.insert(0, _backend) +import httpx import pytest from pydantic import ValidationError @@ -32,7 +35,7 @@ from models.inference import ( from core.inference.anthropic_compat import ( anthropic_tool_choice_to_openai, ) -from routes.inference import _build_passthrough_payload +from routes.inference import _build_passthrough_payload, _friendly_error # ===================================================================== @@ -324,3 +327,52 @@ class TestBuildPassthroughPayloadToolChoice: body = _build_passthrough_payload(**self._args(), repetition_penalty = 1.1) assert body.get("repeat_penalty") == 1.1 assert "repetition_penalty" not in body + + +# ===================================================================== +# _friendly_error — httpx transport failures +# ===================================================================== + + +class TestFriendlyErrorHttpx: + """The async pass-through helpers talk to llama-server via httpx. + When the subprocess is down, httpx raises RequestError subclasses + whose string form (``"All connection attempts failed"``, ``"[Errno 111] + Connection refused"``, ...) does NOT contain the substring + ``"Lost connection to llama-server"`` the sync path uses, so the + previous substring-only `_friendly_error` returned a useless generic + message. These tests pin the new isinstance-based mapping. + """ + + def _req(self): + return httpx.Request("POST", "http://127.0.0.1:65535/v1/chat/completions") + + def test_connect_error_mapped(self): + exc = httpx.ConnectError("All connection attempts failed", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_read_error_mapped(self): + exc = httpx.ReadError("EOF", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_remote_protocol_error_mapped(self): + exc = httpx.RemoteProtocolError("peer closed", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_read_timeout_mapped(self): + exc = httpx.ReadTimeout("timed out", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_non_httpx_unchanged(self): + # Non-httpx exceptions still fall through to the existing substring + # heuristics — a context-size message must still produce the + # "Message too long" path. + ctx_msg = ( + "request (4096 tokens) exceeds the available context size (2048 tokens)" + ) + assert "Message too long" in _friendly_error(ValueError(ctx_msg)) + + def test_generic_exception_returns_generic_message(self): + assert ( + _friendly_error(RuntimeError("unrelated")) == "An internal error occurred" + )