diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index da49de000f..6ba4370aee 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -10544,6 +10544,13 @@ class LlamaCppBackend: # Process is alive: either a concurrent caller already respawned # it (healthy), or this connection error wasn't a dead server. return self._healthy + with self._mtp_runtime_fallback_lock: + if self._mtp_runtime_fallback_in_progress: + # An MTP-free reload already owns this corpse. Replaying the + # old kwargs would restart the crashing MTP config and make + # that reload abort on its "newer load is active" check. + logger.info("Respawn skipped: an MTP-free reload is already recovering.") + return False kwargs = self._last_load_kwargs if not kwargs: return False @@ -10569,6 +10576,12 @@ class LlamaCppBackend: effects. Resolve ``base_url`` on each attempt because a respawn may use a new port. The one-retry budget is per model request, not per chat turn, so a long tool loop never discards a completed tool just because an earlier turn recovered. + + A child that dies during prefill has already accepted the socket, so it + surfaces as ReadError/WriteError/RemoteProtocolError rather than + ConnectError; all of them mean the transport is gone. Timeouts are excluded + on purpose: they mean the server is slow, not dead, and replaying one would + just spend the first-token budget twice. """ for attempt in range(2): response_opened = False @@ -10578,7 +10591,7 @@ class LlamaCppBackend: response_opened = True yield opened return - except httpx.ConnectError as exc: + except (httpx.NetworkError, httpx.RemoteProtocolError) as exc: if response_opened: raise if self._maybe_recover_from_mtp_crash(exc): diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py index 35f95ccca9..9cbb45fda0 100644 --- a/studio/backend/tests/test_llama_cpp_tool_loop.py +++ b/studio/backend/tests/test_llama_cpp_tool_loop.py @@ -2422,6 +2422,59 @@ def test_connect_error_retry_is_bounded(monkeypatch): assert len(payloads) == 2 +def test_pre_header_transport_errors_also_respawn(monkeypatch): + """A child that dies during prefill already accepted the socket, so it does + not surface as ConnectError. Nothing has streamed yet, so replay is safe.""" + import httpx + for exc in ( + httpx.RemoteProtocolError("server disconnected without sending a response"), + httpx.ReadError("connection reset by peer"), + httpx.WriteError("broken pipe"), + ): + payloads: list[dict] = [] + backend = _make_backend( + monkeypatch, [exc, [_sse({"content": "Recovered."}), _done()]], payloads + ) + respawn_calls = _patch_successful_respawn(monkeypatch, backend) + + events = list( + backend.generate_chat_completion_with_tools( + messages = [{"role": "user", "content": "hello"}], + tools = [{"type": "function", "function": {"name": "python"}}], + max_tool_iterations = 1, + ) + ) + + assert respawn_calls == [True], type(exc).__name__ + assert len(payloads) == 2, type(exc).__name__ + assert any(e.get("type") == "content" and e.get("text") == "Recovered." for e in events) + + +def test_prefill_timeout_is_not_retried(monkeypatch): + """A slow-but-alive server must not have its first-token budget spent twice.""" + import httpx + for exc in (httpx.ReadTimeout("no first token"), httpx.PoolTimeout("pool")): + payloads: list[dict] = [] + backend = _make_backend(monkeypatch, [exc], payloads) + respawn_calls = _patch_successful_respawn(monkeypatch, backend) + + raised = False + try: + list( + backend.generate_chat_completion_with_tools( + messages = [{"role": "user", "content": "hello"}], + tools = [{"type": "function", "function": {"name": "python"}}], + max_tool_iterations = 1, + ) + ) + except httpx.TimeoutException: + raised = True + + assert raised, type(exc).__name__ + assert respawn_calls == [], type(exc).__name__ + assert len(payloads) == 1, type(exc).__name__ + + def test_mtp_crash_recovery_wins_over_respawn(monkeypatch): """An MTP crash reloads without MTP, so never respawn the same config on top.""" import httpx diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.py index 00c7aeac69..69733c8081 100644 --- a/studio/backend/tests/test_tensor_parallel.py +++ b/studio/backend/tests/test_tensor_parallel.py @@ -528,6 +528,25 @@ def test_runtime_recovery_is_single_flight(monkeypatch): release.set() +def test_respawn_defers_to_an_inflight_mtp_reload(monkeypatch): + # The single-flight "already recovering" False must not read as "not an MTP + # crash": respawning would replay the crashing MTP kwargs and make the + # in-flight no-MTP reload abort on its "newer load is active" check. + b = _recovery_backend() + b._mtp_runtime_fallback_in_progress = True + loads: list[dict] = [] + monkeypatch.setattr(b, "load_model", lambda **kwargs: loads.append(kwargs) or True) + + assert b._respawn_if_dead() is False + assert loads == [] + + # Once that reload finishes, an ordinary respawn works again. + b._mtp_runtime_fallback_in_progress = False + b._process.returncode = -9 # only the respawn path logs it + assert b._respawn_if_dead() is True + assert [kw.get("speculative_type") for kw in loads] == ["auto"] + + def test_runtime_recovery_rechecks_cancel_before_reload(): # recover() must re-check the cancel flag after the death poll (load_model # clears it), so a reload scheduled just before /unload can't resurrect it.