From bc85ecd145ed5ceae8dab662e3686485c1e6a559 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 11 Jun 2026 07:49:55 -0700 Subject: [PATCH] Studio: report the real llama-server context window and add an opt-in overflow policy for OpenAI-compatible serving (#6164) * Studio: report the real llama-server context window and add an opt-in overflow policy for OpenAI-compatible serving A community report showed OpenCode failing tool calls every few minutes against Studio's OpenAI-compatible API while the same GGUF was stable on LM Studio. Root cause: Studio advertises the requested context length, but llama-server can allocate less (memory-fit step on small GPUs, --parallel slot split), so clients budget against a window that does not exist. Their generations truncate mid tool call at the real wall (finish_reason=length with cut JSON arguments) and eventually the prompt itself exceeds the real window, returning a 400 that agentic clients treat as non-retryable. Changes: - After llama-server health, read default_generation_settings.n_ctx from /props and adopt it whenever it is below Studio's computed context, with a warning. The load response, status route, UI value, and the passthrough max_tokens ceiling all become honest automatically. - Expose context_length and max_context_length on /v1/models so clients can budget against the enforced window. - Accept empty role=tool content (commands with no output are routine in agentic loops; OpenAI and llama-server both accept it) instead of a 400. - Add context_overflow=truncate_middle (per request, or server-wide via UNSLOTH_CONTEXT_OVERFLOW=truncate_middle): on exceed_context_size_error the passthrough drops whole middle turn-groups (system prompt, first turn, and recent turns kept; tool calls stay paired with their results), clips oversized contents middle-out when group-dropping is not enough, clamps max_tokens to the generation headroom, and retries. Default stays 'error' with code=context_length_exceeded so clients running their own compaction keep full control. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: allocate the requested context for real (kv-unified, fit-ctx floor) Two launch-flag gaps caused the advertised vs allocated divergence at the source: - llama-server enables --kv-unified only when the slot count is auto; Studio always passes --parallel N, which silently splits -c into per-slot windows of -c/N. Pass --kv-unified when N > 1 so a single request can use the full advertised window (same total KV memory, shared pool). - with --fit on the fit step may set ctx as low as 4096; pass --fit-ctx for explicit requests so fit offloads or fails into the existing --fit off retry instead of silently shrinking the window. Both flags are gated on --help capability probing so older builds keep the current behavior, where the /props readback remains the backstop. Verified live: -c 98304 --parallel 4 now serves per-slot n_ctx 98304 (was 24576), 48k-token requests pass through the passthrough, and the readback warning no longer fires. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/core/inference/llama_cpp.py | 78 +++++ studio/backend/models/inference.py | 16 +- studio/backend/routes/inference.py | 288 +++++++++++++++--- .../tests/test_context_overflow_truncation.py | 277 +++++++++++++++++ .../tests/test_llama_cpp_props_readback.py | 254 +++++++++++++++ .../tests/test_openai_tool_passthrough.py | 9 +- .../tests/test_tool_message_empty_content.py | 53 ++++ 7 files changed, 924 insertions(+), 51 deletions(-) create mode 100644 studio/backend/tests/test_context_overflow_truncation.py create mode 100644 studio/backend/tests/test_llama_cpp_props_readback.py create mode 100644 studio/backend/tests/test_tool_message_empty_content.py diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index e46d464784..2560f9e5ac 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1153,6 +1153,8 @@ class LlamaCppBackend: "ngram_mod_flavor": None, "supports_ngram_mod": False, "spec_draft_n_max_flag": None, + "supports_kv_unified": False, + "supports_fit_ctx": False, } try: mtime = int(Path(bin_path).stat().st_mtime) @@ -1166,6 +1168,8 @@ class LlamaCppBackend: mtp_token: Optional[str] = None ngram_mod_flavor: Optional[str] = None spec_draft_n_max_flag: Optional[str] = None + supports_kv_unified = False + supports_fit_ctx = False try: result = subprocess.run( [bin_path, "--help"], @@ -1253,6 +1257,9 @@ class LlamaCppBackend: spec_draft_n_max_flag = "--spec-draft-n-max" elif _is_real("--draft-max"): spec_draft_n_max_flag = "--draft-max" + + supports_kv_unified = _is_real("--kv-unified") + supports_fit_ctx = _is_real("--fit-ctx") except (OSError, subprocess.SubprocessError) as exc: logger.debug(f"llama-server --help probe failed: {exc}") @@ -1263,6 +1270,8 @@ class LlamaCppBackend: "ngram_mod_flavor": ngram_mod_flavor, "supports_ngram_mod": ngram_mod_flavor is not None, "spec_draft_n_max_flag": spec_draft_n_max_flag, + "supports_kv_unified": supports_kv_unified, + "supports_fit_ctx": supports_fit_ctx, } cls._capability_cache[cache_key] = info return info @@ -3232,6 +3241,16 @@ class LlamaCppBackend: # Fits on selected GPU(s) -- offload all layers cmd.extend(["-ngl", "-1"]) + cmd.extend( + self._ctx_integrity_flags( + n_parallel, + use_fit, + requested_ctx, + effective_ctx, + self.probe_server_capabilities(binary), + ) + ) + # -1 = llama.cpp auto-detect (physical cores). Pass explicitly # so we don't inherit llama-server's internal default, which # has varied (hardware concurrency incl. hyperthreads on some @@ -3585,6 +3604,7 @@ class LlamaCppBackend: self._effective_context_length = ( effective_ctx if effective_ctx > 0 else self._context_length ) + self._reconcile_effective_ctx_with_server() self._max_context_length = ( max_available_ctx if max_available_ctx > 0 else self._effective_context_length ) @@ -4468,6 +4488,64 @@ class LlamaCppBackend: logger.error(f"llama-server health check timed out after {timeout}s") return False + @staticmethod + def _ctx_integrity_flags( + n_parallel: int, use_fit: bool, requested_ctx: int, effective_ctx: int, caps: dict + ) -> list[str]: + """Flags that keep the per-request window equal to the advertised ctx. + + Explicit ``--parallel`` disables llama-server's auto-slots + ``--kv-unified`` default, silently splitting ``-c`` into per-slot + windows of ``-c / N``; restore the shared pool so one request can use + the full context. With ``--fit on``, ``--fit-ctx`` floors the fit step + at an explicitly requested ctx (default floor is 4096) so it offloads + or fails instead of silently shrinking the window. + """ + flags: list[str] = [] + if n_parallel > 1 and caps.get("supports_kv_unified"): + flags.append("--kv-unified") + if use_fit and requested_ctx > 0 and effective_ctx > 0 and caps.get("supports_fit_ctx"): + flags.extend(["--fit-ctx", str(effective_ctx)]) + return flags + + def _query_server_n_ctx(self) -> Optional[int]: + """Per-slot context llama-server actually allocated, from ``/props``. + + The memory-fit step or ``--parallel`` slot split can leave this below + the requested ``-c``; requests are validated against this value. + """ + url = f"http://127.0.0.1:{self._port}/props" + try: + resp = httpx.get(url, timeout = 5.0) + if resp.status_code != 200: + return None + settings = resp.json().get("default_generation_settings") or {} + n_ctx = settings.get("n_ctx") + return int(n_ctx) if n_ctx else None + except Exception: + return None + + def _reconcile_effective_ctx_with_server(self) -> None: + """Adopt the server's real ``n_ctx`` when it is below Studio's value. + + Keeps ``context_length`` (load response, status route, passthrough + ``max_tokens`` ceiling) honest; clients sized to the requested value + would otherwise hit ``exceed_context_size_error`` 400s early. + """ + actual_n_ctx = self._query_server_n_ctx() + if not actual_n_ctx or actual_n_ctx <= 0: + return + if self._effective_context_length and actual_n_ctx < self._effective_context_length: + logger.warning( + "llama-server allocated a smaller per-request context than " + f"requested ({self._effective_context_length} -> {actual_n_ctx}; " + "memory fit or --parallel slot split); clients must treat " + f"{actual_n_ctx} as the real context window." + ) + self._effective_context_length = actual_n_ctx + elif not self._effective_context_length: + self._effective_context_length = actual_n_ctx + # ── Message building (OpenAI format) ────────────────────────── @staticmethod diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index f58a76d16e..f94d5366eb 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -544,8 +544,10 @@ class ChatMessage(BaseModel): if self.role == "tool": # tool_call_id resolution happens at ChatCompletionRequest scope. - if not self.content: - raise ValueError('role="tool" messages require non-empty "content".') + # OpenAI accepts empty tool results (commands with no output); + # normalize to "" instead of a 400 agentic clients treat as fatal. + if self.content is None or self.content == []: + self.content = "" elif self.role == "assistant": # Post-Stop sentinel: collapse content="" / [] to None. if (self.content == "" or self.content == []) and not self.tool_calls: @@ -692,6 +694,16 @@ class ChatCompletionRequest(BaseModel): True, description = "[x-unsloth] Auto-detect and fix malformed tool calls from model output.", ) + context_overflow: Optional[Literal["error", "truncate_middle"]] = Field( + None, + description = ( + "[x-unsloth] Passthrough behavior when the prompt exceeds the real " + "context window. 'error' (default) returns a 400 with " + "code=context_length_exceeded. 'truncate_middle' drops middle " + "turn-groups (system prompt, first turn, and recent turns kept; " + "tool calls stay paired with their results) and retries." + ), + ) max_tool_calls_per_message: Optional[int] = Field( 25, ge = 0, diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0244956b72..9dc9feb920 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -242,6 +242,169 @@ def _openai_passthrough_error(status_code, text) -> "HTTPException": ) +_OVERFLOW_TRUNCATE_MAX_RETRIES = 3 +# Truncated-prompt share of the real window; the rest is generation headroom +# so a near-full prompt cannot cut a tool call mid-JSON at the wall. +_OVERFLOW_PROMPT_TARGET_FRACTION = 0.75 + + +def _overflow_truncation_requested(payload) -> bool: + """True when the request (or the UNSLOTH_CONTEXT_OVERFLOW server default, + for clients that cannot send custom fields) opted into truncation.""" + requested = getattr(payload, "context_overflow", None) + if requested is not None: + return requested == "truncate_middle" + return os.environ.get("UNSLOTH_CONTEXT_OVERFLOW", "").strip().lower() == "truncate_middle" + + +def _parse_overflow_counts(err_text: str): + """(n_prompt_tokens, n_ctx) from an exceed_context_size_error body, or + None. Tolerates \\" around keys (body may be a re-wrapped JSON string).""" + m_prompt = _re.search(r'n_prompt_tokens\\?"?\s*:\s*(\d+)', err_text) + m_ctx = _re.search(r'n_ctx\\?"?\s*:\s*(\d+)', err_text) + if m_prompt and m_ctx: + return int(m_prompt.group(1)), int(m_ctx.group(1)) + return None + + +def _estimate_message_tokens(msg: dict) -> int: + try: + return max(1, len(json.dumps(msg, ensure_ascii = False)) // 4) + except Exception: + return 1 + + +def _truncate_middle_messages(messages: list, keep_ratio: float): + """Drop whole turn-groups from the middle of an OpenAI message list. + + Always kept: leading system message(s), the first group (task anchor), + and the trailing groups. A group is a user message, or an assistant + message plus its following tool results, so surviving tool_calls stay + paired with their results as chat templates require. + Returns (new_messages, dropped_message_count). + """ + if not messages or keep_ratio >= 1.0: + return messages, 0 + + head: list = [] + idx = 0 + while idx < len(messages) and messages[idx].get("role") in ("system", "developer"): + head.append(messages[idx]) + idx += 1 + + groups: list[list] = [] + for msg in messages[idx:]: + role = msg.get("role") + if role == "tool" and groups: + groups[-1].append(msg) + elif role == "tool": + groups.append([msg]) # orphan tool result; treat as its own group + else: + groups.append([msg]) + + # Anchor group plus the last 3 groups stay. + protected_tail = min(3, max(1, len(groups) - 1)) + if len(groups) <= 1 + protected_tail: + return messages, 0 + + total_est = sum(_estimate_message_tokens(m) for m in messages) + target_est = int(total_est * keep_ratio) + + anchor = groups[0] + middle = groups[1:-protected_tail] + tail = groups[-protected_tail:] + + current_est = total_est + kept_middle: list[list] = list(middle) + dropped = 0 + # Drop oldest-first until the estimate fits the target. + while kept_middle and current_est > target_est: + victim = kept_middle.pop(0) + dropped += len(victim) + current_est -= sum(_estimate_message_tokens(m) for m in victim) + + if dropped == 0: + return messages, 0 + + new_messages = head + anchor + for grp in kept_middle: + new_messages.extend(grp) + for grp in tail: + new_messages.extend(grp) + return new_messages, dropped + + +_CLIP_MARKER = "\n[... truncated by context_overflow=truncate_middle ...]\n" +# Generous head+tail first; cut harder if the estimate still misses the target. +_CLIP_KEEP_CHARS = (1500, 400) + + +def _clip_long_contents(messages: list, target_est: int) -> int: + """Clip oversized string contents middle-out until ``target_est`` is met. + + Tool results first, then earlier user turns, the final message last. + Message count and roles never change, so tool pairing holds even when + group-dropping could not free enough. Returns messages clipped. + """ + + def _candidates(): + tools = [m for m in messages if m.get("role") == "tool"] + users = [m for m in messages[:-1] if m.get("role") == "user"] + last = [messages[-1]] if messages else [] + return tools + users + last + + clipped = 0 + for keep in _CLIP_KEEP_CHARS: + for msg in _candidates(): + if sum(_estimate_message_tokens(m) for m in messages) <= target_est: + return clipped + content = msg.get("content") + if not isinstance(content, str) or len(content) <= 2 * keep + len(_CLIP_MARKER): + continue + msg["content"] = content[:keep] + _CLIP_MARKER + content[-keep:] + clipped += 1 + return clipped + + +def _apply_overflow_truncation(body: dict, err_text: str) -> bool: + """Shrink a passthrough body after an upstream context overflow: drop + middle turn-groups, clip still-oversized contents, clamp ``max_tokens`` + to the generation headroom. Returns False when nothing could shrink.""" + counts = _parse_overflow_counts(err_text) + messages = body.get("messages") or [] + total_est = sum(_estimate_message_tokens(m) for m in messages) + if counts: + n_prompt, n_ctx = counts + keep_ratio = min(0.95, (_OVERFLOW_PROMPT_TARGET_FRACTION * n_ctx) / max(1, n_prompt)) + # Scale the server-token target into char-estimate units. + target_est = int(total_est * keep_ratio) + else: + n_ctx = None + keep_ratio = 0.6 # no counts in the error; cut conservatively + target_est = int(total_est * keep_ratio) + + new_messages, dropped = _truncate_middle_messages(messages, keep_ratio) + if dropped: + body["messages"] = new_messages + clipped = 0 + if sum(_estimate_message_tokens(m) for m in body.get("messages") or []) > target_est: + clipped = _clip_long_contents(body.get("messages") or [], target_est) + if not dropped and not clipped: + return False + if n_ctx: + headroom = max(1024, int(n_ctx * (1.0 - _OVERFLOW_PROMPT_TARGET_FRACTION))) + cur_max = body.get("max_tokens") + body["max_tokens"] = min(cur_max, headroom) if cur_max else headroom + logger.warning( + "context_overflow=truncate_middle: dropped %d middle messages, clipped " + "%d contents (keep_ratio %.2f); retrying within the real window", + dropped, + clipped, + keep_ratio, + ) + return True + + def _anthropic_stream_error_event(exc): """Anthropic in-band SSE ``error`` event for a mid-stream failure, or ``None`` to fall through to a normal message_delta finish. Returns an event only for a @@ -4456,26 +4619,35 @@ def _openai_model_objects() -> list[dict]: # Check GGUF backend llama_backend = get_llama_cpp_backend() if llama_backend.is_loaded: - models.append( - { - "id": llama_backend.model_identifier, - "object": "model", - "created": _created, - "owned_by": "local", - } - ) + entry = { + "id": llama_backend.model_identifier, + "object": "model", + "created": _created, + "owned_by": "local", + } + # Extension fields: the real per-request window (post /props readback) + # so clients can budget/compact against the enforced limit. + if llama_backend.context_length: + entry["context_length"] = llama_backend.context_length + if llama_backend.max_context_length: + entry["max_context_length"] = llama_backend.max_context_length + models.append(entry) # Check Unsloth backend backend = get_inference_backend() if backend.active_model_name: - models.append( - { - "id": backend.active_model_name, - "object": "model", - "created": _created, - "owned_by": "local", - } + entry = { + "id": backend.active_model_name, + "object": "model", + "created": _created, + "owned_by": "local", + } + _sf_ctx = getattr(backend, "context_length", None) or getattr( + backend, "max_seq_length", None ) + if _sf_ctx: + entry["context_length"] = _sf_ctx + models.append(entry) return models @@ -6822,27 +6994,32 @@ async def _openai_passthrough_stream( limits = httpx.Limits(max_keepalive_connections = 0), ) resp = None - try: - req = client.build_request("POST", target_url, json = body) - resp = await client.send(req, stream = True) - except httpx.RequestError as e: - # llama-server subprocess crashed / starting / unreachable. - logger.error("openai passthrough stream: upstream unreachable: %s", e) - if resp is not None: + _truncate_budget = ( + _OVERFLOW_TRUNCATE_MAX_RETRIES if _overflow_truncation_requested(payload) else 0 + ) + while True: + try: + req = client.build_request("POST", target_url, json = body) + resp = await client.send(req, stream = True) + except httpx.RequestError as e: + # llama-server subprocess crashed / starting / unreachable. + logger.error("openai passthrough stream: upstream unreachable: %s", e) + if resp is not None: + try: + await resp.aclose() + except Exception: + pass try: - await resp.aclose() + await client.aclose() except Exception: pass - try: - await client.aclose() - except Exception: - pass - raise HTTPException( - status_code = 502, - detail = _friendly_error(e), - ) + raise HTTPException( + status_code = 502, + detail = _friendly_error(e), + ) - if resp.status_code != 200: + if resp.status_code == 200: + break err_bytes = await resp.aread() err_text = err_bytes.decode("utf-8", errors = "replace") logger.error( @@ -6855,6 +7032,14 @@ async def _openai_passthrough_stream( await resp.aclose() except Exception: pass + # Opt-in overflow policy: shrink and retry instead of a fatal 400. + if ( + _truncate_budget > 0 + and _classify_llama_generation_error(Exception(err_text)) + and _apply_overflow_truncation(body, err_text) + ): + _truncate_budget -= 1 + continue try: await client.aclose() except Exception: @@ -6953,20 +7138,33 @@ async def _openai_passthrough_non_streaming(llama_backend, payload, model_name): payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend ) - try: - async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json = body, timeout = 600) - except httpx.RequestError as e: - # llama-server subprocess crashed / starting / unreachable. Surface the - # same friendly message the sync chat path emits so operators don't see - # a bare 500 with no diagnostic. - logger.error("openai passthrough non-streaming: upstream unreachable: %s", e) - raise HTTPException( - status_code = 502, - detail = _friendly_error(e), - ) + _truncate_budget = ( + _OVERFLOW_TRUNCATE_MAX_RETRIES if _overflow_truncation_requested(payload) else 0 + ) + while True: + try: + async with httpx.AsyncClient() as client: + resp = await client.post(target_url, json = body, timeout = 600) + except httpx.RequestError as e: + # llama-server subprocess crashed / starting / unreachable. Surface the + # same friendly message the sync chat path emits so operators don't see + # a bare 500 with no diagnostic. + logger.error("openai passthrough non-streaming: upstream unreachable: %s", e) + raise HTTPException( + status_code = 502, + detail = _friendly_error(e), + ) - if resp.status_code != 200: + if resp.status_code == 200: + break + # Opt-in overflow policy: shrink and retry instead of a fatal 400. + if ( + _truncate_budget > 0 + and _classify_llama_generation_error(Exception(resp.text)) + and _apply_overflow_truncation(body, resp.text) + ): + _truncate_budget -= 1 + continue raise _openai_passthrough_error(resp.status_code, resp.text) # The guided-decoding fence wraps each choice's JSON content in a diff --git a/studio/backend/tests/test_context_overflow_truncation.py b/studio/backend/tests/test_context_overflow_truncation.py new file mode 100644 index 0000000000..4f4c240934 --- /dev/null +++ b/studio/backend/tests/test_context_overflow_truncation.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for the opt-in ``context_overflow="truncate_middle"`` passthrough policy. + +On ``exceed_context_size_error`` the passthrough drops middle turn-groups and +retries inside the real window instead of surfacing a fatal 400. Truncation +keeps the system prompt, the first turn, and recent turns, and never orphans +a tool result from its tool_calls turn. Also covers ``/v1/models`` exposing +the real post-readback context window. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +from routes.inference import ( + _apply_overflow_truncation, + _clip_long_contents, + _CLIP_MARKER, + _estimate_message_tokens, + _openai_model_objects, + _overflow_truncation_requested, + _parse_overflow_counts, + _truncate_middle_messages, +) +import routes.inference as routes_mod + + +# Nick's actual error body from the Discord report logs. +_NICK_ERROR = ( + '{"detail":"llama-server error: {\\"error\\":{\\"code\\":400,' + '\\"message\\":\\"request (70494 tokens) exceeds the available context size ' + '(67584 tokens), try increasing it\\",\\"type\\":\\"exceed_context_size_error\\",' + '\\"n_prompt_tokens\\":70494,\\"n_ctx\\":67584}}"}' +) + + +def _tool_turn(i: int, result_chars: int = 400) -> list[dict]: + """An assistant tool_calls turn paired with its tool result.""" + return [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": f"call_{i}", + "type": "function", + "function": {"name": "read", "arguments": f'{{"filePath":"/f{i}"}}'}, + } + ], + }, + {"role": "tool", "tool_call_id": f"call_{i}", "content": "x" * result_chars}, + ] + + +def _conversation(n_tool_turns: int = 12) -> list[dict]: + msgs = [ + {"role": "system", "content": "You are an agent." * 20}, + {"role": "user", "content": "Do the big task." * 20}, + ] + for i in range(n_tool_turns): + msgs.extend(_tool_turn(i)) + msgs.append({"role": "assistant", "content": "halfway summary"}) + msgs.append({"role": "user", "content": "keep going"}) + return msgs + + +# --------------------------------------------------------------------------- +# _parse_overflow_counts +# --------------------------------------------------------------------------- + + +def test_parse_overflow_counts_nick_error(): + assert _parse_overflow_counts(_NICK_ERROR) == (70494, 67584) + + +def test_parse_overflow_counts_missing_fields(): + assert _parse_overflow_counts('{"error":"something else"}') is None + + +# --------------------------------------------------------------------------- +# _truncate_middle_messages +# --------------------------------------------------------------------------- + + +def test_truncation_drops_middle_keeps_anchors(): + msgs = _conversation() + new, dropped = _truncate_middle_messages(msgs, keep_ratio = 0.5) + assert dropped > 0 + assert len(new) == len(msgs) - dropped + # System prompt and task anchor survive. + assert new[0]["role"] == "system" + assert new[1] == msgs[1] + # The most recent turns survive verbatim. + assert new[-1] == msgs[-1] + assert new[-2] == msgs[-2] + + +def test_truncation_never_orphans_tool_results(): + msgs = _conversation() + new, dropped = _truncate_middle_messages(msgs, keep_ratio = 0.4) + assert dropped > 0 + surviving_call_ids = { + tc["id"] for m in new if m.get("role") == "assistant" for tc in (m.get("tool_calls") or []) + } + for m in new: + if m.get("role") == "tool": + assert m["tool_call_id"] in surviving_call_ids + + +def test_truncation_reduces_estimated_size_toward_target(): + msgs = _conversation() + total = sum(_estimate_message_tokens(m) for m in msgs) + new, dropped = _truncate_middle_messages(msgs, keep_ratio = 0.5) + new_total = sum(_estimate_message_tokens(m) for m in new) + assert dropped > 0 + assert new_total < total + # Should land at or below the requested share, modulo one whole group. + biggest_group = max( + _estimate_message_tokens(a) + _estimate_message_tokens(b) + for a, b in zip(msgs[2:-2:2], msgs[3:-2:2]) + ) + assert new_total <= int(total * 0.5) + biggest_group + + +def test_truncation_noop_when_keep_ratio_full(): + msgs = _conversation() + new, dropped = _truncate_middle_messages(msgs, keep_ratio = 1.0) + assert dropped == 0 + assert new == msgs + + +def test_truncation_noop_when_only_protected_turns_remain(): + msgs = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "task"}, + *_tool_turn(0), + {"role": "user", "content": "latest"}, + ] + new, dropped = _truncate_middle_messages(msgs, keep_ratio = 0.1) + assert dropped == 0 + assert new == msgs + + +# --------------------------------------------------------------------------- +# _apply_overflow_truncation +# --------------------------------------------------------------------------- + + +def test_apply_overflow_truncation_mutates_body_and_clamps_max_tokens(): + body = {"messages": _conversation(), "max_tokens": 32000} + assert _apply_overflow_truncation(body, _NICK_ERROR) is True + assert len(body["messages"]) < len(_conversation()) + # Generation headroom: max_tokens clamped to the non-prompt share of n_ctx. + assert body["max_tokens"] <= max(1024, int(67584 * 0.25)) + + +def test_apply_overflow_truncation_returns_false_when_nothing_droppable(): + body = { + "messages": [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "task"}, + {"role": "user", "content": "latest"}, + ], + "max_tokens": 32000, + } + assert _apply_overflow_truncation(body, _NICK_ERROR) is False + + +def test_apply_overflow_truncation_clips_giant_protected_tool_results(): + """One giant burst (few turn-groups, all protected) must still shrink: + stage 2 clips oversized tool contents instead of giving up.""" + msgs = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "task"}, + *_tool_turn(0, result_chars = 60000), + *_tool_turn(1, result_chars = 60000), + ] + body = {"messages": msgs, "max_tokens": 32000} + n_before = len(msgs) + assert _apply_overflow_truncation(body, _NICK_ERROR) is True + # No message disappeared (pairing intact), but contents were clipped. + assert len(body["messages"]) == n_before + clipped = [m for m in body["messages"] if _CLIP_MARKER in str(m.get("content"))] + assert clipped, "expected at least one clipped tool result" + surviving_call_ids = { + tc["id"] + for m in body["messages"] + if m.get("role") == "assistant" + for tc in (m.get("tool_calls") or []) + } + for m in body["messages"]: + if m.get("role") == "tool": + assert m["tool_call_id"] in surviving_call_ids + + +def test_clip_long_contents_reaches_target_and_keeps_structure(): + msgs = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "task"}, + *_tool_turn(0, result_chars = 40000), + {"role": "user", "content": "latest question"}, + ] + total = sum(_estimate_message_tokens(m) for m in msgs) + clipped = _clip_long_contents(msgs, target_est = total // 4) + assert clipped >= 1 + assert sum(_estimate_message_tokens(m) for m in msgs) <= total // 4 + # Roles and count unchanged; the short final user message untouched. + assert [m["role"] for m in msgs] == ["system", "user", "assistant", "tool", "user"] + assert msgs[-1]["content"] == "latest question" + + +def test_overflow_truncation_requested_reads_field(monkeypatch): + monkeypatch.delenv("UNSLOTH_CONTEXT_OVERFLOW", raising = False) + + class _P: + context_overflow = "truncate_middle" + + class _Q: + context_overflow = None + + assert _overflow_truncation_requested(_P()) is True + assert _overflow_truncation_requested(_Q()) is False + assert _overflow_truncation_requested(object()) is False + + +def test_overflow_truncation_server_default_env(monkeypatch): + """UNSLOTH_CONTEXT_OVERFLOW enables the policy for clients that cannot + send custom body fields; an explicit per-request 'error' still wins.""" + + class _Unset: + context_overflow = None + + class _ExplicitError: + context_overflow = "error" + + monkeypatch.setenv("UNSLOTH_CONTEXT_OVERFLOW", "truncate_middle") + assert _overflow_truncation_requested(_Unset()) is True + assert _overflow_truncation_requested(_ExplicitError()) is False + monkeypatch.setenv("UNSLOTH_CONTEXT_OVERFLOW", "error") + assert _overflow_truncation_requested(_Unset()) is False + + +# --------------------------------------------------------------------------- +# /v1/models context metadata +# --------------------------------------------------------------------------- + + +class _FakeLlamaBackend: + is_loaded = True + model_identifier = "unsloth/Qwen3.6-27B-GGUF" + context_length = 67584 + max_context_length = 262144 + + +class _FakeEmptyBackend: + active_model_name = None + + +def test_v1_models_exposes_real_context_window(monkeypatch): + monkeypatch.setattr(routes_mod, "get_llama_cpp_backend", lambda: _FakeLlamaBackend()) + monkeypatch.setattr(routes_mod, "get_inference_backend", lambda: _FakeEmptyBackend()) + models = _openai_model_objects() + assert len(models) == 1 + entry = models[0] + assert entry["id"] == "unsloth/Qwen3.6-27B-GGUF" + # The REAL (post /props readback) window, not the requested one. + assert entry["context_length"] == 67584 + assert entry["max_context_length"] == 262144 diff --git a/studio/backend/tests/test_llama_cpp_props_readback.py b/studio/backend/tests/test_llama_cpp_props_readback.py new file mode 100644 index 0000000000..d87c05f2c6 --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_props_readback.py @@ -0,0 +1,254 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for the post-launch /props context readback. + +llama-server's memory-fit step or --parallel slot split can allocate less +context than the requested -c while Studio keeps advertising the requested +value; clients sized to it then die on exceed_context_size_error 400s. +``_reconcile_effective_ctx_with_server`` must adopt the server's real +``default_generation_settings.n_ctx`` whenever it is smaller. + +Stubbed httpx; no subprocess, GPU, or network. Cross-platform. +""" + +from __future__ import annotations + +import json +import sys +import types as _types +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Stub heavy/unavailable deps before importing the module under test. +# Mirrors test_llama_cpp_context_fit.py. +# --------------------------------------------------------------------------- + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +# Prefer the real modules so importing this file first cannot poison later +# test modules with stubs; only stub what the environment genuinely lacks. +try: + import loggers # noqa: F401 +except ImportError: + _loggers_stub = _types.ModuleType("loggers") + _loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) + sys.modules.setdefault("loggers", _loggers_stub) + +try: + import structlog # noqa: F401 +except ImportError: + sys.modules.setdefault("structlog", _types.ModuleType("structlog")) + +try: + import httpx # noqa: F401 +except ImportError: + _httpx_stub = _types.ModuleType("httpx") + for _exc_name in ( + "ConnectError", + "TimeoutException", + "ReadTimeout", + "ReadError", + "RemoteProtocolError", + "CloseError", + "WriteError", + "HTTPError", + ): + setattr(_httpx_stub, _exc_name, type(_exc_name, (Exception,), {})) + + class _FakeTimeout: + def __init__(self, *a, **kw): + pass + + _httpx_stub.Timeout = _FakeTimeout + _httpx_stub.Client = type( + "Client", + (), + { + "__init__": lambda self, **kw: None, + "__enter__": lambda self: self, + "__exit__": lambda self, *a: None, + }, + ) + _httpx_stub.get = lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("unstubbed httpx.get")) + sys.modules.setdefault("httpx", _httpx_stub) + +from core.inference.llama_cpp import LlamaCppBackend +import core.inference.llama_cpp as llama_cpp_mod + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _FakeResponse: + def __init__( + self, + status_code = 200, + body = None, + ): + self.status_code = status_code + self._body = body or {} + + def json(self): + return self._body + + +def _make_backend(effective_ctx = 98304, port = 51234): + inst = LlamaCppBackend.__new__(LlamaCppBackend) + inst._port = port + inst._effective_context_length = effective_ctx + inst._context_length = 262144 + return inst + + +def _stub_props( + monkeypatch, + status_code = 200, + body = None, + exc = None, +): + def fake_get(url, timeout = None): + assert url.endswith("/props") + if exc is not None: + raise exc + return _FakeResponse(status_code, body) + + monkeypatch.setattr(llama_cpp_mod.httpx, "get", fake_get, raising = False) + + +# --------------------------------------------------------------------------- +# _query_server_n_ctx parsing +# --------------------------------------------------------------------------- + + +def test_query_n_ctx_reads_default_generation_settings(monkeypatch): + _stub_props( + monkeypatch, + body = {"default_generation_settings": {"n_ctx": 67584}}, + ) + assert _make_backend()._query_server_n_ctx() == 67584 + + +def test_query_n_ctx_non_200_returns_none(monkeypatch): + _stub_props(monkeypatch, status_code = 503) + assert _make_backend()._query_server_n_ctx() is None + + +def test_query_n_ctx_missing_key_returns_none(monkeypatch): + _stub_props(monkeypatch, body = {"default_generation_settings": {}}) + assert _make_backend()._query_server_n_ctx() is None + + +def test_query_n_ctx_swallows_transport_errors(monkeypatch): + _stub_props(monkeypatch, exc = RuntimeError("connection refused")) + assert _make_backend()._query_server_n_ctx() is None + + +# --------------------------------------------------------------------------- +# _reconcile_effective_ctx_with_server decisions +# --------------------------------------------------------------------------- + + +def test_fit_shrunk_ctx_overwrites_advertised_value(monkeypatch): + """The Nick repro: requested/advertised 98304, server really at 67584.""" + inst = _make_backend(effective_ctx = 98304) + _stub_props( + monkeypatch, + body = {"default_generation_settings": {"n_ctx": 67584}}, + ) + inst._reconcile_effective_ctx_with_server() + assert inst._effective_context_length == 67584 + assert inst.context_length == 67584 + + +def test_matching_ctx_is_left_alone(monkeypatch): + inst = _make_backend(effective_ctx = 98304) + _stub_props( + monkeypatch, + body = {"default_generation_settings": {"n_ctx": 98304}}, + ) + inst._reconcile_effective_ctx_with_server() + assert inst._effective_context_length == 98304 + + +def test_larger_server_ctx_does_not_inflate_advertised_value(monkeypatch): + """Never advertise more than the user asked for, even if the server could.""" + inst = _make_backend(effective_ctx = 32768) + _stub_props( + monkeypatch, + body = {"default_generation_settings": {"n_ctx": 65536}}, + ) + inst._reconcile_effective_ctx_with_server() + assert inst._effective_context_length == 32768 + + +def test_unset_effective_ctx_adopts_server_value(monkeypatch): + inst = _make_backend(effective_ctx = None) + inst._context_length = None + _stub_props( + monkeypatch, + body = {"default_generation_settings": {"n_ctx": 40960}}, + ) + inst._reconcile_effective_ctx_with_server() + assert inst._effective_context_length == 40960 + + +def test_props_failure_keeps_studio_value(monkeypatch): + """A flaky /props must never wipe the computed context.""" + inst = _make_backend(effective_ctx = 98304) + _stub_props(monkeypatch, exc = RuntimeError("boom")) + inst._reconcile_effective_ctx_with_server() + assert inst._effective_context_length == 98304 + + +# --------------------------------------------------------------------------- +# _ctx_integrity_flags: keep the per-request window equal to the advertised ctx +# --------------------------------------------------------------------------- + +_CAPS_ALL = {"supports_kv_unified": True, "supports_fit_ctx": True} +_CAPS_NONE = {"supports_kv_unified": False, "supports_fit_ctx": False} + + +def test_kv_unified_added_for_multi_slot(): + """Explicit --parallel N disables llama-server's auto-slots kv-unified + default, splitting -c into per-slot windows of -c/N; Studio must restore + the shared pool so one request can use the full advertised context.""" + flags = LlamaCppBackend._ctx_integrity_flags(4, False, 98304, 98304, _CAPS_ALL) + assert "--kv-unified" in flags + + +def test_kv_unified_skipped_for_single_slot_or_old_build(): + assert "--kv-unified" not in LlamaCppBackend._ctx_integrity_flags( + 1, False, 98304, 98304, _CAPS_ALL + ) + assert "--kv-unified" not in LlamaCppBackend._ctx_integrity_flags( + 4, False, 98304, 98304, _CAPS_NONE + ) + + +def test_fit_ctx_floors_explicit_request_under_fit(): + flags = LlamaCppBackend._ctx_integrity_flags(1, True, 98304, 98304, _CAPS_ALL) + assert flags[flags.index("--fit-ctx") + 1] == "98304" + + +def test_fit_ctx_skipped_without_fit_or_explicit_ctx_or_support(): + assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags( + 1, False, 98304, 98304, _CAPS_ALL + ) + assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags(1, True, 0, 262144, _CAPS_ALL) + assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags( + 1, True, 98304, 98304, _CAPS_NONE + ) + + +def test_probe_missing_binary_reports_new_capabilities_false(): + info = LlamaCppBackend.probe_server_capabilities(binary = "/nonexistent/llama-server") + assert info["found"] is False + assert info["supports_kv_unified"] is False + assert info["supports_fit_ctx"] is False diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index d8a965c1f2..1d994b46c0 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -166,10 +166,11 @@ class TestChatMessageToolRoles: with pytest.raises(ValidationError): ChatMessage(role = "user", content = []) - def test_tool_empty_content_rejected(self): - with pytest.raises(ValidationError) as exc_info: - ChatMessage(role = "tool", tool_call_id = "call_1", content = "") - assert "content" in str(exc_info.value) + def test_tool_empty_content_accepted(self): + # Empty tool output (mkdir, git add, ...) is routine in agentic loops; + # OpenAI and llama-server both accept it, so Studio must not 400. + msg = ChatMessage(role = "tool", tool_call_id = "call_1", content = "") + assert msg.content == "" def test_assistant_without_content_or_tool_calls_tolerated(self): # Stop-button leaves an empty assistant turn; tolerate for replay. diff --git a/studio/backend/tests/test_tool_message_empty_content.py b/studio/backend/tests/test_tool_message_empty_content.py new file mode 100644 index 0000000000..d63b16ce80 --- /dev/null +++ b/studio/backend/tests/test_tool_message_empty_content.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Empty ``role="tool"`` content must be accepted on the OpenAI-compat surface. + +Agentic clients send ``content: ""`` when a command produced no output; +OpenAI and llama-server both accept it. Studio used to 400, which standard +clients treat as non-retryable and kill the session. The validator must +normalize empty/missing tool content to ``""`` instead of raising. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +from models.inference import ChatMessage + + +def test_tool_message_empty_string_content_is_accepted(): + msg = ChatMessage(role = "tool", content = "", tool_call_id = "call_1") + assert msg.content == "" + + +def test_tool_message_none_content_normalizes_to_empty_string(): + msg = ChatMessage(role = "tool", content = None, tool_call_id = "call_1") + assert msg.content == "" + + +def test_tool_message_empty_list_content_normalizes_to_empty_string(): + msg = ChatMessage(role = "tool", content = [], tool_call_id = "call_1") + assert msg.content == "" + + +def test_tool_message_real_content_is_preserved(): + msg = ChatMessage(role = "tool", content = "ok", tool_call_id = "call_1") + assert msg.content == "ok" + + +def test_user_message_still_requires_content(): + with pytest.raises(ValueError): + ChatMessage(role = "user", content = None) + + +def test_assistant_empty_content_still_collapses_to_none(): + msg = ChatMessage(role = "assistant", content = "") + assert msg.content is None