diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index f34ef5fd62..8f34bb23fc 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -3873,14 +3873,16 @@ class ExternalProviderClient: ), } ) + # Per-card text; last call gets overwritten + # with citations at response.completed. + per_call_result = ( + f"Searching: {query}" if query else "" + ) yield _emit_tool_event( { "type": "tool_end", "tool_call_id": item_id, - # Empty result — the last call gets - # overwritten with citations at - # response.completed. - "result": "", + "result": per_call_result, } ) elif item.get("type") == "shell_call": @@ -3908,7 +3910,11 @@ class ExternalProviderClient: ) shell_calls.setdefault( item_id, - {"commands": [], "output": None}, + { + "commands": [], + "output": None, + "tool_end_emitted": False, + }, ) shell_calls[item_id]["commands"] = ( list(commands) @@ -3926,6 +3932,24 @@ class ExternalProviderClient: }, } ) + # Fallback: output may be bundled on the + # shell_call done event itself. + embedded_output = item.get("output") + if ( + isinstance(embedded_output, list) + and embedded_output + ): + shell_calls[item_id]["output"] = embedded_output + shell_calls[item_id]["tool_end_emitted"] = True + yield _emit_tool_event( + { + "type": "tool_end", + "tool_call_id": item_id, + "result": _format_shell_output( + embedded_output + ), + } + ) elif item.get("type") == "shell_call_output": # `call_id` links back to the shell_call's # `id`, which is what we used as the @@ -3936,8 +3960,15 @@ class ExternalProviderClient: item.get("call_id") or item.get("id") or "" ) output = item.get("output") or [] + # Skip if bundled-output path already + # finalised this card. + if shell_calls.get(call_id, {}).get( + "tool_end_emitted" + ): + continue if call_id in shell_calls: shell_calls[call_id]["output"] = output + shell_calls[call_id]["tool_end_emitted"] = True result_text = _format_shell_output(output) yield _emit_tool_event( { @@ -4093,15 +4124,10 @@ class ExternalProviderClient: } ) container_id_emitted = True - # Apply the aggregated citation list onto the - # *last* web_search call by overwriting its - # tool_end result. The frontend's - # parseSourcesFromResult flatMaps every - # web_search tool-call result, so a single - # non-empty result is enough to surface the - # whole source-pill set at the message tail — - # no need to fan out across every card (which - # would just duplicate the same pills). + # Overwrite the last web_search call with the + # citation list; the source-pill extractor + # flatMaps across cards. Earlier cards keep + # their per-call "Searching:" text. if web_search_calls and all_url_citations: last_id = list(web_search_calls.keys())[-1] blocks: list[str] = [] @@ -4119,6 +4145,21 @@ class ExternalProviderClient: "result": "\n---\n".join(blocks), } ) + # Final flush: finalise any orphan shell_call + # so the card stops spinning. + for sc_id, sc_state in shell_calls.items(): + if sc_state.get("tool_end_emitted"): + continue + yield _emit_tool_event( + { + "type": "tool_end", + "tool_call_id": sc_id, + "result": _format_shell_output( + sc_state.get("output") or [] + ), + } + ) + sc_state["tool_end_emitted"] = True chunk = { "id": completion_id, "object": "chat.completion.chunk", @@ -4197,6 +4238,22 @@ class ExternalProviderClient: "result": "\n---\n".join(blocks), } ) + # Mirror the response.completed flush so + # truncated streams also finalise orphan + # shell_calls. + for sc_id, sc_state in shell_calls.items(): + if sc_state.get("tool_end_emitted"): + continue + yield _emit_tool_event( + { + "type": "tool_end", + "tool_call_id": sc_id, + "result": _format_shell_output( + sc_state.get("output") or [] + ), + } + ) + sc_state["tool_end_emitted"] = True chunk = { "id": completion_id, "object": "chat.completion.chunk", diff --git a/studio/backend/tests/test_openai_tool_result_fallbacks.py b/studio/backend/tests/test_openai_tool_result_fallbacks.py new file mode 100644 index 0000000000..7c033bc348 --- /dev/null +++ b/studio/backend/tests/test_openai_tool_result_fallbacks.py @@ -0,0 +1,372 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regression tests for OpenAI Responses tool-result rendering. + +Covers two bug classes: empty web_search cards (per-card result seeded +with "Searching: ") and orphan shell_call cards (bundled-output +fallback + final flush at response.completed / response.incomplete). +""" + +import asyncio +import json + +import httpx + +from core.inference import external_provider as ep_mod +from core.inference.external_provider import ExternalProviderClient + + +def _drive(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +async def _collect(agen): + out = [] + async for line in agen: + out.append(line) + return out + + +def _mock_http_client(monkeypatch, handler): + transport = httpx.MockTransport(handler) + monkeypatch.setattr(ep_mod, "_http_client", httpx.AsyncClient(transport = transport)) + + +def _make_client(base_url: str = "https://api.openai.com/v1") -> ExternalProviderClient: + return ExternalProviderClient( + provider_type = "openai", + base_url = base_url, + api_key = "sk-test", + ) + + +def _openai_sse(events: list[dict]) -> bytes: + chunks: list[str] = [] + for event in events: + chunks.append(f"event: {event['type']}") + chunks.append(f"data: {json.dumps(event)}") + chunks.append("") + return ("\n".join(chunks) + "\n").encode("utf-8") + + +def _tool_events(lines: list[str]) -> list[dict]: + out: list[dict] = [] + for line in lines: + if not line.startswith("data:"): + continue + raw = line[len("data:") :].strip() + if not raw or raw == "[DONE]": + continue + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict) and "_toolEvent" in parsed: + out.append(parsed["_toolEvent"]) + return out + + +def _drive_stream(sse_events, enabled_tools, monkeypatch): + def handler(request): + return httpx.Response( + 200, + content = _openai_sse(sse_events), + headers = {"content-type": "text/event-stream"}, + ) + + _mock_http_client(monkeypatch, handler) + + async def run(): + client = _make_client() + return await _collect( + client._stream_openai_responses( + messages = [{"role": "user", "content": "x"}], + model = "gpt-5.5", + temperature = 0.7, + top_p = 0.95, + max_tokens = 4096, + enable_thinking = None, + reasoning_effort = None, + enabled_tools = enabled_tools, + ) + ) + + return _drive(run()) + + +# ── web_search per-card result ───────────────────────────────────────── + + +def test_web_search_each_call_carries_its_own_query_as_result(monkeypatch): + """Each card carries its own `Searching: ` text; no empties.""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_1", + "action": {"query": "popular animals 2026"}, + }, + }, + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_2", + "action": {"query": "most loved animals poll"}, + }, + }, + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_3", + "action": {"query": "tiger ranking"}, + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["web_search"], monkeypatch) + events = _tool_events(lines) + ends = [e for e in events if e["type"] == "tool_end"] + by_id = {e["tool_call_id"]: e for e in ends} + assert by_id["ws_1"]["result"] == "Searching: popular animals 2026" + assert by_id["ws_2"]["result"] == "Searching: most loved animals poll" + assert by_id["ws_3"]["result"] == "Searching: tiger ranking" + + +def test_web_search_last_call_overwritten_with_citations(monkeypatch): + """Last call still gets the aggregated citation list; earlier calls + keep their per-call `Searching:` text.""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_1", + "action": {"query": "first query"}, + }, + }, + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_2", + "action": {"query": "second query"}, + }, + }, + { + "type": "response.output_text.annotation.added", + "annotation": { + "type": "url_citation", + "url": "https://example.com/a", + "title": "Example A", + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["web_search"], monkeypatch) + events = _tool_events(lines) + ends = [e for e in events if e["type"] == "tool_end"] + by_id: dict = {} + # Keep the LAST tool_end per id (the citation overwrite for ws_2). + for e in ends: + by_id[e["tool_call_id"]] = e + # First call keeps its own query. + assert by_id["ws_1"]["result"] == "Searching: first query" + # Last call gets overwritten with the citation block. + assert "Title: Example A" in by_id["ws_2"]["result"] + assert "URL: https://example.com/a" in by_id["ws_2"]["result"] + + +def test_web_search_empty_query_falls_back_to_empty_result(monkeypatch): + """No query -> empty result (no `Searching:` placeholder).""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_only", + "action": {}, + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["web_search"], monkeypatch) + events = _tool_events(lines) + ends = [e for e in events if e["type"] == "tool_end"] + assert len(ends) == 1 + assert ends[0]["result"] == "" + + +# ── shell_call output fallbacks ──────────────────────────────────────── + + +def test_shell_call_emits_tool_end_when_output_bundled_on_done(monkeypatch): + """Output bundled on the shell_call done event emits tool_end.""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "shell_call", + "id": "scall_bundled", + "action": {"commands": ["echo hi"]}, + "output": [ + { + "stdout": "hi\n", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + } + ], + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["code_execution"], monkeypatch) + events = _tool_events(lines) + starts = [e for e in events if e["type"] == "tool_start"] + ends = [e for e in events if e["type"] == "tool_end"] + assert len(starts) == 1 + assert starts[0]["tool_call_id"] == "scall_bundled" + assert len(ends) == 1 + assert ends[0]["tool_call_id"] == "scall_bundled" + assert "hi" in ends[0]["result"] + + +def test_shell_call_bundled_then_separate_output_does_not_double_emit(monkeypatch): + """Separate shell_call_output after bundled-output is a no-op.""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "shell_call", + "id": "scall_both", + "action": {"commands": ["echo bundle"]}, + "output": [ + { + "stdout": "bundle\n", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + } + ], + }, + }, + { + "type": "response.output_item.done", + "item": { + "type": "shell_call_output", + "id": "scout_both", + "call_id": "scall_both", + "output": [ + { + "stdout": "should not double-emit\n", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + } + ], + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["code_execution"], monkeypatch) + events = _tool_events(lines) + ends = [e for e in events if e["type"] == "tool_end"] + assert len(ends) == 1 + assert ends[0]["tool_call_id"] == "scall_both" + assert "bundle" in ends[0]["result"] + assert "should not double-emit" not in ends[0]["result"] + + +def test_shell_call_final_flush_on_completed_when_no_output_event(monkeypatch): + """Orphan shell_call finalises via the response.completed flush.""" + sse_events = [ + { + "type": "response.output_item.added", + "item": { + "type": "shell_call", + "id": "scall_orphan", + "action": {"commands": ["true"]}, + }, + }, + { + "type": "response.output_item.done", + "item": { + "type": "shell_call", + "id": "scall_orphan", + "action": {"commands": ["true"]}, + "status": "completed", + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["code_execution"], monkeypatch) + events = _tool_events(lines) + ends = [e for e in events if e["type"] == "tool_end"] + assert any(e["tool_call_id"] == "scall_orphan" for e in ends) + + +def test_shell_call_flushed_on_response_incomplete_truncation(monkeypatch): + """Truncated streams (response.incomplete) also flush orphan calls.""" + sse_events = [ + { + "type": "response.output_item.added", + "item": { + "type": "shell_call", + "id": "scall_truncated", + "action": {"commands": ["long_running"]}, + }, + }, + { + "type": "response.output_item.done", + "item": { + "type": "shell_call", + "id": "scall_truncated", + "action": {"commands": ["long_running"]}, + "status": "in_progress", + }, + }, + { + "type": "response.incomplete", + "response": { + "incomplete_details": {"reason": "max_output_tokens"}, + }, + }, + ] + lines = _drive_stream(sse_events, ["code_execution"], monkeypatch) + events = _tool_events(lines) + ends = [e for e in events if e["type"] == "tool_end"] + assert any(e["tool_call_id"] == "scall_truncated" for e in ends) + + +def test_shell_call_incomplete_does_not_double_emit(monkeypatch): + """response.incomplete is idempotent against already-finalised calls.""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "shell_call", + "id": "scall_done", + "action": {"commands": ["echo done"]}, + "output": [ + { + "stdout": "done\n", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + } + ], + }, + }, + { + "type": "response.incomplete", + "response": { + "incomplete_details": {"reason": "max_output_tokens"}, + }, + }, + ] + lines = _drive_stream(sse_events, ["code_execution"], monkeypatch) + events = _tool_events(lines) + ends = [e for e in events if e["type"] == "tool_end"] + assert len(ends) == 1 + assert ends[0]["tool_call_id"] == "scall_done" + assert "done" in ends[0]["result"] diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 49622c8090..ce59429762 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -21,6 +21,7 @@ import { pickFriendlyContainerName } from "../lib/friendly-names"; import { EXTERNAL_MAX_OUTPUT_TOKENS, clampReasoningEffortToLevels, + getExternalMaxOutputTokens, getExternalMinOutputTokens, getExternalReasoningCapabilities, getProviderCapabilities, @@ -1703,18 +1704,17 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(externalCapabilities?.topP !== false ? { top_p: params.topP } : {}), - // Clamp to the cross-provider output cap so a maxTokens value - // carried over from a local-model session does not blow past - // provider limits (e.g. Claude Opus 400s on >128k). Also - // floor to the provider's documented minimum — Kimi's - // thinking models need >=16k or the response truncates - // before the answer fits alongside reasoning_content. + // Floor at the provider's documented min (Kimi thinking + // needs >=16k); clamp at the per-model max. max_tokens: Math.min( Math.max( params.maxTokens, getExternalMinOutputTokens(externalProvider?.providerType), ), - EXTERNAL_MAX_OUTPUT_TOKENS, + getExternalMaxOutputTokens( + externalProvider?.providerType, + externalSelection?.modelId, + ), ), // Only forward sampling knobs the provider actually accepts; the // backend's external-provider proxy is param-permissive and would diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index ac1ef8a24c..3714fb128a 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -85,6 +85,7 @@ import { import { EXTERNAL_MAX_OUTPUT_TOKENS, type ProviderCapabilities, + getExternalMaxOutputTokens, getExternalMinOutputTokens, providerSupportsBuiltinCodeExecution, providerSupportsFastMode, @@ -1309,7 +1310,10 @@ export function ChatSettingsPanel({ } max={ isExternalModel - ? EXTERNAL_MAX_OUTPUT_TOKENS + ? getExternalMaxOutputTokens( + externalProviderType, + externalSelection?.modelId, + ) : isGguf && ggufContextLength ? ggufContextLength : 32768 diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index ef805305be..5adc01ea2b 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -71,18 +71,95 @@ export function clampReasoningEffortToLevels( } /** - * Output-token cap for any external provider request. Picked to stay below the - * tightest declared limit across the providers we ship (Anthropic Claude Opus - * tops out at 128k, GPT-5.x ~128k, Gemini 2.5 ~65k, DeepSeek 8k) while staying - * well above what a typical chat reply needs. The local-model path is not - * subject to this — local backends honour whatever the loaded context allows. - * - * If a user's stored maxTokens (e.g. carried over from a prior local-model - * session with a 128k+ context) exceeds this, chat-adapter clamps the - * outbound request so the provider does not 400 on it. + * Fallback cap for unknown providers / models. Prefer + * `getExternalMaxOutputTokens(providerType, modelId)` for the real cap. */ export const EXTERNAL_MAX_OUTPUT_TOKENS = 32768; +/** + * Per-model max-output caps from each provider's docs: + * OpenAI: developers.openai.com/api/docs/models/gpt-5.5 + * Anthropic: platform.claude.com/docs/en/about-claude/models + * Gemini: ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview + * DeepSeek: api-docs.deepseek.com/quick_start/pricing (V4 family) + * Local-model path is unaffected. + */ +const EXTERNAL_MAX_OUTPUT_TOKENS_BY_MODEL: Array<{ + providerType: string; + prefixes: readonly string[]; + cap: number; +}> = [ + // OpenAI + { providerType: "openai", prefixes: ["gpt-5.5-pro", "gpt-5.5"], cap: 128000 }, + { providerType: "openai", prefixes: ["gpt-5.4-pro", "gpt-5.4"], cap: 65536 }, + { providerType: "openai", prefixes: ["gpt-5.3"], cap: 16384 }, + // Anthropic + { + providerType: "anthropic", + prefixes: ["claude-opus-4-7"], + cap: 128000, + }, + { + providerType: "anthropic", + prefixes: [ + "claude-opus-4-6", + "claude-sonnet-4-6", + "claude-opus-4-5", + "claude-sonnet-4-5", + "claude-haiku-4-5", + ], + cap: 64000, + }, + // Gemini + { + providerType: "gemini", + prefixes: ["gemini-3", "gemini-pro", "gemini-flash"], + cap: 65536, + }, + // DeepSeek (V4: deepseek-chat / deepseek-reasoner alias V4-flash). + { providerType: "deepseek", prefixes: ["deepseek"], cap: 384000 }, +]; + +/** + * Documented per-model output cap; unknown ids fall back to + * `EXTERNAL_MAX_OUTPUT_TOKENS` (32k). OpenRouter ids are + * `provider/model`; the prefix is stripped before matching. + */ +export function getExternalMaxOutputTokens( + providerType: string | null | undefined, + modelId: string | null | undefined, +): number { + if (!providerType || !modelId) return EXTERNAL_MAX_OUTPUT_TOKENS; + const normalized = modelId.trim().toLowerCase(); + if (!normalized) return EXTERNAL_MAX_OUTPUT_TOKENS; + const stripped = + providerType === "openrouter" && normalized.includes("/") + ? normalized.split("/").slice(-1)[0] + : normalized; + const effectiveProvider = + providerType === "openrouter" + ? _inferProviderFromOpenrouterId(normalized) ?? providerType + : providerType; + for (const entry of EXTERNAL_MAX_OUTPUT_TOKENS_BY_MODEL) { + if (entry.providerType !== effectiveProvider) continue; + if (entry.prefixes.some((prefix) => stripped.startsWith(prefix))) { + return entry.cap; + } + } + return EXTERNAL_MAX_OUTPUT_TOKENS; +} + +function _inferProviderFromOpenrouterId( + normalizedId: string, +): string | null { + // Map OpenRouter `provider/model` prefix to our internal providerType. + if (normalizedId.startsWith("openai/")) return "openai"; + if (normalizedId.startsWith("anthropic/")) return "anthropic"; + if (normalizedId.startsWith("google/")) return "gemini"; + if (normalizedId.startsWith("deepseek/")) return "deepseek"; + return null; +} + /** * Whether the external provider offers a built-in web-search tool that the * model invokes server-side. When `true`, the chat composer's Search button diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index c78f02a474..a71e4127a2 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -14,7 +14,9 @@ import { DEFAULT_INFERENCE_PARAMS, type InferenceParams, } from "../types/runtime"; -import { isExternalModelId } from "../external-providers"; +import { isExternalModelId, parseExternalModelId } from "../external-providers"; +import { getExternalMaxOutputTokens } from "../provider-capabilities"; +import { useExternalProvidersStore } from "./external-providers-store"; import { loadChatSettingsWithLegacyImport, savePersistedChatSettingsPatch, @@ -747,10 +749,30 @@ export const useChatRuntimeStore = create((set, get) => ({ // external-provider render gate would otherwise show old counters // until the next completion overwrites them. const checkpointChanged = state.params.checkpoint !== modelId; + // Clamp maxTokens to the new model's cap on switch into an + // external model so a value carried over from a prior local + // session does not render above the slider's max. + let nextMaxTokens = state.params.maxTokens; + if (checkpointChanged && isExternalModelId(modelId)) { + const parsed = parseExternalModelId(modelId); + const provider = parsed + ? useExternalProvidersStore + .getState() + .providers.find((p) => p.id === parsed.providerId) + : null; + const cap = getExternalMaxOutputTokens( + provider?.providerType, + parsed?.modelId, + ); + if (nextMaxTokens > cap) { + nextMaxTokens = cap; + } + } return { params: { ...state.params, checkpoint: modelId, + maxTokens: nextMaxTokens, }, activeGgufVariant: ggufVariant ?? null, ...(checkpointChanged ? { contextUsage: null } : {}),