diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index de928e83ee..3f4e054b00 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -977,40 +977,59 @@ jobs: skipped -> no `modeling_.py` file (expected for some umbrella packages like `auto`, `deprecated`) known -> in KNOWN_BROKEN_COMPILE; tracked for follow-up. - Any uncaught failure fails the cell.""" + Any uncaught failure fails the cell. + + Per-model SIGALRM cap so one infinite-looping model_type + cannot wedge the whole sweep + nuke the job timeout + (observed on transformers >=5,<6 -- 30+ min hang before + this guard landed).""" import importlib as _il + import signal ok = 0 skipped = [] known = [] new_failures = [] - for model_type in _all_model_types(): - modeling_path = f"transformers.models.{model_type}.modeling_{model_type}" - try: - _il.import_module(modeling_path) - except (ModuleNotFoundError, ImportError): - skipped.append((model_type, "no modeling file")) - continue - try: - unsloth_compile_transformers( - model_type=model_type, fast_lora_forwards=False, - ) - except Exception as e: - msg = f"{type(e).__name__}: {str(e)[:200]}" + models = _all_model_types() + def _on_timeout(signum, frame): + raise TimeoutError("compile exceeded per-model budget") + prev_handler = signal.signal(signal.SIGALRM, _on_timeout) + try: + for i, model_type in enumerate(models): + if i % 25 == 0: + print(f" sweep progress: {i}/{len(models)} -> {model_type}", flush=True) + modeling_path = f"transformers.models.{model_type}.modeling_{model_type}" + try: + _il.import_module(modeling_path) + except (ModuleNotFoundError, ImportError): + skipped.append((model_type, "no modeling file")) + continue + signal.alarm(60) + try: + unsloth_compile_transformers( + model_type=model_type, fast_lora_forwards=False, + ) + except Exception as e: + signal.alarm(0) + msg = f"{type(e).__name__}: {str(e)[:200]}" + if model_type in KNOWN_BROKEN_COMPILE: + known.append((model_type, msg)) + else: + new_failures.append((model_type, msg)) + continue + signal.alarm(0) if model_type in KNOWN_BROKEN_COMPILE: - known.append((model_type, msg)) - else: - new_failures.append((model_type, msg)) - continue - if model_type in KNOWN_BROKEN_COMPILE: - # Came back green unexpectedly -- that's GOOD news, - # the bug was fixed. Surface it so we can drop the - # entry from KNOWN_BROKEN_COMPILE. - print( - f" UNEXPECTED-OK {model_type}: was in " - "KNOWN_BROKEN_COMPILE, now compiles cleanly. " - "Drop the entry." - ) - ok += 1 + # Came back green unexpectedly -- that's GOOD news, + # the bug was fixed. Surface it so we can drop the + # entry from KNOWN_BROKEN_COMPILE. + print( + f" UNEXPECTED-OK {model_type}: was in " + "KNOWN_BROKEN_COMPILE, now compiles cleanly. " + "Drop the entry." + ) + ok += 1 + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, prev_handler) print(f"\nCompile sweep: ok={ok} skipped={len(skipped)} " f"known-broken={len(known)} new-failures={len(new_failures)}") for m, r in known: diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index bd14e73d16..48a5f0e879 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -24,11 +24,17 @@ import structlog # sites use printf-style positional args, which structlog accepts. logger = structlog.get_logger(__name__) -# Claude 4.7 (Opus/Sonnet/Haiku) deprecated top_k and returns 400 -# "top_k is deprecated for this model" when it is set. 3.x and 4.5/4.6 -# still accept it. Match the 4-7 line specifically so we keep the knob -# live on every other Claude generation. -_ANTHROPIC_TOP_K_DEPRECATED = re.compile(r"^claude-(?:opus|sonnet|haiku)-4-7(?:[-.]|$)") +# Claude 4.7 (Opus/Sonnet/Haiku) removed temperature, top_p, and top_k — +# the API returns 400 " is deprecated for this model" if any of +# them is set to a non-default value. The "Sampling parameters removed" +# section of the 4.7 release notes is the authoritative reference: +# https://platform.claude.com/docs/en/about-claude/models/whats-new-claude-4-7 +# 3.x and 4.5/4.6 still accept all three; match the 4-7 line strictly so +# the knobs keep working on earlier families. The trailing -4-7[-.]/EOL +# anchor keeps future versions (e.g. claude-opus-5) unaffected. +_ANTHROPIC_4_7_SAMPLING_REMOVED = re.compile( + r"^claude-(?:opus|sonnet|haiku)-4-7(?:[-.]|$)" +) _OPENAI_REASONING_SUMMARY_UNSUPPORTED = re.compile(r"^o3(?:[-.]|$)") @@ -229,6 +235,7 @@ class ExternalProviderClient: enable_thinking: Optional[bool] = None, reasoning_effort: Optional[str] = None, enabled_tools: Optional[list[str]] = None, + enable_prompt_caching: Optional[bool] = None, stream: bool = True, ) -> AsyncGenerator[str, None]: """ @@ -253,6 +260,7 @@ class ExternalProviderClient: enable_thinking, reasoning_effort, enabled_tools, + enable_prompt_caching, ): yield line return @@ -272,6 +280,7 @@ class ExternalProviderClient: enable_thinking, reasoning_effort, enabled_tools, + enable_prompt_caching, ): yield line return @@ -346,6 +355,13 @@ class ExternalProviderClient: _apply_mistral_reasoning_controls( body, model, enable_thinking, reasoning_effort ) + elif self.provider_type == "vllm" and enable_thinking is not None: + # vLLM gates thinking via chat_template_kwargs.enable_thinking. + tpl_kw = body.get("chat_template_kwargs") + if not isinstance(tpl_kw, dict): + tpl_kw = {} + tpl_kw["enable_thinking"] = bool(enable_thinking) + body["chat_template_kwargs"] = tpl_kw # OpenRouter exposes a unified `reasoning` parameter on every # chat-completion request — the gateway routes it to whichever @@ -1043,6 +1059,7 @@ class ExternalProviderClient: enable_thinking: Optional[bool] = None, reasoning_effort: Optional[str] = None, enabled_tools: Optional[list[str]] = None, + enable_prompt_caching: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ Call the Anthropic Messages API and translate its SSE to OpenAI format. @@ -1112,24 +1129,79 @@ class ExternalProviderClient: else: filtered.append(msg) + # Claude 4.7 family removed temperature / top_p / top_k entirely. + # The earlier guard only handled top_k; temperature is now also + # rejected with 400 "temperature is deprecated for this model". + # Latch the match once and reuse it everywhere temperature or + # top_k would otherwise be set — including the thinking-mode + # override below, which used to force temperature=1. + sampling_removed = bool(_ANTHROPIC_4_7_SAMPLING_REMOVED.match(model)) + body: dict[str, Any] = { "model": model, "messages": filtered, "max_tokens": max_tokens or 1024, # required by Anthropic - "temperature": temperature, "stream": True, } - # top_k is deprecated on Claude 4.7 (Opus/Sonnet/Haiku) — the API - # returns 400 "top_k is deprecated for this model" when it is set. - # 3.x and 4.5/4.6 still accept it, so gate strictly on the 4.7 ids. - if ( - top_k is not None - and top_k > 0 - and not _ANTHROPIC_TOP_K_DEPRECATED.match(model) - ): + if not sampling_removed: + body["temperature"] = temperature + if top_k is not None and top_k > 0 and not sampling_removed: body["top_k"] = top_k + # Anthropic only caches a prefix when at least one cache_control + # marker is attached to it — the frontend defaults + # enable_prompt_caching to True for Anthropic, so treat `None` the + # same as True here (callers that don't set the flag still get + # caching). Pass False explicitly to opt out. + prompt_caching_enabled = enable_prompt_caching is not False + if system: - body["system"] = system + if prompt_caching_enabled: + # System block is the most stable prefix across turns, so + # it gets its own breakpoint. Skipped when system is + # empty — there's nothing to cache, and an empty marker + # is a no-op. + body["system"] = [ + { + "type": "text", + "text": system, + "cache_control": {"type": "ephemeral"}, + } + ] + else: + body["system"] = system + + if prompt_caching_enabled and filtered: + # Second breakpoint at the end of the conversation. Anthropic + # caches the longest matching prefix up to a cache_control + # marker; placing one on the latest message means turn N+1 + # rehydrates everything up through turn N from cache instead + # of recomputing it. This is what makes caching actually work + # when the system prompt is empty or shorter than Anthropic's + # ~1024-token cache floor — the conversation history carries + # the bulk of the input tokens. Anthropic allows up to 4 + # breakpoints per request; we use at most 2 (system + tail). + last_msg = filtered[-1] + content = last_msg.get("content") + if isinstance(content, str): + last_msg["content"] = [ + { + "type": "text", + "text": content, + "cache_control": {"type": "ephemeral"}, + } + ] + elif isinstance(content, list) and content: + # Don't mutate the caller's list. Rebuild the tail with + # cache_control attached to the final block so an + # upstream image-bearing turn still cleanly slots into + # the cache as part of the conversational prefix. + head = list(content[:-1]) + tail = content[-1] + if isinstance(tail, dict): + head.append({**tail, "cache_control": {"type": "ephemeral"}}) + else: + head.append(tail) + last_msg["content"] = head thinking_spec = _anthropic_thinking_spec(model) allowed_efforts = ( thinking_spec.efforts @@ -1155,13 +1227,15 @@ class ExternalProviderClient: if effort and effort != "none": # Anthropic rejects top_k whenever thinking is enabled. body.pop("top_k", None) - # Anthropic requires temperature=1 whenever thinking is enabled, - # AND forbids top_p in the same request: setting both produces + # Earlier families (4.5/4.6) require temperature=1 when + # thinking is enabled and forbid top_p in the same request: # "temperature and top_p cannot both be specified for this # model. Please use only one." - # The base body never sets top_p, but pop defensively in case - # an upstream edit ever adds it before this branch runs. - body["temperature"] = 1 + # On Claude 4.7, temperature was removed entirely — sending + # any value (including 1) returns 400 — so skip the override + # there and let the model use its default sampling. + if not sampling_removed: + body["temperature"] = 1 body.pop("top_p", None) if thinking_spec and thinking_spec.kind == "adaptive": # `display` defaults to "omitted" on Claude Opus 4.7 (per the @@ -1278,6 +1352,13 @@ class ExternalProviderClient: current_server_tool_use: Optional[dict[str, Any]] = None current_result_block: Optional[dict[str, Any]] = None web_search_calls: dict[str, dict[str, Any]] = {} + # Cache usage tracking. message_start carries the input + # accounting (incl. cache_creation_input_tokens and + # cache_read_input_tokens); message_delta carries cumulative + # output_tokens. Both are surfaced in the "stream complete" + # log so prompt caching can be verified per-request without + # opening the Anthropic dashboard. + last_usage: dict[str, Any] = {} def _content_chunk(text: str) -> str: chunk = { @@ -1352,6 +1433,16 @@ class ExternalProviderClient: key = event_type or "" event_counts[key] = event_counts.get(key, 0) + 1 + # message_start carries the input-side usage block + # including cache_creation_input_tokens and + # cache_read_input_tokens. message_delta updates + # output_tokens (and may overwrite the input fields + # with final values). Merge both into last_usage. + if event_type == "message_start": + start_usage = (event.get("message") or {}).get("usage") + if isinstance(start_usage, dict): + last_usage.update(start_usage) + if event_type == "content_block_start": content_block = event.get("content_block") or {} block_type = content_block.get("type") @@ -1489,6 +1580,9 @@ class ExternalProviderClient: thinking_open = False elif event_type == "message_delta": + delta_usage = event.get("usage") + if isinstance(delta_usage, dict): + last_usage.update(delta_usage) stop_reason = event.get("delta", {}).get("stop_reason") if stop_reason: if thinking_open: @@ -1538,15 +1632,28 @@ class ExternalProviderClient: for sc in web_search_calls.values() if sc.get("query") ] + # cache_read_input_tokens > 0 on turn N proves the + # cache_control marker on the system block is doing + # its job — turn 1 will show cache_creation > 0 + # instead. cache_creation tokens are billed at a + # small premium; cache_read tokens are billed at a + # discount. logger.info( "Anthropic stream complete (model=%s, " "web_search_requested=%s, web_search_invocations=%s, " - "results=%s, queries=%s, events=%s)", + "results=%s, queries=%s, " + "input_tokens=%s, output_tokens=%s, " + "cache_creation_input_tokens=%s, " + "cache_read_input_tokens=%s, events=%s)", model, web_search_requested, web_search_invocations, total_results, queries, + last_usage.get("input_tokens"), + last_usage.get("output_tokens"), + last_usage.get("cache_creation_input_tokens"), + last_usage.get("cache_read_input_tokens"), event_counts, ) await response.aclose() @@ -1584,6 +1691,7 @@ class ExternalProviderClient: enable_thinking: Optional[bool], reasoning_effort: Optional[str], enabled_tools: Optional[list[str]] = None, + enable_prompt_caching: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ Call OpenAI's /v1/responses endpoint and translate its SSE stream back @@ -1685,6 +1793,27 @@ class ExternalProviderClient: if max_tokens is not None: body["max_output_tokens"] = max_tokens + # Prompt caching on /v1/responses is automatic and free, but the + # default in-memory policy only survives ~5-10 min of inactivity + # (up to ~1 hr). Opt into the 24-hour retention policy so a chat + # left idle overnight still hits the cache on the next turn. + # Pricing is identical to in_memory per OpenAI's docs. + # + # Gated on the base URL because ollama / llama.cpp / "custom" + # presets all collapse to provider_type="openai" in + # toExternalBackendProviderType, so they also land in this + # helper. Those servers expose /v1/responses-shaped routes in + # some configurations but don't implement + # prompt_cache_retention — sending the field unconditionally + # would 400 them. Match the public OpenAI host strictly so the + # field only goes to OpenAI cloud. Studio's openai model picker + # is registry-scoped to gpt-5.x / o3 / gpt-4.5, all of which + # accept this parameter (gpt-5.5+ already defaults to "24h" and + # rejects "in_memory", so it's a safe no-op there). + is_openai_cloud = "api.openai.com" in (self.base_url or "") + if is_openai_cloud and enable_prompt_caching is not False: + body["prompt_cache_retention"] = "24h" + # OpenAI server-side tools — see # https://developers.openai.com/api/docs/guides/tools # The frontend's Search button maps to the unified @@ -1731,6 +1860,12 @@ class ExternalProviderClient: done_emitted = False reasoning_open = False reasoning_emitted = False + # Latched from response.completed / response.incomplete so + # the final log can surface input_tokens_details.cached_tokens — + # the field that proves prompt_cache_retention="24h" is + # actually hitting OpenAI's cache instead of recomputing + # the prefix every turn. + last_usage: Optional[dict[str, Any]] = None # Per-call state for OpenAI's server-side web_search tool. Mapped # back into our local _toolEvent shape so the existing chat-UI # renderer surfaces web_search the same way it does for local @@ -1955,6 +2090,9 @@ class ExternalProviderClient: reasoning_emitted = True elif event_type == "response.completed": + completed_usage = (event.get("response") or {}).get("usage") + if isinstance(completed_usage, dict): + last_usage = completed_usage if reasoning_open: yield _chunk_with_text("") reasoning_open = False @@ -1998,6 +2136,11 @@ class ExternalProviderClient: yield f"data: {_json.dumps(chunk)}" elif event_type == "response.incomplete": + incomplete_usage = (event.get("response") or {}).get( + "usage" + ) + if isinstance(incomplete_usage, dict): + last_usage = incomplete_usage if reasoning_open: yield _chunk_with_text("") reasoning_open = False @@ -2071,16 +2214,33 @@ class ExternalProviderClient: for sc in web_search_calls.values() if sc.get("query") ] + # cached_input_tokens > 0 on turn N proves + # prompt_cache_retention="24h" is letting the previous + # turn's prefix hit the cache instead of being + # recomputed. On /v1/responses the field is nested as + # usage.input_tokens_details.cached_tokens (not + # prompt_tokens_details, which is the /v1/chat/completions + # shape). + cached_input_tokens = None + if isinstance(last_usage, dict): + details = last_usage.get("input_tokens_details") + if isinstance(details, dict): + cached_input_tokens = details.get("cached_tokens") logger.info( "OpenAI Responses stream complete (model=%s, " "web_search_requested=%s, web_search_invocations=%s, " - "citations=%s, queries=%s, reasoning_emitted=%s)", + "citations=%s, queries=%s, reasoning_emitted=%s, " + "input_tokens=%s, output_tokens=%s, " + "cached_input_tokens=%s)", model, web_search_requested, web_search_invocations, total_citations, queries, reasoning_emitted, + (last_usage or {}).get("input_tokens"), + (last_usage or {}).get("output_tokens"), + cached_input_tokens, ) await response.aclose() await lines_gen.aclose() diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 4b6d7d6b17..143ced95f1 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -218,6 +218,28 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # are always among the top regardless of the API's order. "model_id_limit": 15, }, + "vllm": { + "display_name": "vLLM", + # User-supplied via provider_base_url; the route layer already falls + # back to the payload's base_url when the registry entry has none. + "base_url": "", + "default_models": [], + "supports_streaming": True, + "supports_vision": True, + "supports_tool_calling": True, + "auth_header": "Authorization", + "auth_prefix": "Bearer ", + # Force /v1/chat/completions in stream_chat_completion — vLLM's + # /v1/responses rebuilds messages and runs them through the loaded + # model's chat template, which 400s on strict-alternation templates + # (Gemma 3 raises "Conversation roles must alternate user/assistant + # /user/assistant/..."). The chat-completions path takes messages + # verbatim and avoids that template gauntlet. + "notes": "Self-hosted vLLM server. Always routed to /v1/chat/completions.", + # Surfaced through the frontend's CUSTOM_PROVIDER_PRESETS, not the + # /api/providers/registry dropdown — see list_available_providers. + "hidden": True, + }, "openrouter": { "display_name": "OpenRouter", "base_url": "https://openrouter.ai/api/v1", @@ -269,9 +291,17 @@ def get_base_url(provider_type: str) -> str | None: def list_available_providers() -> list[dict[str, Any]]: - """Return all registered providers (for the /registry endpoint).""" + """Return all registered providers (for the /registry endpoint). + + Hidden entries (``"hidden": True``) are filtered out — they exist in the + registry only for backend lookups (e.g. ``supports_vision`` for vLLM) and + are surfaced in the frontend via ``CUSTOM_PROVIDER_PRESETS`` instead of + the cloud-provider dropdown. + """ result = [] for provider_type, info in PROVIDER_REGISTRY.items(): + if info.get("hidden"): + continue result.append( { "provider_type": provider_type, diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 013328f6c6..f2eed314ee 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -593,6 +593,17 @@ class ChatCompletionRequest(BaseModel): None, description = "[x-unsloth] Override base URL for the external provider.", ) + enable_prompt_caching: Optional[bool] = Field( + None, + description = ( + "[x-unsloth] Opt in to provider-side prompt caching. On Anthropic, " + "attaches cache_control={type:ephemeral} to the system block so the " + "static prefix is reused across turns. On OpenAI cloud, caching is " + "automatic for prompts >=1024 tokens and this flag is informational. " + "Ignored for every other provider (mistral, gemini, kimi, openrouter, " + "vllm, local, etc.). Treated as enabled when omitted." + ), + ) # ── Streaming response chunks ──────────────────────────────────── diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 55d74d1cbe..b223bcf981 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1596,6 +1596,7 @@ async def _proxy_to_external_provider( enable_thinking = payload.enable_thinking, reasoning_effort = payload.reasoning_effort, enabled_tools = payload.enabled_tools, + enable_prompt_caching = payload.enable_prompt_caching, stream = payload.stream, ) try: diff --git a/studio/frontend/public/provider-logos/llama_cpp.svg b/studio/frontend/public/provider-logos/llama_cpp.svg new file mode 100644 index 0000000000..218cc1de88 --- /dev/null +++ b/studio/frontend/public/provider-logos/llama_cpp.svg @@ -0,0 +1 @@ + diff --git a/studio/frontend/public/provider-logos/ollama.svg b/studio/frontend/public/provider-logos/ollama.svg new file mode 100644 index 0000000000..d3b6a42dd7 --- /dev/null +++ b/studio/frontend/public/provider-logos/ollama.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/studio/frontend/public/provider-logos/vllm.svg b/studio/frontend/public/provider-logos/vllm.svg new file mode 100644 index 0000000000..0c8a13de01 --- /dev/null +++ b/studio/frontend/public/provider-logos/vllm.svg @@ -0,0 +1 @@ + diff --git a/studio/frontend/src/components/assistant-ui/model-selector.tsx b/studio/frontend/src/components/assistant-ui/model-selector.tsx index b4f7dd08d2..dc8bffb2b7 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector.tsx @@ -10,10 +10,12 @@ import { } from "@/components/ui/popover"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { usePlatformStore } from "@/config/env"; +import { isCustomProviderType } from "@/features/chat/external-providers"; import { cn } from "@/lib/utils"; import { ArrowDown01Icon, CloudIcon, + DashboardSquare01Icon, FolderSearchIcon, Logout01Icon, Search01Icon, @@ -40,6 +42,9 @@ const PROVIDER_LOGO_EXT: Record = { kimi: "jpg", qwen: "png", openrouter: "svg", + vllm: "svg", + ollama: "svg", + llama_cpp: "svg", }; function providerLogoSrc(providerType: string | undefined): string | undefined { @@ -59,6 +64,17 @@ function ExternalProviderLogo({ title?: string; }) { const src = providerLogoSrc(providerType); + if (!src && isCustomProviderType(providerType)) { + return ( + + + + ); + } + if (!src) return null; return ( { ? getExternalReasoningCapabilities( selectedExternalProvider?.providerType, effectiveExternalModelId, + { + isReasoningProvider: + selectedExternalProvider?.isReasoningModel === true, + }, ) : null; const effectiveReasoningStyle = diff --git a/studio/frontend/src/features/chat/api-provider-logo.tsx b/studio/frontend/src/features/chat/api-provider-logo.tsx index bd4f05b2ff..09794e3acf 100644 --- a/studio/frontend/src/features/chat/api-provider-logo.tsx +++ b/studio/frontend/src/features/chat/api-provider-logo.tsx @@ -4,6 +4,7 @@ import { cn } from "@/lib/utils"; import { DashboardSquare01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; +import { isCustomProviderType } from "./external-providers"; /** * Registry logos live at `public/provider-logos/{provider_type}.{ext}` where `provider_type` @@ -19,6 +20,9 @@ const PROVIDER_LOGO_EXT: Record = { kimi: "jpg", qwen: "png", openrouter: "svg", + vllm: "svg", + ollama: "svg", + llama_cpp: "svg", }; export function apiProviderLogoSrc( @@ -42,7 +46,8 @@ interface ApiProviderLogoProps { * OpenAI's asset is black-on-transparent; it is inverted in dark mode for contrast. */ export function ApiProviderLogo({ providerType, className, title }: ApiProviderLogoProps) { - if (providerType === "custom") { + const src = apiProviderLogoSrc(providerType); + if (!src && isCustomProviderType(providerType)) { return ( @@ -50,7 +55,6 @@ export function ApiProviderLogo({ providerType, className, title }: ApiProviderL ); } - const src = apiProviderLogoSrc(providerType); if (!src) return null; return ( s.settingsPanelOpen); const setSettingsOpen = useChatRuntimeStore((s) => s.setSettingsPanelOpen); const externalProviders = useExternalProvidersStore((s) => s.providers); + const setExternalProviders = useExternalProvidersStore((s) => s.setProviders); useEffect(() => { const threadId = search.thread; @@ -629,14 +630,16 @@ export function ChatPage(): ReactElement { const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle); const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort); const supportsReasoningOff = useChatRuntimeStore((s) => s.supportsReasoningOff); - const activeExternalProviderType = useMemo(() => { + const activeExternalProvider = useMemo(() => { const selection = parseExternalModelId(inferenceParams.checkpoint); if (!selection) return null; - const provider = externalProviders.find( - (p) => p.id === selection.providerId, + return ( + externalProviders.find( + (p) => p.id === selection.providerId, + ) ?? null ); - return provider?.providerType ?? null; }, [externalProviders, inferenceParams.checkpoint]); + const activeExternalProviderType = activeExternalProvider?.providerType ?? null; const activeProviderCapabilities = useMemo(() => { const selection = parseExternalModelId(inferenceParams.checkpoint); if (!selection) return null; @@ -671,6 +674,7 @@ export function ChatPage(): ReactElement { const reasoningCaps = getExternalReasoningCapabilities( provider?.providerType, selection.modelId, + { isReasoningProvider: provider?.isReasoningModel === true }, ); const state = useChatRuntimeStore.getState(); const preferredEffort = state.reasoningEffort; @@ -863,6 +867,10 @@ export function ChatPage(): ReactElement { const reasoningCaps = getExternalReasoningCapabilities( selectedProvider?.providerType, selectedExternal?.modelId, + { + isReasoningProvider: + selectedProvider?.isReasoningModel === true, + }, ); const preferredEffort = store.reasoningEffort; const effortLevels = reasoningCaps.reasoningEffortLevels; @@ -1426,6 +1434,14 @@ export function ChatPage(): ReactElement { onParamsChange={setInferenceParams} isExternalModel={isExternalModel} providerCapabilities={activeProviderCapabilities} + activeExternalProvider={activeExternalProvider} + onExternalProviderChange={(updatedProvider) => { + setExternalProviders( + externalProviders.map((provider) => + provider.id === updatedProvider.id ? updatedProvider : provider, + ), + ); + }} externalProviderType={activeExternalProviderType} onReloadModel={() => { const state = useChatRuntimeStore.getState(); diff --git a/studio/frontend/src/features/chat/chat-providers-dialog.tsx b/studio/frontend/src/features/chat/chat-providers-dialog.tsx index 3297e6a992..0127369eb4 100644 --- a/studio/frontend/src/features/chat/chat-providers-dialog.tsx +++ b/studio/frontend/src/features/chat/chat-providers-dialog.tsx @@ -15,7 +15,9 @@ import { Label } from "@/components/ui/label"; import { Select, SelectContent, + SelectGroup, SelectItem, + SelectSeparator, SelectTrigger, SelectValue, } from "@/components/ui/select"; @@ -23,7 +25,6 @@ import { Spinner } from "@/components/ui/spinner"; import { Textarea } from "@/components/ui/textarea"; import { ArrowLeft02Icon, - DashboardSquare01Icon, Delete02Icon, Edit03Icon, PlusSignIcon, @@ -47,9 +48,19 @@ import { } from "./api/providers-api"; import type { ExternalProviderConfig } from "./external-providers"; import { + CUSTOM_BACKEND_PROVIDER_TYPE, + CUSTOM_PROVIDER_PRESETS, + customProviderBaseUrlPlaceholder, + customProviderDisplayName, + customProviderModelIdsPlaceholder, getExternalProviderApiKey, + isCustomProviderType, + LEGACY_CUSTOM_PROVIDER_TYPE, removeExternalProviderApiKey, setExternalProviderApiKey, + supportsProviderPromptCaching, + supportsProviderReasoningToggle, + toExternalBackendProviderType, } from "./external-providers"; /** Matches navbar / thread layout easing (see index.css --ease-out-quart) */ @@ -57,12 +68,11 @@ const PROVIDER_FORM_EASE: [number, number, number, number] = [ 0.165, 0.84, 0.44, 1, ]; const PROVIDER_FORM_DURATION = 0.2; -const CUSTOM_PROVIDER_TYPE = "custom"; -const CUSTOM_BACKEND_PROVIDER_TYPE = "openai"; const CUSTOM_PROVIDER_MISSING_KEY_MESSAGE = "No API key found, please make sure API key is added and valid for this provider."; const ANTHROPIC_DATED_SNAPSHOT_SUFFIX = /-\d{8}$/; const OPENAI_DEPRECATED_MODELS = new Set(["gpt-5.3"]); +const HIDDEN_PROVIDER_TYPES = new Set(["qwen"]); const OPENROUTER_EXCLUDED_MODELS = new Set([ "google/chirp-3", "kwaivgi/kling-v3.0-pro", @@ -82,37 +92,37 @@ function resolveUiProviderTypeFromConfig( registryRows: ProviderRegistryEntry[], existingProviderType: string | undefined, ): string { - if (existingProviderType === CUSTOM_PROVIDER_TYPE) { - return CUSTOM_PROVIDER_TYPE; + if (existingProviderType && isCustomProviderType(existingProviderType)) { + return existingProviderType; } if (configProviderType !== CUSTOM_BACKEND_PROVIDER_TYPE) { return configProviderType; } + const displayName = (configDisplayName ?? "").trim().toLowerCase(); + const matchingCustomPreset = CUSTOM_PROVIDER_PRESETS.find( + (preset) => preset.displayName.toLowerCase() === displayName, + ); + if (matchingCustomPreset) { + return matchingCustomPreset.providerType; + } const openAiRegistry = registryRows.find( (entry) => entry.provider_type === CUSTOM_BACKEND_PROVIDER_TYPE, ); if (!openAiRegistry) { return configProviderType; } - const displayName = (configDisplayName ?? "").trim().toLowerCase(); const openAiDisplayName = openAiRegistry.display_name.trim().toLowerCase(); if (displayName.length > 0 && displayName !== openAiDisplayName) { - return CUSTOM_PROVIDER_TYPE; + return LEGACY_CUSTOM_PROVIDER_TYPE; } const configUrl = normalizeUrl(configBaseUrl ?? ""); const defaultUrl = normalizeUrl(openAiRegistry.base_url ?? ""); if (configUrl.length > 0 && configUrl !== defaultUrl) { - return CUSTOM_PROVIDER_TYPE; + return LEGACY_CUSTOM_PROVIDER_TYPE; } return configProviderType; } -function toBackendProviderType(uiProviderType: string): string { - return uiProviderType === CUSTOM_PROVIDER_TYPE - ? CUSTOM_BACKEND_PROVIDER_TYPE - : uiProviderType; -} - function parseManualModelIds(text: string): string[] { const seen = new Set(); const out: string[] = []; @@ -175,22 +185,22 @@ export function ChatProvidersSettings({ const [manualModelIds, setManualModelIds] = useState(""); const [modelSearchQuery, setModelSearchQuery] = useState(""); const [customProviderName, setCustomProviderName] = useState("Custom"); + const [isReasoningModel, setIsReasoningModel] = useState(false); const reduceMotion = useReducedMotion(); - const isCustomProvider = providerType === CUSTOM_PROVIDER_TYPE; + const isCustomProvider = isCustomProviderType(providerType); + const showReasoningToggle = supportsProviderReasoningToggle(providerType); const registryByType = useMemo( () => new Map(registry.map((entry) => [entry.provider_type, entry])), [registry], ); - const hasCustomInRegistry = registryByType.has(CUSTOM_PROVIDER_TYPE); - const isCuratedModelList = useMemo(() => { return registryByType.get(providerType)?.model_list_mode === "curated"; }, [registryByType, providerType]); const isManualModelList = isCustomProvider || isCuratedModelList; const modelsPanelKey = isCustomProvider - ? "custom" + ? providerType || "custom" : isCuratedModelList ? "curated" : "remote"; @@ -225,7 +235,12 @@ export function ChatProvidersSettings({ useEffect(() => { if (!providerType || editingProviderId) return; const entry = registryByType.get(providerType); - if (!entry) return; + if (!entry) { + if (isCustomProviderType(providerType)) { + setCustomProviderName(customProviderDisplayName(providerType)); + } + return; + } // Seed the registry's default_models for every provider — curated and // remote alike. For remote-mode providers, loadModels() will replace // this with the union of defaults + the live /models response once the @@ -297,6 +312,12 @@ export function ChatProvidersSettings({ baseUrl: config.base_url ?? "", models: existingModels, availableModels: existing?.availableModels ?? [], + enablePromptCaching: supportsProviderPromptCaching(uiProviderType) + ? (existing?.enablePromptCaching ?? true) + : undefined, + isReasoningModel: supportsProviderReasoningToggle(uiProviderType) + ? existing?.isReasoningModel === true + : undefined, createdAt: existing?.createdAt ?? createdAt, updatedAt, }; @@ -328,7 +349,8 @@ export function ChatProvidersSettings({ setSelectedModelIds([]); setManualModelIds(""); setModelSearchQuery(""); - setCustomProviderName("Custom"); + setCustomProviderName(customProviderDisplayName(providerType)); + setIsReasoningModel(false); } function openAddProvider() { @@ -384,7 +406,7 @@ export function ChatProvidersSettings({ const trimmed = input.trim(); if (!trimmed) { if (required) { - throw new Error("Base URL is required for custom providers."); + throw new Error("Base URL is required for this connection."); } return null; } @@ -397,7 +419,7 @@ export function ChatProvidersSettings({ return; } if (isCustomProvider) { - toast.info("Custom providers use manual model IDs."); + toast.info("This connection uses manual model IDs."); return; } if (isCuratedModelList) { @@ -458,10 +480,10 @@ export function ChatProvidersSettings({ toast.error("Choose a provider first."); return; } - const backendProviderType = toBackendProviderType(providerType); + const backendProviderType = toExternalBackendProviderType(providerType); const selectedRegistryEntry = registryByType.get(backendProviderType); const displayName = isCustomProvider - ? customProviderName.trim() || "Custom" + ? customProviderName.trim() || customProviderDisplayName(providerType) : (selectedRegistryEntry?.display_name ?? providerType); if (!isCustomProvider && !apiKey.trim()) { toast.error("API key is required."); @@ -511,17 +533,21 @@ export function ChatProvidersSettings({ const updatedAt = Number.isFinite(Date.parse(created.updated_at)) ? Date.parse(created.updated_at) : Date.now(); + const uiProviderType = isCustomProvider + ? providerType + : created.provider_type; const provider: ExternalProviderConfig = { id: created.id, - providerType: isCustomProvider - ? CUSTOM_PROVIDER_TYPE - : created.provider_type, + providerType: uiProviderType, name: created.display_name, baseUrl: created.base_url ?? "", models: modelsToSave, availableModels: manualModels ? [] : pruneProviderModelIds(providerType, availableModels), + isReasoningModel: supportsProviderReasoningToggle(uiProviderType) + ? isReasoningModel + : undefined, createdAt, updatedAt, }; @@ -553,7 +579,7 @@ export function ChatProvidersSettings({ return; } const isEditingCustomProvider = - existing.providerType === CUSTOM_PROVIDER_TYPE; + isCustomProviderType(existing.providerType); if (!isEditingCustomProvider && !apiKey.trim()) { toast.error("API key is required."); return; @@ -597,7 +623,8 @@ export function ChatProvidersSettings({ ); const updated = await updateProviderConfig(editingProviderId, { displayName: isEditingCustomProvider - ? customProviderName.trim() || "Custom" + ? customProviderName.trim() || + customProviderDisplayName(existing.providerType) : existing.name, baseUrl, }); @@ -620,6 +647,11 @@ export function ChatProvidersSettings({ availableModels: manualModels ? [] : pruneProviderModelIds(existing.providerType, availableModels), + isReasoningModel: supportsProviderReasoningToggle( + existing.providerType, + ) + ? isReasoningModel + : undefined, updatedAt, } : provider, @@ -640,12 +672,19 @@ export function ChatProvidersSettings({ setEditingProviderId(provider.id); setPage("form"); setProviderType(provider.providerType); - setCustomProviderName(provider.name || "Custom"); + setCustomProviderName( + provider.name || customProviderDisplayName(provider.providerType), + ); setApiKey(getExternalProviderApiKey(provider.id)); setShowApiKey(false); setBaseUrlDraft(provider.baseUrl); setModelSearchQuery(""); - if (provider.providerType === CUSTOM_PROVIDER_TYPE) { + setIsReasoningModel( + supportsProviderReasoningToggle(provider.providerType) + ? provider.isReasoningModel === true + : false, + ); + if (isCustomProviderType(provider.providerType)) { setAvailableModels([]); setSelectedModelIds([]); setManualModelIds(provider.models.join("\n")); @@ -696,7 +735,7 @@ export function ChatProvidersSettings({ async function testProvider(provider: ExternalProviderConfig) { const savedKey = getExternalProviderApiKey(provider.id).trim(); if (!savedKey) { - if (provider.providerType === CUSTOM_PROVIDER_TYPE) { + if (isCustomProviderType(provider.providerType)) { await editProvider(provider); toast.info(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE); return; @@ -707,7 +746,9 @@ export function ChatProvidersSettings({ } try { const result = await testProviderConnection({ - providerType: toBackendProviderType(provider.providerType), + providerType: + toExternalBackendProviderType(provider.providerType) ?? + provider.providerType, apiKey: savedKey, baseUrl: provider.baseUrl || null, }); @@ -715,7 +756,7 @@ export function ChatProvidersSettings({ toast.success(result.message); } else { if ( - provider.providerType === CUSTOM_PROVIDER_TYPE && + isCustomProviderType(provider.providerType) && result.message.includes("Illegal header value b'Bearer '") ) { toast.error(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE); @@ -726,7 +767,7 @@ export function ChatProvidersSettings({ } catch (error) { const message = error instanceof Error ? error.message : "Unknown error"; if ( - provider.providerType === CUSTOM_PROVIDER_TYPE && + isCustomProviderType(provider.providerType) && message.includes("Illegal header value b'Bearer '") ) { toast.error(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE); @@ -753,7 +794,7 @@ export function ChatProvidersSettings({
- Cloud + Connections @@ -774,7 +815,7 @@ export function ChatProvidersSettings({ Provider

- Supported registry or Custom. + Supported registry or local OpenAI-compatible connection.

@@ -902,11 +960,35 @@ export function ChatProvidersSettings({ type="text" value={baseUrlDraft} onChange={(event) => setBaseUrlDraft(event.target.value)} - placeholder="https://my-vllm-server.com/v1" + placeholder={customProviderBaseUrlPlaceholder(providerType)} className="h-9 text-sm" /> ) : null} + + {showReasoningToggle ? ( +
+ + +
+ ) : null} @@ -952,7 +1034,7 @@ export function ChatProvidersSettings({ } title={ isCustomProvider - ? "Custom providers use manual model IDs" + ? "This connection uses manual model IDs" : isCuratedModelList ? "Full catalog is not fetched for this provider" : undefined @@ -986,7 +1068,7 @@ export function ChatProvidersSettings({ onChange={(event) => setManualModelIds(event.target.value) } - placeholder={"gpt-4o-mini\nQwen/Qwen3-14B"} + placeholder={customProviderModelIdsPlaceholder(providerType)} rows={5} className="min-h-[100px] resize-y font-mono text-sm" /> @@ -1197,9 +1279,9 @@ export function ChatProvidersSettings({
-

Cloud

+

Connections

- Manage cloud provider connections for chat through the Studio proxy. + Manage model provider connections for chat through the Studio proxy.

@@ -1237,7 +1319,8 @@ export function ChatProvidersSettings({ const detail = provider.baseUrl || registryEntry?.base_url || ""; const providerLabel = - registryEntry?.display_name ?? provider.providerType; + registryEntry?.display_name ?? + customProviderDisplayName(provider.providerType); const modelSummary = formatModelSummary(provider.models); return (
- Cloud + Connections - Manage external model providers for chat. + Manage external model connections for chat. void; /** * Backend provider type for the active external model (e.g. "kimi", * "anthropic", "openai"), or `null` for local models. Drives the @@ -533,6 +539,8 @@ export function ChatSettingsPanel({ onParamsChange, isExternalModel = false, providerCapabilities = null, + activeExternalProvider = null, + onExternalProviderChange, externalProviderType = null, onReloadModel, }: ChatSettingsPanelProps) { @@ -662,6 +670,11 @@ export function ChatSettingsPanel({ Boolean(currentCheckpoint) && modelRequiresTrustRemoteCode && !(params.trustRemoteCode ?? false); + const showPromptCachingControl = + activeExternalProvider != null && + supportsProviderPromptCaching(activeExternalProvider.providerType); + const promptCachingEnabled = + activeExternalProvider?.enablePromptCaching !== false; function set(key: K) { return (v: InferenceParams[K]) => { @@ -1145,6 +1158,32 @@ export function ChatSettingsPanel({
+ {showPromptCachingControl && activeExternalProvider ? ( + +
+
+ + Prompt caching + + + Reuse compatible prompt prefixes for lower latency and cost. + +
+ { + onExternalProviderChange?.({ + ...activeExternalProvider, + enablePromptCaching: checked, + }); + }} + aria-label="Enable prompt caching" + /> +
+ + ) : null} +