Merge branch 'main' into studio-fla-tilelang-qwen3.5
This commit is contained in:
Daniel Han
GitHub
commit
e7aeb32672
20 changed files with 728 additions and 139 deletions
75
.github/workflows/consolidated-tests-ci.yml
vendored
75
.github/workflows/consolidated-tests-ci.yml
vendored
|
|
@ -977,40 +977,59 @@ jobs:
|
|||
skipped -> no `modeling_<x>.py` file (expected for some
|
||||
umbrella packages like `auto`, `deprecated`)
|
||||
known -> in KNOWN_BROKEN_COMPILE; tracked for follow-up.
|
||||
Any uncaught failure fails the cell."""
|
||||
Any uncaught failure fails the cell.
|
||||
|
||||
Per-model SIGALRM cap so one infinite-looping model_type
|
||||
cannot wedge the whole sweep + nuke the job timeout
|
||||
(observed on transformers >=5,<6 -- 30+ min hang before
|
||||
this guard landed)."""
|
||||
import importlib as _il
|
||||
import signal
|
||||
ok = 0
|
||||
skipped = []
|
||||
known = []
|
||||
new_failures = []
|
||||
for model_type in _all_model_types():
|
||||
modeling_path = f"transformers.models.{model_type}.modeling_{model_type}"
|
||||
try:
|
||||
_il.import_module(modeling_path)
|
||||
except (ModuleNotFoundError, ImportError):
|
||||
skipped.append((model_type, "no modeling file"))
|
||||
continue
|
||||
try:
|
||||
unsloth_compile_transformers(
|
||||
model_type=model_type, fast_lora_forwards=False,
|
||||
)
|
||||
except Exception as e:
|
||||
msg = f"{type(e).__name__}: {str(e)[:200]}"
|
||||
models = _all_model_types()
|
||||
def _on_timeout(signum, frame):
|
||||
raise TimeoutError("compile exceeded per-model budget")
|
||||
prev_handler = signal.signal(signal.SIGALRM, _on_timeout)
|
||||
try:
|
||||
for i, model_type in enumerate(models):
|
||||
if i % 25 == 0:
|
||||
print(f" sweep progress: {i}/{len(models)} -> {model_type}", flush=True)
|
||||
modeling_path = f"transformers.models.{model_type}.modeling_{model_type}"
|
||||
try:
|
||||
_il.import_module(modeling_path)
|
||||
except (ModuleNotFoundError, ImportError):
|
||||
skipped.append((model_type, "no modeling file"))
|
||||
continue
|
||||
signal.alarm(60)
|
||||
try:
|
||||
unsloth_compile_transformers(
|
||||
model_type=model_type, fast_lora_forwards=False,
|
||||
)
|
||||
except Exception as e:
|
||||
signal.alarm(0)
|
||||
msg = f"{type(e).__name__}: {str(e)[:200]}"
|
||||
if model_type in KNOWN_BROKEN_COMPILE:
|
||||
known.append((model_type, msg))
|
||||
else:
|
||||
new_failures.append((model_type, msg))
|
||||
continue
|
||||
signal.alarm(0)
|
||||
if model_type in KNOWN_BROKEN_COMPILE:
|
||||
known.append((model_type, msg))
|
||||
else:
|
||||
new_failures.append((model_type, msg))
|
||||
continue
|
||||
if model_type in KNOWN_BROKEN_COMPILE:
|
||||
# Came back green unexpectedly -- that's GOOD news,
|
||||
# the bug was fixed. Surface it so we can drop the
|
||||
# entry from KNOWN_BROKEN_COMPILE.
|
||||
print(
|
||||
f" UNEXPECTED-OK {model_type}: was in "
|
||||
"KNOWN_BROKEN_COMPILE, now compiles cleanly. "
|
||||
"Drop the entry."
|
||||
)
|
||||
ok += 1
|
||||
# Came back green unexpectedly -- that's GOOD news,
|
||||
# the bug was fixed. Surface it so we can drop the
|
||||
# entry from KNOWN_BROKEN_COMPILE.
|
||||
print(
|
||||
f" UNEXPECTED-OK {model_type}: was in "
|
||||
"KNOWN_BROKEN_COMPILE, now compiles cleanly. "
|
||||
"Drop the entry."
|
||||
)
|
||||
ok += 1
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
signal.signal(signal.SIGALRM, prev_handler)
|
||||
print(f"\nCompile sweep: ok={ok} skipped={len(skipped)} "
|
||||
f"known-broken={len(known)} new-failures={len(new_failures)}")
|
||||
for m, r in known:
|
||||
|
|
|
|||
|
|
@ -24,11 +24,17 @@ import structlog
|
|||
# sites use printf-style positional args, which structlog accepts.
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# Claude 4.7 (Opus/Sonnet/Haiku) deprecated top_k and returns 400
|
||||
# "top_k is deprecated for this model" when it is set. 3.x and 4.5/4.6
|
||||
# still accept it. Match the 4-7 line specifically so we keep the knob
|
||||
# live on every other Claude generation.
|
||||
_ANTHROPIC_TOP_K_DEPRECATED = re.compile(r"^claude-(?:opus|sonnet|haiku)-4-7(?:[-.]|$)")
|
||||
# Claude 4.7 (Opus/Sonnet/Haiku) removed temperature, top_p, and top_k —
|
||||
# the API returns 400 "<param> is deprecated for this model" if any of
|
||||
# them is set to a non-default value. The "Sampling parameters removed"
|
||||
# section of the 4.7 release notes is the authoritative reference:
|
||||
# https://platform.claude.com/docs/en/about-claude/models/whats-new-claude-4-7
|
||||
# 3.x and 4.5/4.6 still accept all three; match the 4-7 line strictly so
|
||||
# the knobs keep working on earlier families. The trailing -4-7[-.]/EOL
|
||||
# anchor keeps future versions (e.g. claude-opus-5) unaffected.
|
||||
_ANTHROPIC_4_7_SAMPLING_REMOVED = re.compile(
|
||||
r"^claude-(?:opus|sonnet|haiku)-4-7(?:[-.]|$)"
|
||||
)
|
||||
_OPENAI_REASONING_SUMMARY_UNSUPPORTED = re.compile(r"^o3(?:[-.]|$)")
|
||||
|
||||
|
||||
|
|
@ -229,6 +235,7 @@ class ExternalProviderClient:
|
|||
enable_thinking: Optional[bool] = None,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
enabled_tools: Optional[list[str]] = None,
|
||||
enable_prompt_caching: Optional[bool] = None,
|
||||
stream: bool = True,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
|
|
@ -253,6 +260,7 @@ class ExternalProviderClient:
|
|||
enable_thinking,
|
||||
reasoning_effort,
|
||||
enabled_tools,
|
||||
enable_prompt_caching,
|
||||
):
|
||||
yield line
|
||||
return
|
||||
|
|
@ -272,6 +280,7 @@ class ExternalProviderClient:
|
|||
enable_thinking,
|
||||
reasoning_effort,
|
||||
enabled_tools,
|
||||
enable_prompt_caching,
|
||||
):
|
||||
yield line
|
||||
return
|
||||
|
|
@ -346,6 +355,13 @@ class ExternalProviderClient:
|
|||
_apply_mistral_reasoning_controls(
|
||||
body, model, enable_thinking, reasoning_effort
|
||||
)
|
||||
elif self.provider_type == "vllm" and enable_thinking is not None:
|
||||
# vLLM gates thinking via chat_template_kwargs.enable_thinking.
|
||||
tpl_kw = body.get("chat_template_kwargs")
|
||||
if not isinstance(tpl_kw, dict):
|
||||
tpl_kw = {}
|
||||
tpl_kw["enable_thinking"] = bool(enable_thinking)
|
||||
body["chat_template_kwargs"] = tpl_kw
|
||||
|
||||
# OpenRouter exposes a unified `reasoning` parameter on every
|
||||
# chat-completion request — the gateway routes it to whichever
|
||||
|
|
@ -1043,6 +1059,7 @@ class ExternalProviderClient:
|
|||
enable_thinking: Optional[bool] = None,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
enabled_tools: Optional[list[str]] = None,
|
||||
enable_prompt_caching: Optional[bool] = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Call the Anthropic Messages API and translate its SSE to OpenAI format.
|
||||
|
|
@ -1112,24 +1129,79 @@ class ExternalProviderClient:
|
|||
else:
|
||||
filtered.append(msg)
|
||||
|
||||
# Claude 4.7 family removed temperature / top_p / top_k entirely.
|
||||
# The earlier guard only handled top_k; temperature is now also
|
||||
# rejected with 400 "temperature is deprecated for this model".
|
||||
# Latch the match once and reuse it everywhere temperature or
|
||||
# top_k would otherwise be set — including the thinking-mode
|
||||
# override below, which used to force temperature=1.
|
||||
sampling_removed = bool(_ANTHROPIC_4_7_SAMPLING_REMOVED.match(model))
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": filtered,
|
||||
"max_tokens": max_tokens or 1024, # required by Anthropic
|
||||
"temperature": temperature,
|
||||
"stream": True,
|
||||
}
|
||||
# top_k is deprecated on Claude 4.7 (Opus/Sonnet/Haiku) — the API
|
||||
# returns 400 "top_k is deprecated for this model" when it is set.
|
||||
# 3.x and 4.5/4.6 still accept it, so gate strictly on the 4.7 ids.
|
||||
if (
|
||||
top_k is not None
|
||||
and top_k > 0
|
||||
and not _ANTHROPIC_TOP_K_DEPRECATED.match(model)
|
||||
):
|
||||
if not sampling_removed:
|
||||
body["temperature"] = temperature
|
||||
if top_k is not None and top_k > 0 and not sampling_removed:
|
||||
body["top_k"] = top_k
|
||||
# Anthropic only caches a prefix when at least one cache_control
|
||||
# marker is attached to it — the frontend defaults
|
||||
# enable_prompt_caching to True for Anthropic, so treat `None` the
|
||||
# same as True here (callers that don't set the flag still get
|
||||
# caching). Pass False explicitly to opt out.
|
||||
prompt_caching_enabled = enable_prompt_caching is not False
|
||||
|
||||
if system:
|
||||
body["system"] = system
|
||||
if prompt_caching_enabled:
|
||||
# System block is the most stable prefix across turns, so
|
||||
# it gets its own breakpoint. Skipped when system is
|
||||
# empty — there's nothing to cache, and an empty marker
|
||||
# is a no-op.
|
||||
body["system"] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": system,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
else:
|
||||
body["system"] = system
|
||||
|
||||
if prompt_caching_enabled and filtered:
|
||||
# Second breakpoint at the end of the conversation. Anthropic
|
||||
# caches the longest matching prefix up to a cache_control
|
||||
# marker; placing one on the latest message means turn N+1
|
||||
# rehydrates everything up through turn N from cache instead
|
||||
# of recomputing it. This is what makes caching actually work
|
||||
# when the system prompt is empty or shorter than Anthropic's
|
||||
# ~1024-token cache floor — the conversation history carries
|
||||
# the bulk of the input tokens. Anthropic allows up to 4
|
||||
# breakpoints per request; we use at most 2 (system + tail).
|
||||
last_msg = filtered[-1]
|
||||
content = last_msg.get("content")
|
||||
if isinstance(content, str):
|
||||
last_msg["content"] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": content,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
elif isinstance(content, list) and content:
|
||||
# Don't mutate the caller's list. Rebuild the tail with
|
||||
# cache_control attached to the final block so an
|
||||
# upstream image-bearing turn still cleanly slots into
|
||||
# the cache as part of the conversational prefix.
|
||||
head = list(content[:-1])
|
||||
tail = content[-1]
|
||||
if isinstance(tail, dict):
|
||||
head.append({**tail, "cache_control": {"type": "ephemeral"}})
|
||||
else:
|
||||
head.append(tail)
|
||||
last_msg["content"] = head
|
||||
thinking_spec = _anthropic_thinking_spec(model)
|
||||
allowed_efforts = (
|
||||
thinking_spec.efforts
|
||||
|
|
@ -1155,13 +1227,15 @@ class ExternalProviderClient:
|
|||
if effort and effort != "none":
|
||||
# Anthropic rejects top_k whenever thinking is enabled.
|
||||
body.pop("top_k", None)
|
||||
# Anthropic requires temperature=1 whenever thinking is enabled,
|
||||
# AND forbids top_p in the same request: setting both produces
|
||||
# Earlier families (4.5/4.6) require temperature=1 when
|
||||
# thinking is enabled and forbid top_p in the same request:
|
||||
# "temperature and top_p cannot both be specified for this
|
||||
# model. Please use only one."
|
||||
# The base body never sets top_p, but pop defensively in case
|
||||
# an upstream edit ever adds it before this branch runs.
|
||||
body["temperature"] = 1
|
||||
# On Claude 4.7, temperature was removed entirely — sending
|
||||
# any value (including 1) returns 400 — so skip the override
|
||||
# there and let the model use its default sampling.
|
||||
if not sampling_removed:
|
||||
body["temperature"] = 1
|
||||
body.pop("top_p", None)
|
||||
if thinking_spec and thinking_spec.kind == "adaptive":
|
||||
# `display` defaults to "omitted" on Claude Opus 4.7 (per the
|
||||
|
|
@ -1278,6 +1352,13 @@ class ExternalProviderClient:
|
|||
current_server_tool_use: Optional[dict[str, Any]] = None
|
||||
current_result_block: Optional[dict[str, Any]] = None
|
||||
web_search_calls: dict[str, dict[str, Any]] = {}
|
||||
# Cache usage tracking. message_start carries the input
|
||||
# accounting (incl. cache_creation_input_tokens and
|
||||
# cache_read_input_tokens); message_delta carries cumulative
|
||||
# output_tokens. Both are surfaced in the "stream complete"
|
||||
# log so prompt caching can be verified per-request without
|
||||
# opening the Anthropic dashboard.
|
||||
last_usage: dict[str, Any] = {}
|
||||
|
||||
def _content_chunk(text: str) -> str:
|
||||
chunk = {
|
||||
|
|
@ -1352,6 +1433,16 @@ class ExternalProviderClient:
|
|||
key = event_type or "<unknown>"
|
||||
event_counts[key] = event_counts.get(key, 0) + 1
|
||||
|
||||
# message_start carries the input-side usage block
|
||||
# including cache_creation_input_tokens and
|
||||
# cache_read_input_tokens. message_delta updates
|
||||
# output_tokens (and may overwrite the input fields
|
||||
# with final values). Merge both into last_usage.
|
||||
if event_type == "message_start":
|
||||
start_usage = (event.get("message") or {}).get("usage")
|
||||
if isinstance(start_usage, dict):
|
||||
last_usage.update(start_usage)
|
||||
|
||||
if event_type == "content_block_start":
|
||||
content_block = event.get("content_block") or {}
|
||||
block_type = content_block.get("type")
|
||||
|
|
@ -1489,6 +1580,9 @@ class ExternalProviderClient:
|
|||
thinking_open = False
|
||||
|
||||
elif event_type == "message_delta":
|
||||
delta_usage = event.get("usage")
|
||||
if isinstance(delta_usage, dict):
|
||||
last_usage.update(delta_usage)
|
||||
stop_reason = event.get("delta", {}).get("stop_reason")
|
||||
if stop_reason:
|
||||
if thinking_open:
|
||||
|
|
@ -1538,15 +1632,28 @@ class ExternalProviderClient:
|
|||
for sc in web_search_calls.values()
|
||||
if sc.get("query")
|
||||
]
|
||||
# cache_read_input_tokens > 0 on turn N proves the
|
||||
# cache_control marker on the system block is doing
|
||||
# its job — turn 1 will show cache_creation > 0
|
||||
# instead. cache_creation tokens are billed at a
|
||||
# small premium; cache_read tokens are billed at a
|
||||
# discount.
|
||||
logger.info(
|
||||
"Anthropic stream complete (model=%s, "
|
||||
"web_search_requested=%s, web_search_invocations=%s, "
|
||||
"results=%s, queries=%s, events=%s)",
|
||||
"results=%s, queries=%s, "
|
||||
"input_tokens=%s, output_tokens=%s, "
|
||||
"cache_creation_input_tokens=%s, "
|
||||
"cache_read_input_tokens=%s, events=%s)",
|
||||
model,
|
||||
web_search_requested,
|
||||
web_search_invocations,
|
||||
total_results,
|
||||
queries,
|
||||
last_usage.get("input_tokens"),
|
||||
last_usage.get("output_tokens"),
|
||||
last_usage.get("cache_creation_input_tokens"),
|
||||
last_usage.get("cache_read_input_tokens"),
|
||||
event_counts,
|
||||
)
|
||||
await response.aclose()
|
||||
|
|
@ -1584,6 +1691,7 @@ class ExternalProviderClient:
|
|||
enable_thinking: Optional[bool],
|
||||
reasoning_effort: Optional[str],
|
||||
enabled_tools: Optional[list[str]] = None,
|
||||
enable_prompt_caching: Optional[bool] = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Call OpenAI's /v1/responses endpoint and translate its SSE stream back
|
||||
|
|
@ -1685,6 +1793,27 @@ class ExternalProviderClient:
|
|||
if max_tokens is not None:
|
||||
body["max_output_tokens"] = max_tokens
|
||||
|
||||
# Prompt caching on /v1/responses is automatic and free, but the
|
||||
# default in-memory policy only survives ~5-10 min of inactivity
|
||||
# (up to ~1 hr). Opt into the 24-hour retention policy so a chat
|
||||
# left idle overnight still hits the cache on the next turn.
|
||||
# Pricing is identical to in_memory per OpenAI's docs.
|
||||
#
|
||||
# Gated on the base URL because ollama / llama.cpp / "custom"
|
||||
# presets all collapse to provider_type="openai" in
|
||||
# toExternalBackendProviderType, so they also land in this
|
||||
# helper. Those servers expose /v1/responses-shaped routes in
|
||||
# some configurations but don't implement
|
||||
# prompt_cache_retention — sending the field unconditionally
|
||||
# would 400 them. Match the public OpenAI host strictly so the
|
||||
# field only goes to OpenAI cloud. Studio's openai model picker
|
||||
# is registry-scoped to gpt-5.x / o3 / gpt-4.5, all of which
|
||||
# accept this parameter (gpt-5.5+ already defaults to "24h" and
|
||||
# rejects "in_memory", so it's a safe no-op there).
|
||||
is_openai_cloud = "api.openai.com" in (self.base_url or "")
|
||||
if is_openai_cloud and enable_prompt_caching is not False:
|
||||
body["prompt_cache_retention"] = "24h"
|
||||
|
||||
# OpenAI server-side tools — see
|
||||
# https://developers.openai.com/api/docs/guides/tools
|
||||
# The frontend's Search button maps to the unified
|
||||
|
|
@ -1731,6 +1860,12 @@ class ExternalProviderClient:
|
|||
done_emitted = False
|
||||
reasoning_open = False
|
||||
reasoning_emitted = False
|
||||
# Latched from response.completed / response.incomplete so
|
||||
# the final log can surface input_tokens_details.cached_tokens —
|
||||
# the field that proves prompt_cache_retention="24h" is
|
||||
# actually hitting OpenAI's cache instead of recomputing
|
||||
# the prefix every turn.
|
||||
last_usage: Optional[dict[str, Any]] = None
|
||||
# Per-call state for OpenAI's server-side web_search tool. Mapped
|
||||
# back into our local _toolEvent shape so the existing chat-UI
|
||||
# renderer surfaces web_search the same way it does for local
|
||||
|
|
@ -1955,6 +2090,9 @@ class ExternalProviderClient:
|
|||
reasoning_emitted = True
|
||||
|
||||
elif event_type == "response.completed":
|
||||
completed_usage = (event.get("response") or {}).get("usage")
|
||||
if isinstance(completed_usage, dict):
|
||||
last_usage = completed_usage
|
||||
if reasoning_open:
|
||||
yield _chunk_with_text("</think>")
|
||||
reasoning_open = False
|
||||
|
|
@ -1998,6 +2136,11 @@ class ExternalProviderClient:
|
|||
yield f"data: {_json.dumps(chunk)}"
|
||||
|
||||
elif event_type == "response.incomplete":
|
||||
incomplete_usage = (event.get("response") or {}).get(
|
||||
"usage"
|
||||
)
|
||||
if isinstance(incomplete_usage, dict):
|
||||
last_usage = incomplete_usage
|
||||
if reasoning_open:
|
||||
yield _chunk_with_text("</think>")
|
||||
reasoning_open = False
|
||||
|
|
@ -2071,16 +2214,33 @@ class ExternalProviderClient:
|
|||
for sc in web_search_calls.values()
|
||||
if sc.get("query")
|
||||
]
|
||||
# cached_input_tokens > 0 on turn N proves
|
||||
# prompt_cache_retention="24h" is letting the previous
|
||||
# turn's prefix hit the cache instead of being
|
||||
# recomputed. On /v1/responses the field is nested as
|
||||
# usage.input_tokens_details.cached_tokens (not
|
||||
# prompt_tokens_details, which is the /v1/chat/completions
|
||||
# shape).
|
||||
cached_input_tokens = None
|
||||
if isinstance(last_usage, dict):
|
||||
details = last_usage.get("input_tokens_details")
|
||||
if isinstance(details, dict):
|
||||
cached_input_tokens = details.get("cached_tokens")
|
||||
logger.info(
|
||||
"OpenAI Responses stream complete (model=%s, "
|
||||
"web_search_requested=%s, web_search_invocations=%s, "
|
||||
"citations=%s, queries=%s, reasoning_emitted=%s)",
|
||||
"citations=%s, queries=%s, reasoning_emitted=%s, "
|
||||
"input_tokens=%s, output_tokens=%s, "
|
||||
"cached_input_tokens=%s)",
|
||||
model,
|
||||
web_search_requested,
|
||||
web_search_invocations,
|
||||
total_citations,
|
||||
queries,
|
||||
reasoning_emitted,
|
||||
(last_usage or {}).get("input_tokens"),
|
||||
(last_usage or {}).get("output_tokens"),
|
||||
cached_input_tokens,
|
||||
)
|
||||
await response.aclose()
|
||||
await lines_gen.aclose()
|
||||
|
|
|
|||
|
|
@ -218,6 +218,28 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
|
|||
# are always among the top regardless of the API's order.
|
||||
"model_id_limit": 15,
|
||||
},
|
||||
"vllm": {
|
||||
"display_name": "vLLM",
|
||||
# User-supplied via provider_base_url; the route layer already falls
|
||||
# back to the payload's base_url when the registry entry has none.
|
||||
"base_url": "",
|
||||
"default_models": [],
|
||||
"supports_streaming": True,
|
||||
"supports_vision": True,
|
||||
"supports_tool_calling": True,
|
||||
"auth_header": "Authorization",
|
||||
"auth_prefix": "Bearer ",
|
||||
# Force /v1/chat/completions in stream_chat_completion — vLLM's
|
||||
# /v1/responses rebuilds messages and runs them through the loaded
|
||||
# model's chat template, which 400s on strict-alternation templates
|
||||
# (Gemma 3 raises "Conversation roles must alternate user/assistant
|
||||
# /user/assistant/..."). The chat-completions path takes messages
|
||||
# verbatim and avoids that template gauntlet.
|
||||
"notes": "Self-hosted vLLM server. Always routed to /v1/chat/completions.",
|
||||
# Surfaced through the frontend's CUSTOM_PROVIDER_PRESETS, not the
|
||||
# /api/providers/registry dropdown — see list_available_providers.
|
||||
"hidden": True,
|
||||
},
|
||||
"openrouter": {
|
||||
"display_name": "OpenRouter",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
|
|
@ -269,9 +291,17 @@ def get_base_url(provider_type: str) -> str | None:
|
|||
|
||||
|
||||
def list_available_providers() -> list[dict[str, Any]]:
|
||||
"""Return all registered providers (for the /registry endpoint)."""
|
||||
"""Return all registered providers (for the /registry endpoint).
|
||||
|
||||
Hidden entries (``"hidden": True``) are filtered out — they exist in the
|
||||
registry only for backend lookups (e.g. ``supports_vision`` for vLLM) and
|
||||
are surfaced in the frontend via ``CUSTOM_PROVIDER_PRESETS`` instead of
|
||||
the cloud-provider dropdown.
|
||||
"""
|
||||
result = []
|
||||
for provider_type, info in PROVIDER_REGISTRY.items():
|
||||
if info.get("hidden"):
|
||||
continue
|
||||
result.append(
|
||||
{
|
||||
"provider_type": provider_type,
|
||||
|
|
|
|||
|
|
@ -593,6 +593,17 @@ class ChatCompletionRequest(BaseModel):
|
|||
None,
|
||||
description = "[x-unsloth] Override base URL for the external provider.",
|
||||
)
|
||||
enable_prompt_caching: Optional[bool] = Field(
|
||||
None,
|
||||
description = (
|
||||
"[x-unsloth] Opt in to provider-side prompt caching. On Anthropic, "
|
||||
"attaches cache_control={type:ephemeral} to the system block so the "
|
||||
"static prefix is reused across turns. On OpenAI cloud, caching is "
|
||||
"automatic for prompts >=1024 tokens and this flag is informational. "
|
||||
"Ignored for every other provider (mistral, gemini, kimi, openrouter, "
|
||||
"vllm, local, etc.). Treated as enabled when omitted."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ── Streaming response chunks ────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -1596,6 +1596,7 @@ async def _proxy_to_external_provider(
|
|||
enable_thinking = payload.enable_thinking,
|
||||
reasoning_effort = payload.reasoning_effort,
|
||||
enabled_tools = payload.enabled_tools,
|
||||
enable_prompt_caching = payload.enable_prompt_caching,
|
||||
stream = payload.stream,
|
||||
)
|
||||
try:
|
||||
|
|
|
|||
1
studio/frontend/public/provider-logos/llama_cpp.svg
Normal file
1
studio/frontend/public/provider-logos/llama_cpp.svg
Normal file
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m356.4 201.3-32.8 58.3c-43.3-33.3-107.4-38.2-150.7-2.4-69.8 57.6-64.9 190.8 43.7 191.6 30.4 0 56.2-14.3 83.9-23.8l14.6 58.1c-24.6 11.4-49.6 23.1-76.6 26.7-246 33.5-231.9-321.6-9.5-340.1 46.7-3.9 87.8 8.3 127.6 31.6zm-169.9-55.9c-37.4 11.2-72.2 31.8-98.5 60.8-4.9-58.8 8.3-177.7 73.7-201 9.7-3.4 43-11.9 42.1 5.3-1 17.3-24.1 46.9-29.7 63-9.7 28.2-.7 47.6 12.6 72.2zm92.4 252.8h-36.5v-41.3h-41.3v-34h37.7l3.6-3.6v-40.1h36.5V323h38.9v34h-38.9zm133.7-41.3v41.3h-36.5v-41.3h-38.9v-34h38.9v-43.8h36.5v40.1l3.6 3.6h37.7v34h-41.3zM305.4 31.4c4.9 7.3-22.6 38.7-27 46.7-12.6 23.8-4.1 37.4 5.3 60-27.5-4.1-53-.7-80.2 2.4C209.6 88.3 239 12.2 305.4 31.4" style="fill:#ff8236"/></svg>
|
||||
|
After Width: | Height: | Size: 763 B |
14
studio/frontend/public/provider-logos/ollama.svg
Normal file
14
studio/frontend/public/provider-logos/ollama.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 8.6 KiB |
1
studio/frontend/public/provider-logos/vllm.svg
Normal file
1
studio/frontend/public/provider-logos/vllm.svg
Normal file
|
|
@ -0,0 +1 @@
|
|||
<svg version="1.1" viewBox="0.0 0.0 96.0 96.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="g31e21232314_0_33.0"><path d="m0 0l96.0 0l0 96.0l-96.0 0l0 -96.0z" clip-rule="nonzero"/></clipPath><g clip-path="url(#g31e21232314_0_33.0)"><path fill="#d9d9d9" d="m41.04961 80.271324l1.8897629 0l0 2.3307114l-1.8897629 0z" fill-rule="evenodd"/><path fill="#d9d9d9" d="m42.221855 81.45145l1.8897629 0l0 2.3307037l-1.8897629 0z" fill-rule="evenodd"/><g filter="url(#shadowFilter-g31e21232314_0_33.1)"><use xlink:href="#g31e21232314_0_33.1" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.0)"/></g><defs><filter id="shadowFilter-g31e21232314_0_33.1" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="2.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.0" intercept="0"/></feComponentTransfer></filter></defs><g id="g31e21232314_0_33.1"><path fill="#d9d9d9" d="m42.22417 28.470434l0 55.307083l-27.653543 -55.307083z" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-g31e21232314_0_33.2)"><use xlink:href="#g31e21232314_0_33.2" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.0)"/></g><defs><filter id="shadowFilter-g31e21232314_0_33.2" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="2.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.0" intercept="0"/></feComponentTransfer></filter></defs><g id="g31e21232314_0_33.2"><path fill="#d9d9d9" d="m42.223038 83.77752l21.729656 0l18.653545 -70.385826l-25.574802 13.461943z" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-g31e21232314_0_33.3)"><use xlink:href="#g31e21232314_0_33.3" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.0)"/></g><defs><filter id="shadowFilter-g31e21232314_0_33.3" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="2.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.0" intercept="0"/></feComponentTransfer></filter></defs><g id="g31e21232314_0_33.3"><path fill="#fdb515" d="m41.0477 27.293962l0 55.30709l-27.653542 -55.30709z" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-g31e21232314_0_33.4)"><use xlink:href="#g31e21232314_0_33.4" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.0)"/></g><defs><filter id="shadowFilter-g31e21232314_0_33.4" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="2.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.0" intercept="0"/></feComponentTransfer></filter></defs><g id="g31e21232314_0_33.4"><path fill="#30a2ff" d="m41.046566 82.60105l21.72966 0l18.653545 -70.385826l-25.574806 13.461943z" fill-rule="evenodd"/></g></g></svg>
|
||||
|
After Width: | Height: | Size: 3.4 KiB |
|
|
@ -10,10 +10,12 @@ import {
|
|||
} from "@/components/ui/popover";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { usePlatformStore } from "@/config/env";
|
||||
import { isCustomProviderType } from "@/features/chat/external-providers";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
ArrowDown01Icon,
|
||||
CloudIcon,
|
||||
DashboardSquare01Icon,
|
||||
FolderSearchIcon,
|
||||
Logout01Icon,
|
||||
Search01Icon,
|
||||
|
|
@ -40,6 +42,9 @@ const PROVIDER_LOGO_EXT: Record<string, "svg" | "png" | "jpg"> = {
|
|||
kimi: "jpg",
|
||||
qwen: "png",
|
||||
openrouter: "svg",
|
||||
vllm: "svg",
|
||||
ollama: "svg",
|
||||
llama_cpp: "svg",
|
||||
};
|
||||
|
||||
function providerLogoSrc(providerType: string | undefined): string | undefined {
|
||||
|
|
@ -59,6 +64,17 @@ function ExternalProviderLogo({
|
|||
title?: string;
|
||||
}) {
|
||||
const src = providerLogoSrc(providerType);
|
||||
if (!src && isCustomProviderType(providerType)) {
|
||||
return (
|
||||
<span title={title} aria-hidden={true} className="inline-flex shrink-0">
|
||||
<HugeiconsIcon
|
||||
icon={DashboardSquare01Icon}
|
||||
className={cn("shrink-0", className)}
|
||||
/>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
if (!src) return null;
|
||||
return (
|
||||
<img
|
||||
|
|
|
|||
|
|
@ -510,6 +510,10 @@ const ReasoningToggle: FC = () => {
|
|||
? getExternalReasoningCapabilities(
|
||||
selectedExternalProvider?.providerType,
|
||||
effectiveExternalModelId,
|
||||
{
|
||||
isReasoningProvider:
|
||||
selectedExternalProvider?.isReasoningModel === true,
|
||||
},
|
||||
)
|
||||
: null;
|
||||
const effectiveReasoningStyle =
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
import { cn } from "@/lib/utils";
|
||||
import { DashboardSquare01Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { isCustomProviderType } from "./external-providers";
|
||||
|
||||
/**
|
||||
* Registry logos live at `public/provider-logos/{provider_type}.{ext}` where `provider_type`
|
||||
|
|
@ -19,6 +20,9 @@ const PROVIDER_LOGO_EXT: Record<string, "svg" | "png" | "jpg"> = {
|
|||
kimi: "jpg",
|
||||
qwen: "png",
|
||||
openrouter: "svg",
|
||||
vllm: "svg",
|
||||
ollama: "svg",
|
||||
llama_cpp: "svg",
|
||||
};
|
||||
|
||||
export function apiProviderLogoSrc(
|
||||
|
|
@ -42,7 +46,8 @@ interface ApiProviderLogoProps {
|
|||
* OpenAI's asset is black-on-transparent; it is inverted in dark mode for contrast.
|
||||
*/
|
||||
export function ApiProviderLogo({ providerType, className, title }: ApiProviderLogoProps) {
|
||||
if (providerType === "custom") {
|
||||
const src = apiProviderLogoSrc(providerType);
|
||||
if (!src && isCustomProviderType(providerType)) {
|
||||
return (
|
||||
<span title={title} aria-hidden className="inline-flex shrink-0">
|
||||
<HugeiconsIcon icon={DashboardSquare01Icon} className={cn("shrink-0", className)} />
|
||||
|
|
@ -50,7 +55,6 @@ export function ApiProviderLogo({ providerType, className, title }: ApiProviderL
|
|||
);
|
||||
}
|
||||
|
||||
const src = apiProviderLogoSrc(providerType);
|
||||
if (!src) return null;
|
||||
return (
|
||||
<img
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ import {
|
|||
getExternalProviderApiKey,
|
||||
loadExternalProviders,
|
||||
parseExternalModelId,
|
||||
supportsProviderPromptCaching,
|
||||
toExternalBackendProviderType,
|
||||
} from "../external-providers";
|
||||
import {
|
||||
EXTERNAL_MAX_OUTPUT_TOKENS,
|
||||
|
|
@ -742,13 +744,13 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
|
||||
if (isExternalRequest && !externalProvider) {
|
||||
toast.error("External provider not found.", {
|
||||
description: "Open API Providers and re-add this provider.",
|
||||
description: "Open Connections and re-add this provider.",
|
||||
});
|
||||
throw new Error("External provider not found.");
|
||||
}
|
||||
if (isExternalRequest && !externalApiKey) {
|
||||
toast.error("Missing API key for selected external provider.", {
|
||||
description: "Open API Providers and set the API key again.",
|
||||
description: "Open Connections and set the API key again.",
|
||||
});
|
||||
throw new Error("Missing external provider API key.");
|
||||
}
|
||||
|
|
@ -929,10 +931,9 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
supportsPreserveThinking,
|
||||
preserveThinking,
|
||||
} = runtime;
|
||||
const externalBackendProviderType =
|
||||
externalProvider?.providerType === "custom"
|
||||
? "openai"
|
||||
: externalProvider?.providerType;
|
||||
const externalBackendProviderType = toExternalBackendProviderType(
|
||||
externalProvider?.providerType,
|
||||
);
|
||||
const externalCapabilities = getProviderCapabilities(
|
||||
externalProvider?.providerType,
|
||||
);
|
||||
|
|
@ -943,6 +944,10 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
? getExternalReasoningCapabilities(
|
||||
externalProvider.providerType,
|
||||
externalSelection.modelId,
|
||||
{
|
||||
isReasoningProvider:
|
||||
externalProvider.isReasoningModel === true,
|
||||
},
|
||||
)
|
||||
: {
|
||||
supportsReasoning,
|
||||
|
|
@ -1028,6 +1033,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
forceRefreshPublicKey,
|
||||
),
|
||||
provider_base_url: externalProvider.baseUrl || null,
|
||||
...(supportsProviderPromptCaching(externalProvider.providerType)
|
||||
? {
|
||||
enable_prompt_caching:
|
||||
externalProvider.enablePromptCaching ?? true,
|
||||
}
|
||||
: {}),
|
||||
...(externalReasoningCaps.supportsReasoning
|
||||
? externalReasoningCaps.reasoningStyle === "reasoning_effort"
|
||||
? externalReasoningEnabled
|
||||
|
|
|
|||
|
|
@ -550,6 +550,7 @@ export function ChatPage(): ReactElement {
|
|||
const settingsOpen = useChatRuntimeStore((s) => s.settingsPanelOpen);
|
||||
const setSettingsOpen = useChatRuntimeStore((s) => s.setSettingsPanelOpen);
|
||||
const externalProviders = useExternalProvidersStore((s) => s.providers);
|
||||
const setExternalProviders = useExternalProvidersStore((s) => s.setProviders);
|
||||
|
||||
useEffect(() => {
|
||||
const threadId = search.thread;
|
||||
|
|
@ -629,14 +630,16 @@ export function ChatPage(): ReactElement {
|
|||
const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle);
|
||||
const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort);
|
||||
const supportsReasoningOff = useChatRuntimeStore((s) => s.supportsReasoningOff);
|
||||
const activeExternalProviderType = useMemo(() => {
|
||||
const activeExternalProvider = useMemo(() => {
|
||||
const selection = parseExternalModelId(inferenceParams.checkpoint);
|
||||
if (!selection) return null;
|
||||
const provider = externalProviders.find(
|
||||
(p) => p.id === selection.providerId,
|
||||
return (
|
||||
externalProviders.find(
|
||||
(p) => p.id === selection.providerId,
|
||||
) ?? null
|
||||
);
|
||||
return provider?.providerType ?? null;
|
||||
}, [externalProviders, inferenceParams.checkpoint]);
|
||||
const activeExternalProviderType = activeExternalProvider?.providerType ?? null;
|
||||
const activeProviderCapabilities = useMemo(() => {
|
||||
const selection = parseExternalModelId(inferenceParams.checkpoint);
|
||||
if (!selection) return null;
|
||||
|
|
@ -671,6 +674,7 @@ export function ChatPage(): ReactElement {
|
|||
const reasoningCaps = getExternalReasoningCapabilities(
|
||||
provider?.providerType,
|
||||
selection.modelId,
|
||||
{ isReasoningProvider: provider?.isReasoningModel === true },
|
||||
);
|
||||
const state = useChatRuntimeStore.getState();
|
||||
const preferredEffort = state.reasoningEffort;
|
||||
|
|
@ -863,6 +867,10 @@ export function ChatPage(): ReactElement {
|
|||
const reasoningCaps = getExternalReasoningCapabilities(
|
||||
selectedProvider?.providerType,
|
||||
selectedExternal?.modelId,
|
||||
{
|
||||
isReasoningProvider:
|
||||
selectedProvider?.isReasoningModel === true,
|
||||
},
|
||||
);
|
||||
const preferredEffort = store.reasoningEffort;
|
||||
const effortLevels = reasoningCaps.reasoningEffortLevels;
|
||||
|
|
@ -1426,6 +1434,14 @@ export function ChatPage(): ReactElement {
|
|||
onParamsChange={setInferenceParams}
|
||||
isExternalModel={isExternalModel}
|
||||
providerCapabilities={activeProviderCapabilities}
|
||||
activeExternalProvider={activeExternalProvider}
|
||||
onExternalProviderChange={(updatedProvider) => {
|
||||
setExternalProviders(
|
||||
externalProviders.map((provider) =>
|
||||
provider.id === updatedProvider.id ? updatedProvider : provider,
|
||||
),
|
||||
);
|
||||
}}
|
||||
externalProviderType={activeExternalProviderType}
|
||||
onReloadModel={() => {
|
||||
const state = useChatRuntimeStore.getState();
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ import { Label } from "@/components/ui/label";
|
|||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectGroup,
|
||||
SelectItem,
|
||||
SelectSeparator,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
|
|
@ -23,7 +25,6 @@ import { Spinner } from "@/components/ui/spinner";
|
|||
import { Textarea } from "@/components/ui/textarea";
|
||||
import {
|
||||
ArrowLeft02Icon,
|
||||
DashboardSquare01Icon,
|
||||
Delete02Icon,
|
||||
Edit03Icon,
|
||||
PlusSignIcon,
|
||||
|
|
@ -47,9 +48,19 @@ import {
|
|||
} from "./api/providers-api";
|
||||
import type { ExternalProviderConfig } from "./external-providers";
|
||||
import {
|
||||
CUSTOM_BACKEND_PROVIDER_TYPE,
|
||||
CUSTOM_PROVIDER_PRESETS,
|
||||
customProviderBaseUrlPlaceholder,
|
||||
customProviderDisplayName,
|
||||
customProviderModelIdsPlaceholder,
|
||||
getExternalProviderApiKey,
|
||||
isCustomProviderType,
|
||||
LEGACY_CUSTOM_PROVIDER_TYPE,
|
||||
removeExternalProviderApiKey,
|
||||
setExternalProviderApiKey,
|
||||
supportsProviderPromptCaching,
|
||||
supportsProviderReasoningToggle,
|
||||
toExternalBackendProviderType,
|
||||
} from "./external-providers";
|
||||
|
||||
/** Matches navbar / thread layout easing (see index.css --ease-out-quart) */
|
||||
|
|
@ -57,12 +68,11 @@ const PROVIDER_FORM_EASE: [number, number, number, number] = [
|
|||
0.165, 0.84, 0.44, 1,
|
||||
];
|
||||
const PROVIDER_FORM_DURATION = 0.2;
|
||||
const CUSTOM_PROVIDER_TYPE = "custom";
|
||||
const CUSTOM_BACKEND_PROVIDER_TYPE = "openai";
|
||||
const CUSTOM_PROVIDER_MISSING_KEY_MESSAGE =
|
||||
"No API key found, please make sure API key is added and valid for this provider.";
|
||||
const ANTHROPIC_DATED_SNAPSHOT_SUFFIX = /-\d{8}$/;
|
||||
const OPENAI_DEPRECATED_MODELS = new Set(["gpt-5.3"]);
|
||||
const HIDDEN_PROVIDER_TYPES = new Set(["qwen"]);
|
||||
const OPENROUTER_EXCLUDED_MODELS = new Set([
|
||||
"google/chirp-3",
|
||||
"kwaivgi/kling-v3.0-pro",
|
||||
|
|
@ -82,37 +92,37 @@ function resolveUiProviderTypeFromConfig(
|
|||
registryRows: ProviderRegistryEntry[],
|
||||
existingProviderType: string | undefined,
|
||||
): string {
|
||||
if (existingProviderType === CUSTOM_PROVIDER_TYPE) {
|
||||
return CUSTOM_PROVIDER_TYPE;
|
||||
if (existingProviderType && isCustomProviderType(existingProviderType)) {
|
||||
return existingProviderType;
|
||||
}
|
||||
if (configProviderType !== CUSTOM_BACKEND_PROVIDER_TYPE) {
|
||||
return configProviderType;
|
||||
}
|
||||
const displayName = (configDisplayName ?? "").trim().toLowerCase();
|
||||
const matchingCustomPreset = CUSTOM_PROVIDER_PRESETS.find(
|
||||
(preset) => preset.displayName.toLowerCase() === displayName,
|
||||
);
|
||||
if (matchingCustomPreset) {
|
||||
return matchingCustomPreset.providerType;
|
||||
}
|
||||
const openAiRegistry = registryRows.find(
|
||||
(entry) => entry.provider_type === CUSTOM_BACKEND_PROVIDER_TYPE,
|
||||
);
|
||||
if (!openAiRegistry) {
|
||||
return configProviderType;
|
||||
}
|
||||
const displayName = (configDisplayName ?? "").trim().toLowerCase();
|
||||
const openAiDisplayName = openAiRegistry.display_name.trim().toLowerCase();
|
||||
if (displayName.length > 0 && displayName !== openAiDisplayName) {
|
||||
return CUSTOM_PROVIDER_TYPE;
|
||||
return LEGACY_CUSTOM_PROVIDER_TYPE;
|
||||
}
|
||||
const configUrl = normalizeUrl(configBaseUrl ?? "");
|
||||
const defaultUrl = normalizeUrl(openAiRegistry.base_url ?? "");
|
||||
if (configUrl.length > 0 && configUrl !== defaultUrl) {
|
||||
return CUSTOM_PROVIDER_TYPE;
|
||||
return LEGACY_CUSTOM_PROVIDER_TYPE;
|
||||
}
|
||||
return configProviderType;
|
||||
}
|
||||
|
||||
function toBackendProviderType(uiProviderType: string): string {
|
||||
return uiProviderType === CUSTOM_PROVIDER_TYPE
|
||||
? CUSTOM_BACKEND_PROVIDER_TYPE
|
||||
: uiProviderType;
|
||||
}
|
||||
|
||||
function parseManualModelIds(text: string): string[] {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
|
|
@ -175,22 +185,22 @@ export function ChatProvidersSettings({
|
|||
const [manualModelIds, setManualModelIds] = useState("");
|
||||
const [modelSearchQuery, setModelSearchQuery] = useState("");
|
||||
const [customProviderName, setCustomProviderName] = useState("Custom");
|
||||
const [isReasoningModel, setIsReasoningModel] = useState(false);
|
||||
const reduceMotion = useReducedMotion();
|
||||
const isCustomProvider = providerType === CUSTOM_PROVIDER_TYPE;
|
||||
const isCustomProvider = isCustomProviderType(providerType);
|
||||
const showReasoningToggle = supportsProviderReasoningToggle(providerType);
|
||||
|
||||
const registryByType = useMemo(
|
||||
() => new Map(registry.map((entry) => [entry.provider_type, entry])),
|
||||
[registry],
|
||||
);
|
||||
const hasCustomInRegistry = registryByType.has(CUSTOM_PROVIDER_TYPE);
|
||||
|
||||
const isCuratedModelList = useMemo(() => {
|
||||
return registryByType.get(providerType)?.model_list_mode === "curated";
|
||||
}, [registryByType, providerType]);
|
||||
const isManualModelList = isCustomProvider || isCuratedModelList;
|
||||
|
||||
const modelsPanelKey = isCustomProvider
|
||||
? "custom"
|
||||
? providerType || "custom"
|
||||
: isCuratedModelList
|
||||
? "curated"
|
||||
: "remote";
|
||||
|
|
@ -225,7 +235,12 @@ export function ChatProvidersSettings({
|
|||
useEffect(() => {
|
||||
if (!providerType || editingProviderId) return;
|
||||
const entry = registryByType.get(providerType);
|
||||
if (!entry) return;
|
||||
if (!entry) {
|
||||
if (isCustomProviderType(providerType)) {
|
||||
setCustomProviderName(customProviderDisplayName(providerType));
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Seed the registry's default_models for every provider — curated and
|
||||
// remote alike. For remote-mode providers, loadModels() will replace
|
||||
// this with the union of defaults + the live /models response once the
|
||||
|
|
@ -297,6 +312,12 @@ export function ChatProvidersSettings({
|
|||
baseUrl: config.base_url ?? "",
|
||||
models: existingModels,
|
||||
availableModels: existing?.availableModels ?? [],
|
||||
enablePromptCaching: supportsProviderPromptCaching(uiProviderType)
|
||||
? (existing?.enablePromptCaching ?? true)
|
||||
: undefined,
|
||||
isReasoningModel: supportsProviderReasoningToggle(uiProviderType)
|
||||
? existing?.isReasoningModel === true
|
||||
: undefined,
|
||||
createdAt: existing?.createdAt ?? createdAt,
|
||||
updatedAt,
|
||||
};
|
||||
|
|
@ -328,7 +349,8 @@ export function ChatProvidersSettings({
|
|||
setSelectedModelIds([]);
|
||||
setManualModelIds("");
|
||||
setModelSearchQuery("");
|
||||
setCustomProviderName("Custom");
|
||||
setCustomProviderName(customProviderDisplayName(providerType));
|
||||
setIsReasoningModel(false);
|
||||
}
|
||||
|
||||
function openAddProvider() {
|
||||
|
|
@ -384,7 +406,7 @@ export function ChatProvidersSettings({
|
|||
const trimmed = input.trim();
|
||||
if (!trimmed) {
|
||||
if (required) {
|
||||
throw new Error("Base URL is required for custom providers.");
|
||||
throw new Error("Base URL is required for this connection.");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -397,7 +419,7 @@ export function ChatProvidersSettings({
|
|||
return;
|
||||
}
|
||||
if (isCustomProvider) {
|
||||
toast.info("Custom providers use manual model IDs.");
|
||||
toast.info("This connection uses manual model IDs.");
|
||||
return;
|
||||
}
|
||||
if (isCuratedModelList) {
|
||||
|
|
@ -458,10 +480,10 @@ export function ChatProvidersSettings({
|
|||
toast.error("Choose a provider first.");
|
||||
return;
|
||||
}
|
||||
const backendProviderType = toBackendProviderType(providerType);
|
||||
const backendProviderType = toExternalBackendProviderType(providerType);
|
||||
const selectedRegistryEntry = registryByType.get(backendProviderType);
|
||||
const displayName = isCustomProvider
|
||||
? customProviderName.trim() || "Custom"
|
||||
? customProviderName.trim() || customProviderDisplayName(providerType)
|
||||
: (selectedRegistryEntry?.display_name ?? providerType);
|
||||
if (!isCustomProvider && !apiKey.trim()) {
|
||||
toast.error("API key is required.");
|
||||
|
|
@ -511,17 +533,21 @@ export function ChatProvidersSettings({
|
|||
const updatedAt = Number.isFinite(Date.parse(created.updated_at))
|
||||
? Date.parse(created.updated_at)
|
||||
: Date.now();
|
||||
const uiProviderType = isCustomProvider
|
||||
? providerType
|
||||
: created.provider_type;
|
||||
const provider: ExternalProviderConfig = {
|
||||
id: created.id,
|
||||
providerType: isCustomProvider
|
||||
? CUSTOM_PROVIDER_TYPE
|
||||
: created.provider_type,
|
||||
providerType: uiProviderType,
|
||||
name: created.display_name,
|
||||
baseUrl: created.base_url ?? "",
|
||||
models: modelsToSave,
|
||||
availableModels: manualModels
|
||||
? []
|
||||
: pruneProviderModelIds(providerType, availableModels),
|
||||
isReasoningModel: supportsProviderReasoningToggle(uiProviderType)
|
||||
? isReasoningModel
|
||||
: undefined,
|
||||
createdAt,
|
||||
updatedAt,
|
||||
};
|
||||
|
|
@ -553,7 +579,7 @@ export function ChatProvidersSettings({
|
|||
return;
|
||||
}
|
||||
const isEditingCustomProvider =
|
||||
existing.providerType === CUSTOM_PROVIDER_TYPE;
|
||||
isCustomProviderType(existing.providerType);
|
||||
if (!isEditingCustomProvider && !apiKey.trim()) {
|
||||
toast.error("API key is required.");
|
||||
return;
|
||||
|
|
@ -597,7 +623,8 @@ export function ChatProvidersSettings({
|
|||
);
|
||||
const updated = await updateProviderConfig(editingProviderId, {
|
||||
displayName: isEditingCustomProvider
|
||||
? customProviderName.trim() || "Custom"
|
||||
? customProviderName.trim() ||
|
||||
customProviderDisplayName(existing.providerType)
|
||||
: existing.name,
|
||||
baseUrl,
|
||||
});
|
||||
|
|
@ -620,6 +647,11 @@ export function ChatProvidersSettings({
|
|||
availableModels: manualModels
|
||||
? []
|
||||
: pruneProviderModelIds(existing.providerType, availableModels),
|
||||
isReasoningModel: supportsProviderReasoningToggle(
|
||||
existing.providerType,
|
||||
)
|
||||
? isReasoningModel
|
||||
: undefined,
|
||||
updatedAt,
|
||||
}
|
||||
: provider,
|
||||
|
|
@ -640,12 +672,19 @@ export function ChatProvidersSettings({
|
|||
setEditingProviderId(provider.id);
|
||||
setPage("form");
|
||||
setProviderType(provider.providerType);
|
||||
setCustomProviderName(provider.name || "Custom");
|
||||
setCustomProviderName(
|
||||
provider.name || customProviderDisplayName(provider.providerType),
|
||||
);
|
||||
setApiKey(getExternalProviderApiKey(provider.id));
|
||||
setShowApiKey(false);
|
||||
setBaseUrlDraft(provider.baseUrl);
|
||||
setModelSearchQuery("");
|
||||
if (provider.providerType === CUSTOM_PROVIDER_TYPE) {
|
||||
setIsReasoningModel(
|
||||
supportsProviderReasoningToggle(provider.providerType)
|
||||
? provider.isReasoningModel === true
|
||||
: false,
|
||||
);
|
||||
if (isCustomProviderType(provider.providerType)) {
|
||||
setAvailableModels([]);
|
||||
setSelectedModelIds([]);
|
||||
setManualModelIds(provider.models.join("\n"));
|
||||
|
|
@ -696,7 +735,7 @@ export function ChatProvidersSettings({
|
|||
async function testProvider(provider: ExternalProviderConfig) {
|
||||
const savedKey = getExternalProviderApiKey(provider.id).trim();
|
||||
if (!savedKey) {
|
||||
if (provider.providerType === CUSTOM_PROVIDER_TYPE) {
|
||||
if (isCustomProviderType(provider.providerType)) {
|
||||
await editProvider(provider);
|
||||
toast.info(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE);
|
||||
return;
|
||||
|
|
@ -707,7 +746,9 @@ export function ChatProvidersSettings({
|
|||
}
|
||||
try {
|
||||
const result = await testProviderConnection({
|
||||
providerType: toBackendProviderType(provider.providerType),
|
||||
providerType:
|
||||
toExternalBackendProviderType(provider.providerType) ??
|
||||
provider.providerType,
|
||||
apiKey: savedKey,
|
||||
baseUrl: provider.baseUrl || null,
|
||||
});
|
||||
|
|
@ -715,7 +756,7 @@ export function ChatProvidersSettings({
|
|||
toast.success(result.message);
|
||||
} else {
|
||||
if (
|
||||
provider.providerType === CUSTOM_PROVIDER_TYPE &&
|
||||
isCustomProviderType(provider.providerType) &&
|
||||
result.message.includes("Illegal header value b'Bearer '")
|
||||
) {
|
||||
toast.error(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE);
|
||||
|
|
@ -726,7 +767,7 @@ export function ChatProvidersSettings({
|
|||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
if (
|
||||
provider.providerType === CUSTOM_PROVIDER_TYPE &&
|
||||
isCustomProviderType(provider.providerType) &&
|
||||
message.includes("Illegal header value b'Bearer '")
|
||||
) {
|
||||
toast.error(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE);
|
||||
|
|
@ -753,7 +794,7 @@ export function ChatProvidersSettings({
|
|||
</Button>
|
||||
<div className="flex min-w-0 items-center gap-2 leading-none">
|
||||
<span className="text-xs font-medium text-muted-foreground">
|
||||
Cloud
|
||||
Connections
|
||||
</span>
|
||||
<span className="size-1 rounded-full bg-muted-foreground/35" />
|
||||
<span className="truncate text-xs font-medium text-muted-foreground">
|
||||
|
|
@ -774,7 +815,7 @@ export function ChatProvidersSettings({
|
|||
Provider
|
||||
</Label>
|
||||
<p className="text-xs leading-snug text-muted-foreground">
|
||||
Supported registry or Custom.
|
||||
Supported registry or local OpenAI-compatible connection.
|
||||
</p>
|
||||
</div>
|
||||
<Select
|
||||
|
|
@ -786,6 +827,9 @@ export function ChatProvidersSettings({
|
|||
setSelectedModelIds([]);
|
||||
setManualModelIds("");
|
||||
setModelSearchQuery("");
|
||||
if (isCustomProviderType(value)) {
|
||||
setCustomProviderName(customProviderDisplayName(value));
|
||||
}
|
||||
}}
|
||||
>
|
||||
<SelectTrigger
|
||||
|
|
@ -796,32 +840,46 @@ export function ChatProvidersSettings({
|
|||
<SelectValue placeholder="Choose a provider" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{registry.map((entry) => (
|
||||
<SelectItem
|
||||
key={entry.provider_type}
|
||||
value={entry.provider_type}
|
||||
>
|
||||
<span className="flex items-center gap-2">
|
||||
<ApiProviderLogo
|
||||
providerType={entry.provider_type}
|
||||
className="size-4"
|
||||
title={entry.display_name}
|
||||
/>
|
||||
{entry.display_name}
|
||||
</span>
|
||||
</SelectItem>
|
||||
))}
|
||||
{hasCustomInRegistry ? null : (
|
||||
<SelectItem value={CUSTOM_PROVIDER_TYPE}>
|
||||
<span className="flex items-center gap-2">
|
||||
<HugeiconsIcon
|
||||
icon={DashboardSquare01Icon}
|
||||
className="size-4"
|
||||
/>
|
||||
Custom
|
||||
</span>
|
||||
</SelectItem>
|
||||
)}
|
||||
<SelectGroup>
|
||||
{CUSTOM_PROVIDER_PRESETS.map((preset) => (
|
||||
<SelectItem
|
||||
key={preset.providerType}
|
||||
value={preset.providerType}
|
||||
>
|
||||
<span className="flex items-center gap-2">
|
||||
<ApiProviderLogo
|
||||
providerType={preset.providerType}
|
||||
className="size-4"
|
||||
title={preset.displayName}
|
||||
/>
|
||||
{preset.displayName}
|
||||
</span>
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectGroup>
|
||||
<SelectSeparator />
|
||||
<SelectGroup>
|
||||
{registry
|
||||
.filter(
|
||||
(entry) =>
|
||||
!HIDDEN_PROVIDER_TYPES.has(entry.provider_type),
|
||||
)
|
||||
.map((entry) => (
|
||||
<SelectItem
|
||||
key={entry.provider_type}
|
||||
value={entry.provider_type}
|
||||
>
|
||||
<span className="flex items-center gap-2">
|
||||
<ApiProviderLogo
|
||||
providerType={entry.provider_type}
|
||||
className="size-4"
|
||||
title={entry.display_name}
|
||||
/>
|
||||
{entry.display_name}
|
||||
</span>
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
|
@ -902,11 +960,35 @@ export function ChatProvidersSettings({
|
|||
type="text"
|
||||
value={baseUrlDraft}
|
||||
onChange={(event) => setBaseUrlDraft(event.target.value)}
|
||||
placeholder="https://my-vllm-server.com/v1"
|
||||
placeholder={customProviderBaseUrlPlaceholder(providerType)}
|
||||
className="h-9 text-sm"
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{showReasoningToggle ? (
|
||||
<div className="grid grid-cols-[minmax(150px,0.8fr)_minmax(260px,1.2fr)] items-center gap-4 px-4 py-3 max-sm:grid-cols-1">
|
||||
<Label
|
||||
htmlFor="provider-is-reasoning"
|
||||
className="text-sm font-medium"
|
||||
>
|
||||
Reasoning model
|
||||
</Label>
|
||||
<label
|
||||
htmlFor="provider-is-reasoning"
|
||||
className="flex cursor-pointer items-center gap-2 text-sm"
|
||||
>
|
||||
<Checkbox
|
||||
id="provider-is-reasoning"
|
||||
checked={isReasoningModel}
|
||||
onCheckedChange={(checked) =>
|
||||
setIsReasoningModel(checked === true)
|
||||
}
|
||||
/>
|
||||
This server runs a reasoning model
|
||||
</label>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -952,7 +1034,7 @@ export function ChatProvidersSettings({
|
|||
}
|
||||
title={
|
||||
isCustomProvider
|
||||
? "Custom providers use manual model IDs"
|
||||
? "This connection uses manual model IDs"
|
||||
: isCuratedModelList
|
||||
? "Full catalog is not fetched for this provider"
|
||||
: undefined
|
||||
|
|
@ -986,7 +1068,7 @@ export function ChatProvidersSettings({
|
|||
onChange={(event) =>
|
||||
setManualModelIds(event.target.value)
|
||||
}
|
||||
placeholder={"gpt-4o-mini\nQwen/Qwen3-14B"}
|
||||
placeholder={customProviderModelIdsPlaceholder(providerType)}
|
||||
rows={5}
|
||||
className="min-h-[100px] resize-y font-mono text-sm"
|
||||
/>
|
||||
|
|
@ -1197,9 +1279,9 @@ export function ChatProvidersSettings({
|
|||
<div className="flex min-h-0 flex-col gap-6">
|
||||
<header className="flex flex-col gap-1 pr-8">
|
||||
<div className="flex min-w-0 flex-col gap-1">
|
||||
<h1 className="font-heading text-lg font-semibold">Cloud</h1>
|
||||
<h1 className="font-heading text-lg font-semibold">Connections</h1>
|
||||
<p className="text-xs leading-relaxed text-muted-foreground">
|
||||
Manage cloud provider connections for chat through the Studio proxy.
|
||||
Manage model provider connections for chat through the Studio proxy.
|
||||
</p>
|
||||
</div>
|
||||
</header>
|
||||
|
|
@ -1237,7 +1319,8 @@ export function ChatProvidersSettings({
|
|||
const detail =
|
||||
provider.baseUrl || registryEntry?.base_url || "";
|
||||
const providerLabel =
|
||||
registryEntry?.display_name ?? provider.providerType;
|
||||
registryEntry?.display_name ??
|
||||
customProviderDisplayName(provider.providerType);
|
||||
const modelSummary = formatModelSummary(provider.models);
|
||||
return (
|
||||
<div
|
||||
|
|
@ -1346,9 +1429,9 @@ export function ChatProvidersDialog({
|
|||
className="flex max-h-[90dvh] w-[96vw] flex-col gap-0 overflow-y-auto p-8 sm:max-w-none md:max-w-[44rem]"
|
||||
>
|
||||
<DialogHeader className="sr-only">
|
||||
<DialogTitle>Cloud</DialogTitle>
|
||||
<DialogTitle>Connections</DialogTitle>
|
||||
<DialogDescription>
|
||||
Manage external model providers for chat.
|
||||
Manage external model connections for chat.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<ChatProvidersSettings
|
||||
|
|
|
|||
|
|
@ -64,6 +64,10 @@ import { Fragment, type ReactNode } from "react";
|
|||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
|
||||
import {
|
||||
type ExternalProviderConfig,
|
||||
supportsProviderPromptCaching,
|
||||
} from "./external-providers";
|
||||
import {
|
||||
applyPresetParams,
|
||||
BUILTIN_PRESET_NAMES,
|
||||
|
|
@ -517,6 +521,8 @@ interface ChatSettingsPanelProps {
|
|||
* per-param visibility in the sampling section.
|
||||
*/
|
||||
providerCapabilities?: ProviderCapabilities | null;
|
||||
activeExternalProvider?: ExternalProviderConfig | null;
|
||||
onExternalProviderChange?: (provider: ExternalProviderConfig) => void;
|
||||
/**
|
||||
* Backend provider type for the active external model (e.g. "kimi",
|
||||
* "anthropic", "openai"), or `null` for local models. Drives the
|
||||
|
|
@ -533,6 +539,8 @@ export function ChatSettingsPanel({
|
|||
onParamsChange,
|
||||
isExternalModel = false,
|
||||
providerCapabilities = null,
|
||||
activeExternalProvider = null,
|
||||
onExternalProviderChange,
|
||||
externalProviderType = null,
|
||||
onReloadModel,
|
||||
}: ChatSettingsPanelProps) {
|
||||
|
|
@ -662,6 +670,11 @@ export function ChatSettingsPanel({
|
|||
Boolean(currentCheckpoint) &&
|
||||
modelRequiresTrustRemoteCode &&
|
||||
!(params.trustRemoteCode ?? false);
|
||||
const showPromptCachingControl =
|
||||
activeExternalProvider != null &&
|
||||
supportsProviderPromptCaching(activeExternalProvider.providerType);
|
||||
const promptCachingEnabled =
|
||||
activeExternalProvider?.enablePromptCaching !== false;
|
||||
|
||||
function set<K extends keyof InferenceParams>(key: K) {
|
||||
return (v: InferenceParams[K]) => {
|
||||
|
|
@ -1145,6 +1158,32 @@ export function ChatSettingsPanel({
|
|||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
{showPromptCachingControl && activeExternalProvider ? (
|
||||
<CollapsibleSection label="Provider" defaultOpen={true}>
|
||||
<div className="flex items-center justify-between gap-3 pt-1">
|
||||
<div className="flex min-w-0 items-center gap-1.5">
|
||||
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
|
||||
Prompt caching
|
||||
</span>
|
||||
<InfoHint>
|
||||
Reuse compatible prompt prefixes for lower latency and cost.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Switch
|
||||
className="panel-switch shrink-0"
|
||||
checked={promptCachingEnabled}
|
||||
onCheckedChange={(checked) => {
|
||||
onExternalProviderChange?.({
|
||||
...activeExternalProvider,
|
||||
enablePromptCaching: checked,
|
||||
});
|
||||
}}
|
||||
aria-label="Enable prompt caching"
|
||||
/>
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
) : null}
|
||||
|
||||
<CollapsibleSection label="System Prompt" defaultOpen={true}>
|
||||
<button
|
||||
type="button"
|
||||
|
|
|
|||
|
|
@ -14,10 +14,147 @@ export interface ExternalProviderConfig {
|
|||
models: string[];
|
||||
/** Cached available model ids from the provider's /models response. */
|
||||
availableModels?: string[];
|
||||
/** Whether to ask supported hosted providers to use prompt caching. */
|
||||
enablePromptCaching?: boolean;
|
||||
/** User-pinned: the loaded vLLM model supports `enable_thinking`. */
|
||||
isReasoningModel?: boolean;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
const PROMPT_CACHING_PROVIDER_TYPES = new Set(["openai", "anthropic"]);
|
||||
|
||||
export function supportsProviderPromptCaching(
|
||||
providerType: string | null | undefined,
|
||||
): boolean {
|
||||
return providerType != null && PROMPT_CACHING_PROVIDER_TYPES.has(providerType);
|
||||
}
|
||||
|
||||
// Provider types that expose the connection-level "reasoning model"
|
||||
// toggle. vLLM's OpenAI-compat endpoint doesn't advertise this per model.
|
||||
const REASONING_TOGGLE_PROVIDER_TYPES = new Set(["vllm"]);
|
||||
|
||||
export function supportsProviderReasoningToggle(
|
||||
providerType: string | null | undefined,
|
||||
): boolean {
|
||||
return (
|
||||
providerType != null && REASONING_TOGGLE_PROVIDER_TYPES.has(providerType)
|
||||
);
|
||||
}
|
||||
|
||||
export const CUSTOM_BACKEND_PROVIDER_TYPE = "openai";
|
||||
export const LEGACY_CUSTOM_PROVIDER_TYPE = "custom";
|
||||
|
||||
export const CUSTOM_PROVIDER_PRESETS = [
|
||||
{
|
||||
providerType: "llama_cpp",
|
||||
displayName: "llama.cpp",
|
||||
baseUrlPlaceholder: "http://localhost:8080/v1",
|
||||
modelIdsPlaceholder: "gpt-oss-20b\nqwen3-14b",
|
||||
},
|
||||
{
|
||||
providerType: "vllm",
|
||||
displayName: "vLLM",
|
||||
baseUrlPlaceholder: "https://my-vllm-server.com/v1",
|
||||
modelIdsPlaceholder: "openai/gpt-oss-20b\nQwen/Qwen3-14B",
|
||||
},
|
||||
{
|
||||
providerType: "ollama",
|
||||
displayName: "Ollama",
|
||||
baseUrlPlaceholder: "http://localhost:11434/v1",
|
||||
modelIdsPlaceholder: "gpt-oss:20b\nqwen3:14b",
|
||||
},
|
||||
] as const;
|
||||
|
||||
const CUSTOM_PROVIDER_LABELS: Record<string, string> = {
|
||||
[LEGACY_CUSTOM_PROVIDER_TYPE]: "Custom",
|
||||
...Object.fromEntries(
|
||||
CUSTOM_PROVIDER_PRESETS.map((preset) => [
|
||||
preset.providerType,
|
||||
preset.displayName,
|
||||
]),
|
||||
),
|
||||
};
|
||||
|
||||
const CUSTOM_PROVIDER_BASE_URL_PLACEHOLDERS: Record<string, string> = {
|
||||
[LEGACY_CUSTOM_PROVIDER_TYPE]: "https://my-vllm-server.com/v1",
|
||||
...Object.fromEntries(
|
||||
CUSTOM_PROVIDER_PRESETS.map((preset) => [
|
||||
preset.providerType,
|
||||
preset.baseUrlPlaceholder,
|
||||
]),
|
||||
),
|
||||
};
|
||||
|
||||
const CUSTOM_PROVIDER_MODEL_IDS_PLACEHOLDERS: Record<string, string> = {
|
||||
[LEGACY_CUSTOM_PROVIDER_TYPE]: "openai/gpt-oss-20b\nQwen/Qwen3-14B",
|
||||
...Object.fromEntries(
|
||||
CUSTOM_PROVIDER_PRESETS.map((preset) => [
|
||||
preset.providerType,
|
||||
preset.modelIdsPlaceholder,
|
||||
]),
|
||||
),
|
||||
};
|
||||
|
||||
export function isCustomProviderType(
|
||||
providerType: string | null | undefined,
|
||||
): boolean {
|
||||
if (!providerType) return false;
|
||||
return providerType in CUSTOM_PROVIDER_LABELS;
|
||||
}
|
||||
|
||||
export function customProviderDisplayName(
|
||||
providerType: string | null | undefined,
|
||||
): string {
|
||||
if (!providerType) return "Custom";
|
||||
return CUSTOM_PROVIDER_LABELS[providerType] ?? providerType;
|
||||
}
|
||||
|
||||
export function customProviderBaseUrlPlaceholder(
|
||||
providerType: string | null | undefined,
|
||||
): string {
|
||||
if (!providerType) {
|
||||
return CUSTOM_PROVIDER_BASE_URL_PLACEHOLDERS[LEGACY_CUSTOM_PROVIDER_TYPE];
|
||||
}
|
||||
return (
|
||||
CUSTOM_PROVIDER_BASE_URL_PLACEHOLDERS[providerType] ??
|
||||
CUSTOM_PROVIDER_BASE_URL_PLACEHOLDERS[LEGACY_CUSTOM_PROVIDER_TYPE]
|
||||
);
|
||||
}
|
||||
|
||||
export function customProviderModelIdsPlaceholder(
|
||||
providerType: string | null | undefined,
|
||||
): string {
|
||||
if (!providerType) {
|
||||
return CUSTOM_PROVIDER_MODEL_IDS_PLACEHOLDERS[LEGACY_CUSTOM_PROVIDER_TYPE];
|
||||
}
|
||||
return (
|
||||
CUSTOM_PROVIDER_MODEL_IDS_PLACEHOLDERS[providerType] ??
|
||||
CUSTOM_PROVIDER_MODEL_IDS_PLACEHOLDERS[LEGACY_CUSTOM_PROVIDER_TYPE]
|
||||
);
|
||||
}
|
||||
|
||||
export function toExternalBackendProviderType(providerType: string): string;
|
||||
export function toExternalBackendProviderType(
|
||||
providerType: null | undefined,
|
||||
): undefined;
|
||||
export function toExternalBackendProviderType(
|
||||
providerType: string | null | undefined,
|
||||
): string | undefined;
|
||||
export function toExternalBackendProviderType(
|
||||
providerType: string | null | undefined,
|
||||
): string | undefined {
|
||||
if (!providerType) return undefined;
|
||||
// vLLM's /v1/responses applies the loaded model's chat template, which
|
||||
// 400s on strict-alternation templates (e.g. Gemma 3). Pass the actual
|
||||
// type through so the backend routes vLLM to /v1/chat/completions instead
|
||||
// of the OpenAI Responses path used for gpt-5.x.
|
||||
if (providerType === "vllm") return "vllm";
|
||||
return isCustomProviderType(providerType)
|
||||
? CUSTOM_BACKEND_PROVIDER_TYPE
|
||||
: providerType;
|
||||
}
|
||||
|
||||
const EXTERNAL_PROVIDERS_KEY = "unsloth_chat_external_providers";
|
||||
const EXTERNAL_PROVIDER_KEYS_KEY = "unsloth_chat_external_provider_keys";
|
||||
const EXTERNAL_MODEL_PREFIX = "external::";
|
||||
|
|
@ -71,9 +208,10 @@ function mapLegacyPresetToProviderType(presetId: string): string {
|
|||
}
|
||||
|
||||
function normalizeProvider(raw: ExternalProviderConfig): ExternalProviderConfig {
|
||||
const providerType = raw.providerType.trim();
|
||||
return {
|
||||
...raw,
|
||||
providerType: raw.providerType.trim(),
|
||||
providerType,
|
||||
name: raw.name.trim(),
|
||||
baseUrl: raw.baseUrl.trim(),
|
||||
models: raw.models
|
||||
|
|
@ -82,6 +220,12 @@ function normalizeProvider(raw: ExternalProviderConfig): ExternalProviderConfig
|
|||
availableModels: (raw.availableModels ?? [])
|
||||
.map((model) => model.trim())
|
||||
.filter((model) => model.length > 0),
|
||||
enablePromptCaching: supportsProviderPromptCaching(providerType)
|
||||
? raw.enablePromptCaching !== false
|
||||
: undefined,
|
||||
isReasoningModel: supportsProviderReasoningToggle(providerType)
|
||||
? raw.isReasoningModel === true
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -221,10 +221,13 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
// OpenRouter silently drops params the target model does not support, so we
|
||||
// surface every knob and let the gateway handle the per-model fan-out.
|
||||
openrouter: ALL_SUPPORTED,
|
||||
// Custom providers are assumed OpenAI-compatible by the backend; users who
|
||||
// point at vLLM/Ollama backends often want top_k / min_p / repetition,
|
||||
// so be permissive.
|
||||
// Local OpenAI-compatible connections are proxied through the OpenAI backend
|
||||
// path, but vLLM/Ollama/llama.cpp users often want top_k / min_p /
|
||||
// repetition controls, so be permissive.
|
||||
custom: ALL_SUPPORTED,
|
||||
vllm: ALL_SUPPORTED,
|
||||
ollama: ALL_SUPPORTED,
|
||||
llama_cpp: ALL_SUPPORTED,
|
||||
};
|
||||
|
||||
const DEFAULT_EXTERNAL_CAPABILITIES = OPENAI_COMPAT_BASE;
|
||||
|
|
@ -420,6 +423,25 @@ function resolveMistralReasoningCapabilities(modelId: string): ExternalReasoning
|
|||
return withEnableThinkingStyle();
|
||||
}
|
||||
|
||||
export interface ExternalReasoningResolveOptions {
|
||||
/** vLLM connection flagged as a reasoning model in provider config. */
|
||||
isReasoningProvider?: boolean;
|
||||
}
|
||||
|
||||
// vLLM has no per-model reasoning signal on OpenAI-compat — pin via user toggle.
|
||||
function resolveConnectionLevelReasoning(
|
||||
normalizedProvider: string,
|
||||
options: ExternalReasoningResolveOptions | undefined,
|
||||
): ExternalReasoningCapabilities | null {
|
||||
if (normalizedProvider === "vllm" && options?.isReasoningProvider) {
|
||||
return withEnableThinkingStyle({
|
||||
supportsReasoning: true,
|
||||
supportsReasoningOff: true,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* resolve external-model thinking capabilities.
|
||||
* provider-specific matching lives in the OpenAI/Anthropic resolvers.
|
||||
|
|
@ -428,9 +450,17 @@ function resolveMistralReasoningCapabilities(modelId: string): ExternalReasoning
|
|||
export function getExternalReasoningCapabilities(
|
||||
providerType: string | null | undefined,
|
||||
modelId: string | null | undefined,
|
||||
options?: ExternalReasoningResolveOptions,
|
||||
): ExternalReasoningCapabilities {
|
||||
const normalizedModel = modelId?.trim().toLowerCase() ?? "";
|
||||
const normalizedProvider = providerType?.trim().toLowerCase() ?? "";
|
||||
const connectionLevel = resolveConnectionLevelReasoning(
|
||||
normalizedProvider,
|
||||
options,
|
||||
);
|
||||
if (connectionLevel) {
|
||||
return connectionLevel;
|
||||
}
|
||||
if (!normalizedModel) {
|
||||
return withEnableThinkingStyle();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -330,6 +330,10 @@ export function SharedComposer({
|
|||
? getExternalReasoningCapabilities(
|
||||
selectedExternalProvider?.providerType,
|
||||
effectiveExternalModelId,
|
||||
{
|
||||
isReasoningProvider:
|
||||
selectedExternalProvider?.isReasoningModel === true,
|
||||
},
|
||||
)
|
||||
: null;
|
||||
const isExternalOpenAIReasoning =
|
||||
|
|
|
|||
|
|
@ -224,6 +224,7 @@ export interface OpenAIChatCompletionsRequest {
|
|||
external_model?: string;
|
||||
encrypted_api_key?: string;
|
||||
provider_base_url?: string | null;
|
||||
enable_prompt_caching?: boolean | null;
|
||||
}
|
||||
|
||||
export interface OpenAIChatDelta {
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ const TABS: TabDef[] = [
|
|||
{ id: "profile", label: "Profile", icon: UserIcon },
|
||||
{ id: "appearance", label: "Appearance", icon: PaintBrush02Icon },
|
||||
{ id: "chat", label: "Chat", icon: Message01Icon },
|
||||
{ id: "connections", label: "Cloud", icon: CloudIcon, badge: "New" },
|
||||
{ id: "connections", label: "Connections", icon: CloudIcon, badge: "New" },
|
||||
{ id: "api-keys", label: "API", icon: Globe02Icon, badge: "New" },
|
||||
{ id: "about", label: "Help", icon: HelpCircleIcon },
|
||||
];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue