From 563fcf89526cde9bb1254c385839e15f01cdf613 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 23 Apr 2026 06:57:36 -0700 Subject: [PATCH] Studio: detect reasoning_effort and preserve_thinking in chat templates (#5149) * Studio: detect reasoning_effort and preserve_thinking in chat templates Previously Studio's chat template sniffer only recognized Qwen's enable_thinking and DeepSeek's thinking markers. For gpt-oss (Harmony templates) and newer Qwen3.6 templates, the Think toggle was hidden or could only be flipped on/off. This change adds two new detections and corresponding UI controls: 1. reasoning_effort style (gpt-oss). When the chat template contains reasoning_effort, the Think button becomes a Low / Medium / High dropdown and the backend forwards {"reasoning_effort": } in chat_template_kwargs. Load-time --chat-template-kwargs flag is also switched to the new style. 2. preserve_thinking kwarg (Qwen3.6). Independent of the reasoning toggle. When the template mentions preserve_thinking, a new Preserve Thinking on/off pill is shown next to Think. Off by default, persisted via localStorage. When on, the backend adds {"preserve_thinking": true} to chat_template_kwargs so past-turn blocks are kept in the prompt instead of being stripped. Backend helper _request_reasoning_kwargs now merges all applicable kwargs into a single chat_template_kwargs dict based on the model's detected style and template capabilities. Inputs are validated with Literal types in the Pydantic request model. Tested end to end against cached GGUFs for unsloth/gpt-oss-20b-GGUF and unsloth/Qwen3.6-35B-A3B-GGUF. Confirmed the llama-server startup --chat-template-kwargs flag and per-request JSON body carry the expected keys for all combinations. * Studio: review pass and CI format fixes for reasoning-styles PR Addresses review feedback and pre-commit CI: - Preserve Thinking pill in shared-composer now gates on modelLoaded only, matching the thread.tsx toggle. Previously the inline version disabled whenever supports_reasoning was false. - The non-GGUF already_loaded LoadResponse now emits reasoning_style (and supports_preserve_thinking=False) so a reconnecting frontend sees the correct style for an already-running gpt-oss safetensors model. - use-chat-model-runtime reconnect path now always clears reasoningEnabled for models without reasoning support instead of inheriting the previous model's state. - _reasoning_default is now reset alongside the other reasoning flags in both backend reset blocks. - supports_reasoning description updated to mention reasoning_effort alongside enable_thinking. - Ran scripts/run_ruff_format.py on the touched Python files to satisfy pre-commit.ci. * Studio: detect reasoning flags on safetensors load + share Qwen param helper Addresses bot review feedback: - Extract the chat-template substring sniffer out of _read_gguf_metadata into a module-level detect_reasoning_flags(template, model_id) helper. Also runs on the safetensors / transformers load paths: - POST /api/inference/load non-GGUF LoadResponse - already_loaded non-GGUF early return - GET /api/inference/status non-GGUF branch The gpt-oss fallback via backend._is_gpt_oss_model() is preserved so safetensors gpt-oss still surfaces reasoning controls even when no chat_template is stored on the model record. - Deduplicate the Qwen3 / Qwen3.5 / Qwen3.6 Think-toggle parameter adjustment into a single features/chat/utils/qwen-params.ts. Both the assistant-ui Think toggle (thread.tsx) and the shared composer (shared-composer.tsx) now import the same helper. The superset that applies presence_penalty=1.5 for Qwen3.5 and Qwen3.6 is now used by both sites (thread.tsx previously did not apply it). * Studio: fill missing reasoning flags on safetensors status + add always_on reset Round 4 review fixes: - routes/inference.py safetensors status response now populates reasoning_always_on and supports_tools from detect_reasoning_flags. Previously Pydantic defaulted both to False, so safetensors models with always-on templates or tool-calling templates were silently losing those flags on /api/inference/status reconnect. - routes/inference.py already_loaded safetensors branch now falls back to backend._is_gpt_oss_model() when the chat template is missing, matching the status-endpoint behaviour. - Safetensors status endpoint log_source set to "Safetensors status" so the emitted template-detection log lines are attributable. - chat-runtime-store clearCheckpoint now also resets reasoningAlwaysOn so switching from an always-on reasoning model to a non-always-on one does not leave the Think button permanently locked on. * Studio: skip reasoning kwargs when always-on; narrow non-GGUF advertisement Round 5 addresses reviewer feedback: - _request_reasoning_kwargs and the load-time --chat-template-kwargs emission now skip when _reasoning_always_on is true. Templates with hardcoded tags do not consume enable_thinking / reasoning_effort so sending them was noise. - Non-GGUF (Unsloth / transformers) LoadResponse and InferenceStatusResponse paths no longer advertise template-derived supports_reasoning / reasoning_style / supports_preserve_thinking / supports_tools. The transformers generation path does not yet forward chat_template_kwargs to tokenizer.apply_chat_template, so exposing the UI controls on those models was misleading. Only the gpt-oss Harmony case is kept (reasoning_style = reasoning_effort) because it is handled via the HarmonyTextStreamer at the tokenizer level. A follow-up PR can thread chat_template_kwargs through the transformers path and re-enable the broader detection. - GGUF / llama-server paths keep the full detect_reasoning_flags output. --- studio/backend/core/inference/llama_cpp.py | 218 +++++++++++++----- studio/backend/models/inference.py | 26 ++- studio/backend/routes/inference.py | 69 +++++- .../src/components/assistant-ui/thread.tsx | 99 +++++++- .../src/features/chat/api/chat-adapter.ts | 28 ++- .../chat/hooks/use-chat-model-runtime.ts | 13 ++ .../src/features/chat/shared-composer.tsx | 140 ++++++++--- .../chat/stores/chat-runtime-store.ts | 47 ++++ .../frontend/src/features/chat/types/api.ts | 6 + .../src/features/chat/utils/qwen-params.ts | 29 +++ 10 files changed, 566 insertions(+), 109 deletions(-) create mode 100644 studio/frontend/src/features/chat/utils/qwen-params.ts diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 6ad05ac52d..c320f03b2c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -87,6 +87,84 @@ _TC_PARAM_START_RE = re.compile(r"\s*") _TC_PARAM_CLOSE_RE = re.compile(r"\s*\s*$") +_TOOL_TEMPLATE_MARKERS = ( + "{%- if tools %}", + "{%- if tools -%}", + "{% if tools %}", + "{% if tools -%}", + '"role" == "tool"', + "'role' == 'tool'", + 'message.role == "tool"', + "message.role == 'tool'", +) + + +def detect_reasoning_flags( + chat_template: Optional[str], + model_identifier: Optional[str] = None, + *, + log_source: Optional[str] = None, +) -> dict: + """Classify a chat template's reasoning and tool-calling capabilities. + + Returns a dict with the same five keys populated by the GGUF sniffer: + ``supports_reasoning``, ``reasoning_style`` + (``"enable_thinking"`` | ``"reasoning_effort"``), + ``reasoning_always_on``, ``supports_preserve_thinking``, and + ``supports_tools``. Used by both the llama-server backend at load + time and the safetensors/transformers paths in ``routes/inference`` + so the two agree on what the frontend will see. + """ + flags = { + "supports_reasoning": False, + "reasoning_style": "enable_thinking", + "reasoning_always_on": False, + "supports_preserve_thinking": False, + "supports_tools": False, + } + if not chat_template: + return flags + tpl = chat_template + prefix = f"{log_source}: " if log_source else "" + + if "enable_thinking" in tpl: + flags["supports_reasoning"] = True + flags["reasoning_style"] = "enable_thinking" + logger.info(f"{prefix}model supports reasoning (enable_thinking)") + elif "reasoning_effort" in tpl: + # gpt-oss / Harmony templates use reasoning_effort + # ("low" | "medium" | "high") instead of a boolean. + flags["supports_reasoning"] = True + flags["reasoning_style"] = "reasoning_effort" + logger.info(f"{prefix}model supports reasoning (reasoning_effort)") + elif "thinking" in tpl: + # DeepSeek uses 'thinking' instead of 'enable_thinking' + normalized_id = (model_identifier or "").lower() + if "deepseek" in normalized_id: + flags["supports_reasoning"] = True + logger.info(f"{prefix}model supports reasoning (DeepSeek thinking)") + + # Hardcoded tags or reasoning_content in the template mean + # thinking is always on (no toggle to disable it). + if not flags["supports_reasoning"]: + if ("" in tpl and "" in tpl) or "reasoning_content" in tpl: + flags["supports_reasoning"] = True + flags["reasoning_always_on"] = True + logger.info(f"{prefix}model always reasons ( tags in template)") + + # preserve_thinking is an independent kwarg on some Qwen templates + # that keeps historical blocks in prior assistant turns. + if "preserve_thinking" in tpl: + flags["supports_preserve_thinking"] = True + logger.info(f"{prefix}model supports preserve_thinking") + + if any(marker in tpl for marker in _TOOL_TEMPLATE_MARKERS): + flags["supports_tools"] = True + logger.info(f"{prefix}model supports tool calling") + + return flags + + class LlamaCppBackend: """ Manages a llama-server subprocess for GGUF model inference. @@ -112,6 +190,8 @@ class LlamaCppBackend: self._chat_template: Optional[str] = None self._supports_reasoning: bool = False self._reasoning_always_on: bool = False + self._reasoning_style: str = "enable_thinking" + self._supports_preserve_thinking: bool = False self._supports_tools: bool = False self._cache_type_kv: Optional[str] = None self._reasoning_default: bool = True @@ -293,10 +373,51 @@ class LlamaCppBackend: def reasoning_always_on(self) -> bool: return self._reasoning_always_on + @property + def reasoning_style(self) -> str: + return self._reasoning_style + + @property + def supports_preserve_thinking(self) -> bool: + return self._supports_preserve_thinking + @property def reasoning_default(self) -> bool: return self._reasoning_default + def _reasoning_kwargs(self, enable_thinking: bool) -> dict: + if self._reasoning_style == "reasoning_effort": + return {"reasoning_effort": "high" if enable_thinking else "low"} + return {"enable_thinking": enable_thinking} + + def _request_reasoning_kwargs( + self, + enable_thinking: Optional[bool], + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, + ) -> Optional[dict]: + """Build chat_template_kwargs from per-request reasoning fields. + + Produces a merged dict covering the active model's reasoning style + (``enable_thinking`` or ``reasoning_effort``) plus the independent + ``preserve_thinking`` kwarg when the template supports it. + """ + kwargs: dict = {} + # Always-on reasoning models hardcode tags in their template + # and do not consume enable_thinking / reasoning_effort -- skip. + if self._supports_reasoning and not self._reasoning_always_on: + if self._reasoning_style == "reasoning_effort": + if reasoning_effort in ("low", "medium", "high"): + kwargs["reasoning_effort"] = reasoning_effort + elif enable_thinking is not None: + kwargs["reasoning_effort"] = "high" if enable_thinking else "low" + else: + if enable_thinking is not None: + kwargs["enable_thinking"] = enable_thinking + if self._supports_preserve_thinking and preserve_thinking is not None: + kwargs["preserve_thinking"] = preserve_thinking + return kwargs or None + @property def supports_tools(self) -> bool: return self._supports_tools @@ -818,6 +939,9 @@ class LlamaCppBackend: self._chat_template = None self._supports_reasoning = False self._reasoning_always_on = False + self._reasoning_style = "enable_thinking" + self._reasoning_default = True + self._supports_preserve_thinking = False self._supports_tools = False self._n_layers = None self._n_kv_heads = None @@ -898,48 +1022,16 @@ class LlamaCppBackend: f"GGUF metadata: chat_template={len(self._chat_template)} chars" ) # Detect thinking/reasoning support from chat template - tpl = self._chat_template - if "enable_thinking" in tpl: - self._supports_reasoning = True - logger.info( - "GGUF metadata: model supports reasoning (enable_thinking)" - ) - elif "thinking" in tpl: - # DeepSeek uses 'thinking' instead of 'enable_thinking' - normalized_id = (self._model_identifier or "").lower() - if "deepseek" in normalized_id: - self._supports_reasoning = True - logger.info( - "GGUF metadata: model supports reasoning (DeepSeek thinking)" - ) - # Models with hardcoded tags or reasoning_content - # in their chat template always produce thinking output - # (no toggle to disable it). - if not self._supports_reasoning: - if ( - "" in tpl - and "" in tpl - or "reasoning_content" in tpl - ): - self._supports_reasoning = True - self._reasoning_always_on = True - logger.info( - "GGUF metadata: model always reasons ( tags in template)" - ) - # Detect tool calling support from chat template - tool_markers = [ - "{%- if tools %}", - "{%- if tools -%}", - "{% if tools %}", - "{% if tools -%}", - '"role" == "tool"', - "'role' == 'tool'", - 'message.role == "tool"', - "message.role == 'tool'", - ] - if any(marker in tpl for marker in tool_markers): - self._supports_tools = True - logger.info("GGUF metadata: model supports tool calling") + flags = detect_reasoning_flags( + self._chat_template, + self._model_identifier, + log_source = "GGUF metadata", + ) + self._supports_reasoning = flags["supports_reasoning"] + self._reasoning_style = flags["reasoning_style"] + self._reasoning_always_on = flags["reasoning_always_on"] + self._supports_preserve_thinking = flags["supports_preserve_thinking"] + self._supports_tools = flags["supports_tools"] except Exception as e: logger.warning(f"Failed to read GGUF metadata: {e}") @@ -1523,7 +1615,8 @@ class LlamaCppBackend: # For reasoning models, set default thinking mode. # Qwen3.5/3.6 models below 9B (0.8B, 2B, 4B) disable thinking by default. # Only 9B and larger enable thinking. - if self._supports_reasoning: + # Always-on templates ignore the kwarg entirely, so skip. + if self._supports_reasoning and not self._reasoning_always_on: thinking_default = True mid = (model_identifier or "").lower() if "qwen3.5" in mid or "qwen3.6" in mid: @@ -1531,15 +1624,14 @@ class LlamaCppBackend: if size_val is not None and size_val < 9: thinking_default = False self._reasoning_default = thinking_default + reasoning_kw = self._reasoning_kwargs(thinking_default) cmd.extend( [ "--chat-template-kwargs", - json.dumps({"enable_thinking": thinking_default}), + json.dumps(reasoning_kw), ] ) - logger.info( - f"Reasoning model: enable_thinking={thinking_default} by default" - ) + logger.info(f"Reasoning model: {reasoning_kw} by default") if mmproj_path: if not Path(mmproj_path).is_file(): @@ -1767,6 +1859,9 @@ class LlamaCppBackend: self._chat_template = None self._supports_reasoning = False self._reasoning_always_on = False + self._reasoning_style = "enable_thinking" + self._reasoning_default = True + self._supports_preserve_thinking = False self._supports_tools = False self._cache_type_kv = None self._speculative_type = None @@ -2317,6 +2412,8 @@ class LlamaCppBackend: stop: Optional[list[str]] = None, cancel_event: Optional[threading.Event] = None, enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str | dict, None, None]: """ Send a chat completion request to llama-server and stream tokens back. @@ -2341,9 +2438,12 @@ class LlamaCppBackend: "repeat_penalty": repetition_penalty, "presence_penalty": presence_penalty, } - # Pass enable_thinking per-request for reasoning models - if self._supports_reasoning and enable_thinking is not None: - payload["chat_template_kwargs"] = {"enable_thinking": enable_thinking} + # Pass enable_thinking / reasoning_effort / preserve_thinking per-request + _reasoning_kw = self._request_reasoning_kwargs( + enable_thinking, reasoning_effort, preserve_thinking + ) + if _reasoning_kw is not None: + payload["chat_template_kwargs"] = _reasoning_kw if max_tokens is not None: payload["max_tokens"] = max_tokens if stop: @@ -2479,6 +2579,8 @@ class LlamaCppBackend: stop: Optional[list[str]] = None, cancel_event: Optional[threading.Event] = None, enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, max_tool_iterations: int = 25, auto_heal_tool_calls: bool = True, tool_call_timeout: int = 300, @@ -2557,8 +2659,11 @@ class LlamaCppBackend: "tools": tools, "tool_choice": "auto", } - if self._supports_reasoning and enable_thinking is not None: - payload["chat_template_kwargs"] = {"enable_thinking": enable_thinking} + _reasoning_kw = self._request_reasoning_kwargs( + enable_thinking, reasoning_effort, preserve_thinking + ) + if _reasoning_kw is not None: + payload["chat_template_kwargs"] = _reasoning_kw if max_tokens is not None: payload["max_tokens"] = max_tokens if stop: @@ -3207,10 +3312,11 @@ class LlamaCppBackend: "repeat_penalty": repetition_penalty, "presence_penalty": presence_penalty, } - if self._supports_reasoning and enable_thinking is not None: - stream_payload["chat_template_kwargs"] = { - "enable_thinking": enable_thinking - } + _reasoning_kw = self._request_reasoning_kwargs( + enable_thinking, reasoning_effort, preserve_thinking + ) + if _reasoning_kw is not None: + stream_payload["chat_template_kwargs"] = _reasoning_kw if max_tokens is not None: stream_payload["max_tokens"] = max_tokens if stop: diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 324002ddbf..e5b037755d 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -157,12 +157,20 @@ class LoadResponse(BaseModel): ) supports_reasoning: bool = Field( False, - description = "Whether model supports thinking/reasoning mode (enable_thinking)", + description = "Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)", + ) + reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field( + "enable_thinking", + description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", ) reasoning_always_on: bool = Field( False, description = "Whether reasoning is always on (hardcoded tags, not toggleable)", ) + supports_preserve_thinking: bool = Field( + False, + description = "Whether the template understands the optional preserve_thinking kwarg (Qwen3.6-style)", + ) supports_tools: bool = Field( False, description = "Whether model supports tool calling (web search, etc.)", @@ -261,9 +269,17 @@ class InferenceStatusResponse(BaseModel): supports_reasoning: bool = Field( False, description = "Whether the active model supports reasoning/thinking mode" ) + reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field( + "enable_thinking", + description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", + ) reasoning_always_on: bool = Field( False, description = "Whether reasoning is always on (not toggleable)" ) + supports_preserve_thinking: bool = Field( + False, + description = "Whether the active model's template understands the optional preserve_thinking kwarg", + ) supports_tools: bool = Field( False, description = "Whether the active model supports tool calling" ) @@ -481,6 +497,14 @@ class ChatCompletionRequest(BaseModel): None, description = "[x-unsloth] Enable/disable thinking/reasoning mode for supported models", ) + reasoning_effort: Optional[Literal["low", "medium", "high"]] = Field( + None, + description = "[x-unsloth] Reasoning effort level ('low'|'medium'|'high') for Harmony-style reasoning models (e.g. gpt-oss). Overrides enable_thinking when the active model uses reasoning_effort style.", + ) + preserve_thinking: Optional[bool] = Field( + None, + description = "[x-unsloth] When true, keep historical blocks from past assistant turns in the prompt (Qwen3.6 templates). Independent of enable_thinking / reasoning_effort.", + ) enable_tools: Optional[bool] = Field( None, description = "[x-unsloth] Enable tool calling for supported models", diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index e17f8f5882..ed331a5660 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -113,7 +113,7 @@ if str(backend_path) not in sys.path: # Import backend functions try: from core.inference import get_inference_backend - from core.inference.llama_cpp import LlamaCppBackend + from core.inference.llama_cpp import LlamaCppBackend, detect_reasoning_flags from utils.models import ModelConfig from utils.inference import load_inference_config from utils.models.model_config import load_model_defaults @@ -122,7 +122,7 @@ except ImportError: if str(parent_backend) not in sys.path: sys.path.insert(0, str(parent_backend)) from core.inference import get_inference_backend - from core.inference.llama_cpp import LlamaCppBackend + from core.inference.llama_cpp import LlamaCppBackend, detect_reasoning_flags from utils.models import ModelConfig from utils.inference import load_inference_config from utils.models.model_config import load_model_defaults @@ -273,7 +273,9 @@ async def load_model( max_context_length = llama_backend.max_context_length, native_context_length = llama_backend.native_context_length, supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, chat_template = llama_backend.chat_template, speculative_type = llama_backend.speculative_type, ) @@ -295,6 +297,21 @@ async def load_model( logger.warning( f"Could not retrieve chat template for {backend.active_model_name}: {e}" ) + # Non-GGUF: only advertise reasoning for gpt-oss Harmony, + # which emits reasoning via channels at the tokenizer level. + # Template-level chat_template_kwargs (enable_thinking / + # preserve_thinking / tools) are not yet forwarded through + # the transformers generation path, so avoid advertising + # controls the server cannot honour outside GGUF. + _sf_supports_reasoning = False + _sf_reasoning_style = "enable_thinking" + if hasattr(backend, "_is_gpt_oss_model"): + try: + if backend._is_gpt_oss_model(): + _sf_supports_reasoning = True + _sf_reasoning_style = "reasoning_effort" + except Exception: + pass return LoadResponse( status = "already_loaded", model = backend.active_model_name, @@ -309,6 +326,11 @@ async def load_model( requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), + supports_reasoning = _sf_supports_reasoning, + reasoning_style = _sf_reasoning_style, + reasoning_always_on = False, + supports_preserve_thinking = False, + supports_tools = False, chat_template = _chat_template, ) @@ -422,7 +444,9 @@ async def load_model( max_context_length = llama_backend.max_context_length, native_context_length = llama_backend.native_context_length, supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, supports_tools = llama_backend.supports_tools, cache_type_kv = llama_backend.cache_type_kv, chat_template = llama_backend.chat_template, @@ -545,6 +569,20 @@ async def load_model( except Exception: pass + # Non-GGUF: gpt-oss Harmony surfaces reasoning via tokenizer-level + # channels; other safetensors reasoning/tools/preserve-thinking + # knobs are not forwarded to tokenizer.apply_chat_template yet, so + # we only advertise support for the Harmony case here. + _sf_supports_reasoning = False + _sf_reasoning_style = "enable_thinking" + if hasattr(backend, "_is_gpt_oss_model"): + try: + if backend._is_gpt_oss_model(): + _sf_supports_reasoning = True + _sf_reasoning_style = "reasoning_effort" + except Exception: + pass + return LoadResponse( status = "loaded", model = config.identifier, @@ -559,6 +597,11 @@ async def load_model( requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), + supports_reasoning = _sf_supports_reasoning, + reasoning_style = _sf_reasoning_style, + reasoning_always_on = False, + supports_preserve_thinking = False, + supports_tools = False, chat_template = _chat_template, ) @@ -766,7 +809,9 @@ async def get_status( (_inference_cfg or {}).get("trust_remote_code", False) ), supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, supports_tools = llama_backend.supports_tools, context_length = llama_backend.context_length, max_context_length = llama_backend.max_context_length, @@ -788,10 +833,18 @@ async def get_status( audio_type = model_info.get("audio_type") has_audio_input = model_info.get("has_audio_input", False) - # gpt-oss safetensors models support reasoning via harmony channels + # Non-GGUF: only gpt-oss Harmony is wired through the transformers + # generation path. Other template-level reasoning / tool kwargs + # are not yet forwarded, so we do not advertise them here. supports_reasoning = False + reasoning_style = "enable_thinking" if backend.active_model_name and hasattr(backend, "_is_gpt_oss_model"): - supports_reasoning = backend._is_gpt_oss_model() + try: + if backend._is_gpt_oss_model(): + supports_reasoning = True + reasoning_style = "reasoning_effort" + except Exception: + pass inference_config = ( load_inference_config(backend.active_model_name) if backend.active_model_name @@ -812,6 +865,10 @@ async def get_status( (inference_config or {}).get("trust_remote_code", False) ), supports_reasoning = supports_reasoning, + reasoning_style = reasoning_style, + reasoning_always_on = False, + supports_preserve_thinking = False, + supports_tools = False, ) except Exception as e: @@ -1393,6 +1450,8 @@ async def openai_chat_completions( presence_penalty = payload.presence_penalty, cancel_event = cancel_event, enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, auto_heal_tool_calls = payload.auto_heal_tool_calls if payload.auto_heal_tool_calls is not None else True, @@ -1562,6 +1621,8 @@ async def openai_chat_completions( presence_penalty = payload.presence_penalty, cancel_event = cancel_event, enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, ) _gguf_sentinel = object() diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index 0b97d98dfd..3f67b11b51 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -24,8 +24,15 @@ import { useScrollThreadToBottom, } from "@/components/assistant-ui/use-intent-aware-autoscroll"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { sentAudioNames } from "@/features/chat/api/chat-adapter"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; +import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; import { deleteThreadMessage } from "@/features/chat/utils/delete-thread-message"; import { AUDIO_ACCEPT, MAX_AUDIO_SIZE, fileToBase64 } from "@/lib/audio-utils"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; @@ -373,18 +380,6 @@ const ComposerAudioUpload: FC = () => { ); }; -/** Qwen3/3.5 recommended params differ between thinking on/off. */ -function applyQwenThinkingParams(thinkingOn: boolean): void { - const store = useChatRuntimeStore.getState(); - const checkpoint = store.params.checkpoint?.toLowerCase() ?? ""; - if (!checkpoint.includes("qwen3")) { - return; - } - const params = thinkingOn - ? { temperature: 0.6, topP: 0.95, topK: 20, minP: 0.0 } - : { temperature: 0.7, topP: 0.8, topK: 20, minP: 0.0 }; - store.setParams({ ...store.params, ...params }); -} const ReasoningToggle: FC = () => { const modelLoaded = useChatRuntimeStore( @@ -393,8 +388,49 @@ const ReasoningToggle: FC = () => { const supportsReasoning = useChatRuntimeStore((s) => s.supportsReasoning); const reasoningEnabled = useChatRuntimeStore((s) => s.reasoningEnabled); const setReasoningEnabled = useChatRuntimeStore((s) => s.setReasoningEnabled); + const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle); + const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort); + const setReasoningEffort = useChatRuntimeStore((s) => s.setReasoningEffort); const disabled = !(modelLoaded && supportsReasoning); + if (reasoningStyle === "reasoning_effort") { + return ( + + + + + + {(["low", "medium", "high"] as const).map((level) => ( + setReasoningEffort(level)} + > + {level.charAt(0).toUpperCase() + level.slice(1)} + {reasoningEffort === level ? " \u2713" : ""} + + ))} + + + ); + } + return ( + ); +}; + const WebSearchToggle: FC = () => { const modelLoaded = useChatRuntimeStore( (s) => !!s.params.checkpoint && !s.modelLoading, @@ -551,6 +625,7 @@ const ComposerAction: FC<{ disabled?: boolean }> = ({ disabled }) => { + diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 53a935cf45..a93120397e 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -385,6 +385,8 @@ async function autoLoadSmallestModel(): Promise<{ supportsReasoning: loadResp.supports_reasoning ?? false, reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, + reasoningStyle: loadResp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: loadResp.supports_preserve_thinking ?? false, supportsTools: loadResp.supports_tools ?? false, toolsEnabled: loadResp.supports_tools ?? false, codeToolsEnabled: loadResp.supports_tools ?? false, @@ -433,6 +435,14 @@ async function autoLoadSmallestModel(): Promise<{ sfLoadResp.requires_trust_remote_code ?? false, ); store.setParams({ ...store.params, maxTokens: 4096 }); + useChatRuntimeStore.setState({ + supportsReasoning: sfLoadResp.supports_reasoning ?? false, + reasoningAlwaysOn: sfLoadResp.reasoning_always_on ?? false, + reasoningEnabled: sfLoadResp.supports_reasoning ?? false, + reasoningStyle: sfLoadResp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: sfLoadResp.supports_preserve_thinking ?? false, + supportsTools: sfLoadResp.supports_tools ?? false, + }); const sfModel: ChatModelSummary = { id: repo.repo_id, name: sfLoadResp.display_name ?? repo.repo_id, @@ -501,6 +511,8 @@ async function autoLoadSmallestModel(): Promise<{ supportsReasoning: loadResp.supports_reasoning ?? false, reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, + reasoningStyle: loadResp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: loadResp.supports_preserve_thinking ?? false, supportsTools: loadResp.supports_tools ?? false, toolsEnabled: loadResp.supports_tools ?? false, codeToolsEnabled: loadResp.supports_tools ?? false, @@ -696,7 +708,14 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { let serverMetadata: { usage?: ServerUsage; timings?: ServerTimings } | null = null; try { - const { supportsReasoning, reasoningEnabled } = runtime; + const { + supportsReasoning, + reasoningEnabled, + reasoningStyle, + reasoningEffort, + supportsPreserveThinking, + preserveThinking, + } = runtime; const stream = streamChatCompletions( { model: params.checkpoint, @@ -712,7 +731,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { image_base64: imageBase64, audio_base64: audioBase64, ...(useAdapter === undefined ? {} : { use_adapter: useAdapter }), - ...(supportsReasoning ? { enable_thinking: reasoningEnabled } : {}), + ...(supportsReasoning + ? reasoningStyle === "reasoning_effort" + ? { reasoning_effort: reasoningEffort } + : { enable_thinking: reasoningEnabled } + : {}), + ...(supportsPreserveThinking ? { preserve_thinking: preserveThinking } : {}), ...(supportsTools && (toolsEnabled || codeToolsEnabled) ? { enable_tools: true, diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 6c870d74a9..c9dcd911a2 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -253,7 +253,9 @@ export function useChatModelRuntime() { max_context_length: statusRes.max_context_length, native_context_length: statusRes.native_context_length, supports_reasoning: statusRes.supports_reasoning, + reasoning_style: statusRes.reasoning_style, reasoning_always_on: statusRes.reasoning_always_on, + supports_preserve_thinking: statusRes.supports_preserve_thinking, supports_tools: statusRes.supports_tools, speculative_type: statusRes.speculative_type, }; @@ -265,6 +267,8 @@ export function useChatModelRuntime() { // Restore reasoning/tools support flags and context length const supportsReasoning = statusRes.supports_reasoning ?? false; const reasoningAlwaysOn = statusRes.reasoning_always_on ?? false; + const reasoningStyle = statusRes.reasoning_style ?? "enable_thinking"; + const supportsPreserveThinking = statusRes.supports_preserve_thinking ?? false; const supportsTools = statusRes.supports_tools ?? false; const currentGgufContextLength = statusRes.is_gguf ? (statusRes.context_length ?? null) @@ -279,7 +283,14 @@ export function useChatModelRuntime() { useChatRuntimeStore.setState({ supportsReasoning, reasoningAlwaysOn, + reasoningStyle, + supportsPreserveThinking, supportsTools, + // Reset per-turn reasoning flag so models that do not support + // reasoning do not inherit a stale off state from a prior model. + reasoningEnabled: supportsReasoning + ? useChatRuntimeStore.getState().reasoningEnabled + : true, ggufContextLength: currentGgufContextLength, ggufMaxContextLength, ggufNativeContextLength, @@ -498,6 +509,8 @@ export function useChatModelRuntime() { supportsReasoning: loadResponse.supports_reasoning ?? false, reasoningAlwaysOn, reasoningEnabled: reasoningAlwaysOn ? true : reasoningDefault, + reasoningStyle: loadResponse.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: loadResponse.supports_preserve_thinking ?? false, supportsTools: loadResponse.supports_tools ?? false, toolsEnabled: loadResponse.supports_tools ?? false, codeToolsEnabled: loadResponse.supports_tools ?? false, diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index e133669a1c..3ba2f8b6f5 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -4,6 +4,13 @@ import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { CodeToggleIcon } from "@/components/assistant-ui/code-toggle-icon"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; import { AUDIO_ACCEPT, MAX_AUDIO_SIZE, fileToBase64 } from "@/lib/audio-utils"; import { useAui } from "@assistant-ui/react"; import { cn } from "@/lib/utils"; @@ -245,6 +252,12 @@ export function SharedComposer({ const reasoningAlwaysOn = useChatRuntimeStore((s) => s.reasoningAlwaysOn); const reasoningEnabled = useChatRuntimeStore((s) => s.reasoningEnabled); const setReasoningEnabled = useChatRuntimeStore((s) => s.setReasoningEnabled); + const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle); + const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort); + const setReasoningEffort = useChatRuntimeStore((s) => s.setReasoningEffort); + const supportsPreserveThinking = useChatRuntimeStore((s) => s.supportsPreserveThinking); + const preserveThinking = useChatRuntimeStore((s) => s.preserveThinking); + const setPreserveThinking = useChatRuntimeStore((s) => s.setPreserveThinking); const supportsTools = useChatRuntimeStore((s) => s.supportsTools); const toolsEnabled = useChatRuntimeStore((s) => s.toolsEnabled); const setToolsEnabled = useChatRuntimeStore((s) => s.setToolsEnabled); @@ -391,6 +404,13 @@ export function SharedComposer({ store.setModelRequiresTrustRemoteCode( resp.requires_trust_remote_code ?? false, ); + useChatRuntimeStore.setState({ + supportsReasoning: resp.supports_reasoning ?? false, + reasoningAlwaysOn: resp.reasoning_always_on ?? false, + reasoningStyle: resp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: resp.supports_preserve_thinking ?? false, + supportsTools: resp.supports_tools ?? false, + }); return resp.status; } @@ -566,41 +586,93 @@ export function SharedComposer({ )} - + + + {(["low", "medium", "high"] as const).map((level) => ( + setReasoningEffort(level)} + > + {level.charAt(0).toUpperCase() + level.slice(1)} + {reasoningEffort === level ? " \u2713" : ""} + + ))} + + + ) : ( + + )} + {supportsPreserveThinking && ( + + > + {preserveThinking && modelLoaded ? ( + + ) : ( + + )} + Preserve Thinking + + )}