diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 6ad05ac52d..c320f03b2c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -87,6 +87,84 @@ _TC_PARAM_START_RE = re.compile(r"\s*") _TC_PARAM_CLOSE_RE = re.compile(r"\s*\s*$") +_TOOL_TEMPLATE_MARKERS = ( + "{%- if tools %}", + "{%- if tools -%}", + "{% if tools %}", + "{% if tools -%}", + '"role" == "tool"', + "'role' == 'tool'", + 'message.role == "tool"', + "message.role == 'tool'", +) + + +def detect_reasoning_flags( + chat_template: Optional[str], + model_identifier: Optional[str] = None, + *, + log_source: Optional[str] = None, +) -> dict: + """Classify a chat template's reasoning and tool-calling capabilities. + + Returns a dict with the same five keys populated by the GGUF sniffer: + ``supports_reasoning``, ``reasoning_style`` + (``"enable_thinking"`` | ``"reasoning_effort"``), + ``reasoning_always_on``, ``supports_preserve_thinking``, and + ``supports_tools``. Used by both the llama-server backend at load + time and the safetensors/transformers paths in ``routes/inference`` + so the two agree on what the frontend will see. + """ + flags = { + "supports_reasoning": False, + "reasoning_style": "enable_thinking", + "reasoning_always_on": False, + "supports_preserve_thinking": False, + "supports_tools": False, + } + if not chat_template: + return flags + tpl = chat_template + prefix = f"{log_source}: " if log_source else "" + + if "enable_thinking" in tpl: + flags["supports_reasoning"] = True + flags["reasoning_style"] = "enable_thinking" + logger.info(f"{prefix}model supports reasoning (enable_thinking)") + elif "reasoning_effort" in tpl: + # gpt-oss / Harmony templates use reasoning_effort + # ("low" | "medium" | "high") instead of a boolean. + flags["supports_reasoning"] = True + flags["reasoning_style"] = "reasoning_effort" + logger.info(f"{prefix}model supports reasoning (reasoning_effort)") + elif "thinking" in tpl: + # DeepSeek uses 'thinking' instead of 'enable_thinking' + normalized_id = (model_identifier or "").lower() + if "deepseek" in normalized_id: + flags["supports_reasoning"] = True + logger.info(f"{prefix}model supports reasoning (DeepSeek thinking)") + + # Hardcoded tags or reasoning_content in the template mean + # thinking is always on (no toggle to disable it). + if not flags["supports_reasoning"]: + if ("" in tpl and "" in tpl) or "reasoning_content" in tpl: + flags["supports_reasoning"] = True + flags["reasoning_always_on"] = True + logger.info(f"{prefix}model always reasons ( tags in template)") + + # preserve_thinking is an independent kwarg on some Qwen templates + # that keeps historical blocks in prior assistant turns. + if "preserve_thinking" in tpl: + flags["supports_preserve_thinking"] = True + logger.info(f"{prefix}model supports preserve_thinking") + + if any(marker in tpl for marker in _TOOL_TEMPLATE_MARKERS): + flags["supports_tools"] = True + logger.info(f"{prefix}model supports tool calling") + + return flags + + class LlamaCppBackend: """ Manages a llama-server subprocess for GGUF model inference. @@ -112,6 +190,8 @@ class LlamaCppBackend: self._chat_template: Optional[str] = None self._supports_reasoning: bool = False self._reasoning_always_on: bool = False + self._reasoning_style: str = "enable_thinking" + self._supports_preserve_thinking: bool = False self._supports_tools: bool = False self._cache_type_kv: Optional[str] = None self._reasoning_default: bool = True @@ -293,10 +373,51 @@ class LlamaCppBackend: def reasoning_always_on(self) -> bool: return self._reasoning_always_on + @property + def reasoning_style(self) -> str: + return self._reasoning_style + + @property + def supports_preserve_thinking(self) -> bool: + return self._supports_preserve_thinking + @property def reasoning_default(self) -> bool: return self._reasoning_default + def _reasoning_kwargs(self, enable_thinking: bool) -> dict: + if self._reasoning_style == "reasoning_effort": + return {"reasoning_effort": "high" if enable_thinking else "low"} + return {"enable_thinking": enable_thinking} + + def _request_reasoning_kwargs( + self, + enable_thinking: Optional[bool], + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, + ) -> Optional[dict]: + """Build chat_template_kwargs from per-request reasoning fields. + + Produces a merged dict covering the active model's reasoning style + (``enable_thinking`` or ``reasoning_effort``) plus the independent + ``preserve_thinking`` kwarg when the template supports it. + """ + kwargs: dict = {} + # Always-on reasoning models hardcode tags in their template + # and do not consume enable_thinking / reasoning_effort -- skip. + if self._supports_reasoning and not self._reasoning_always_on: + if self._reasoning_style == "reasoning_effort": + if reasoning_effort in ("low", "medium", "high"): + kwargs["reasoning_effort"] = reasoning_effort + elif enable_thinking is not None: + kwargs["reasoning_effort"] = "high" if enable_thinking else "low" + else: + if enable_thinking is not None: + kwargs["enable_thinking"] = enable_thinking + if self._supports_preserve_thinking and preserve_thinking is not None: + kwargs["preserve_thinking"] = preserve_thinking + return kwargs or None + @property def supports_tools(self) -> bool: return self._supports_tools @@ -818,6 +939,9 @@ class LlamaCppBackend: self._chat_template = None self._supports_reasoning = False self._reasoning_always_on = False + self._reasoning_style = "enable_thinking" + self._reasoning_default = True + self._supports_preserve_thinking = False self._supports_tools = False self._n_layers = None self._n_kv_heads = None @@ -898,48 +1022,16 @@ class LlamaCppBackend: f"GGUF metadata: chat_template={len(self._chat_template)} chars" ) # Detect thinking/reasoning support from chat template - tpl = self._chat_template - if "enable_thinking" in tpl: - self._supports_reasoning = True - logger.info( - "GGUF metadata: model supports reasoning (enable_thinking)" - ) - elif "thinking" in tpl: - # DeepSeek uses 'thinking' instead of 'enable_thinking' - normalized_id = (self._model_identifier or "").lower() - if "deepseek" in normalized_id: - self._supports_reasoning = True - logger.info( - "GGUF metadata: model supports reasoning (DeepSeek thinking)" - ) - # Models with hardcoded tags or reasoning_content - # in their chat template always produce thinking output - # (no toggle to disable it). - if not self._supports_reasoning: - if ( - "" in tpl - and "" in tpl - or "reasoning_content" in tpl - ): - self._supports_reasoning = True - self._reasoning_always_on = True - logger.info( - "GGUF metadata: model always reasons ( tags in template)" - ) - # Detect tool calling support from chat template - tool_markers = [ - "{%- if tools %}", - "{%- if tools -%}", - "{% if tools %}", - "{% if tools -%}", - '"role" == "tool"', - "'role' == 'tool'", - 'message.role == "tool"', - "message.role == 'tool'", - ] - if any(marker in tpl for marker in tool_markers): - self._supports_tools = True - logger.info("GGUF metadata: model supports tool calling") + flags = detect_reasoning_flags( + self._chat_template, + self._model_identifier, + log_source = "GGUF metadata", + ) + self._supports_reasoning = flags["supports_reasoning"] + self._reasoning_style = flags["reasoning_style"] + self._reasoning_always_on = flags["reasoning_always_on"] + self._supports_preserve_thinking = flags["supports_preserve_thinking"] + self._supports_tools = flags["supports_tools"] except Exception as e: logger.warning(f"Failed to read GGUF metadata: {e}") @@ -1523,7 +1615,8 @@ class LlamaCppBackend: # For reasoning models, set default thinking mode. # Qwen3.5/3.6 models below 9B (0.8B, 2B, 4B) disable thinking by default. # Only 9B and larger enable thinking. - if self._supports_reasoning: + # Always-on templates ignore the kwarg entirely, so skip. + if self._supports_reasoning and not self._reasoning_always_on: thinking_default = True mid = (model_identifier or "").lower() if "qwen3.5" in mid or "qwen3.6" in mid: @@ -1531,15 +1624,14 @@ class LlamaCppBackend: if size_val is not None and size_val < 9: thinking_default = False self._reasoning_default = thinking_default + reasoning_kw = self._reasoning_kwargs(thinking_default) cmd.extend( [ "--chat-template-kwargs", - json.dumps({"enable_thinking": thinking_default}), + json.dumps(reasoning_kw), ] ) - logger.info( - f"Reasoning model: enable_thinking={thinking_default} by default" - ) + logger.info(f"Reasoning model: {reasoning_kw} by default") if mmproj_path: if not Path(mmproj_path).is_file(): @@ -1767,6 +1859,9 @@ class LlamaCppBackend: self._chat_template = None self._supports_reasoning = False self._reasoning_always_on = False + self._reasoning_style = "enable_thinking" + self._reasoning_default = True + self._supports_preserve_thinking = False self._supports_tools = False self._cache_type_kv = None self._speculative_type = None @@ -2317,6 +2412,8 @@ class LlamaCppBackend: stop: Optional[list[str]] = None, cancel_event: Optional[threading.Event] = None, enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str | dict, None, None]: """ Send a chat completion request to llama-server and stream tokens back. @@ -2341,9 +2438,12 @@ class LlamaCppBackend: "repeat_penalty": repetition_penalty, "presence_penalty": presence_penalty, } - # Pass enable_thinking per-request for reasoning models - if self._supports_reasoning and enable_thinking is not None: - payload["chat_template_kwargs"] = {"enable_thinking": enable_thinking} + # Pass enable_thinking / reasoning_effort / preserve_thinking per-request + _reasoning_kw = self._request_reasoning_kwargs( + enable_thinking, reasoning_effort, preserve_thinking + ) + if _reasoning_kw is not None: + payload["chat_template_kwargs"] = _reasoning_kw if max_tokens is not None: payload["max_tokens"] = max_tokens if stop: @@ -2479,6 +2579,8 @@ class LlamaCppBackend: stop: Optional[list[str]] = None, cancel_event: Optional[threading.Event] = None, enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, max_tool_iterations: int = 25, auto_heal_tool_calls: bool = True, tool_call_timeout: int = 300, @@ -2557,8 +2659,11 @@ class LlamaCppBackend: "tools": tools, "tool_choice": "auto", } - if self._supports_reasoning and enable_thinking is not None: - payload["chat_template_kwargs"] = {"enable_thinking": enable_thinking} + _reasoning_kw = self._request_reasoning_kwargs( + enable_thinking, reasoning_effort, preserve_thinking + ) + if _reasoning_kw is not None: + payload["chat_template_kwargs"] = _reasoning_kw if max_tokens is not None: payload["max_tokens"] = max_tokens if stop: @@ -3207,10 +3312,11 @@ class LlamaCppBackend: "repeat_penalty": repetition_penalty, "presence_penalty": presence_penalty, } - if self._supports_reasoning and enable_thinking is not None: - stream_payload["chat_template_kwargs"] = { - "enable_thinking": enable_thinking - } + _reasoning_kw = self._request_reasoning_kwargs( + enable_thinking, reasoning_effort, preserve_thinking + ) + if _reasoning_kw is not None: + stream_payload["chat_template_kwargs"] = _reasoning_kw if max_tokens is not None: stream_payload["max_tokens"] = max_tokens if stop: diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 324002ddbf..e5b037755d 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -157,12 +157,20 @@ class LoadResponse(BaseModel): ) supports_reasoning: bool = Field( False, - description = "Whether model supports thinking/reasoning mode (enable_thinking)", + description = "Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)", + ) + reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field( + "enable_thinking", + description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", ) reasoning_always_on: bool = Field( False, description = "Whether reasoning is always on (hardcoded tags, not toggleable)", ) + supports_preserve_thinking: bool = Field( + False, + description = "Whether the template understands the optional preserve_thinking kwarg (Qwen3.6-style)", + ) supports_tools: bool = Field( False, description = "Whether model supports tool calling (web search, etc.)", @@ -261,9 +269,17 @@ class InferenceStatusResponse(BaseModel): supports_reasoning: bool = Field( False, description = "Whether the active model supports reasoning/thinking mode" ) + reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field( + "enable_thinking", + description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", + ) reasoning_always_on: bool = Field( False, description = "Whether reasoning is always on (not toggleable)" ) + supports_preserve_thinking: bool = Field( + False, + description = "Whether the active model's template understands the optional preserve_thinking kwarg", + ) supports_tools: bool = Field( False, description = "Whether the active model supports tool calling" ) @@ -481,6 +497,14 @@ class ChatCompletionRequest(BaseModel): None, description = "[x-unsloth] Enable/disable thinking/reasoning mode for supported models", ) + reasoning_effort: Optional[Literal["low", "medium", "high"]] = Field( + None, + description = "[x-unsloth] Reasoning effort level ('low'|'medium'|'high') for Harmony-style reasoning models (e.g. gpt-oss). Overrides enable_thinking when the active model uses reasoning_effort style.", + ) + preserve_thinking: Optional[bool] = Field( + None, + description = "[x-unsloth] When true, keep historical blocks from past assistant turns in the prompt (Qwen3.6 templates). Independent of enable_thinking / reasoning_effort.", + ) enable_tools: Optional[bool] = Field( None, description = "[x-unsloth] Enable tool calling for supported models", diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index e17f8f5882..ed331a5660 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -113,7 +113,7 @@ if str(backend_path) not in sys.path: # Import backend functions try: from core.inference import get_inference_backend - from core.inference.llama_cpp import LlamaCppBackend + from core.inference.llama_cpp import LlamaCppBackend, detect_reasoning_flags from utils.models import ModelConfig from utils.inference import load_inference_config from utils.models.model_config import load_model_defaults @@ -122,7 +122,7 @@ except ImportError: if str(parent_backend) not in sys.path: sys.path.insert(0, str(parent_backend)) from core.inference import get_inference_backend - from core.inference.llama_cpp import LlamaCppBackend + from core.inference.llama_cpp import LlamaCppBackend, detect_reasoning_flags from utils.models import ModelConfig from utils.inference import load_inference_config from utils.models.model_config import load_model_defaults @@ -273,7 +273,9 @@ async def load_model( max_context_length = llama_backend.max_context_length, native_context_length = llama_backend.native_context_length, supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, chat_template = llama_backend.chat_template, speculative_type = llama_backend.speculative_type, ) @@ -295,6 +297,21 @@ async def load_model( logger.warning( f"Could not retrieve chat template for {backend.active_model_name}: {e}" ) + # Non-GGUF: only advertise reasoning for gpt-oss Harmony, + # which emits reasoning via channels at the tokenizer level. + # Template-level chat_template_kwargs (enable_thinking / + # preserve_thinking / tools) are not yet forwarded through + # the transformers generation path, so avoid advertising + # controls the server cannot honour outside GGUF. + _sf_supports_reasoning = False + _sf_reasoning_style = "enable_thinking" + if hasattr(backend, "_is_gpt_oss_model"): + try: + if backend._is_gpt_oss_model(): + _sf_supports_reasoning = True + _sf_reasoning_style = "reasoning_effort" + except Exception: + pass return LoadResponse( status = "already_loaded", model = backend.active_model_name, @@ -309,6 +326,11 @@ async def load_model( requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), + supports_reasoning = _sf_supports_reasoning, + reasoning_style = _sf_reasoning_style, + reasoning_always_on = False, + supports_preserve_thinking = False, + supports_tools = False, chat_template = _chat_template, ) @@ -422,7 +444,9 @@ async def load_model( max_context_length = llama_backend.max_context_length, native_context_length = llama_backend.native_context_length, supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, supports_tools = llama_backend.supports_tools, cache_type_kv = llama_backend.cache_type_kv, chat_template = llama_backend.chat_template, @@ -545,6 +569,20 @@ async def load_model( except Exception: pass + # Non-GGUF: gpt-oss Harmony surfaces reasoning via tokenizer-level + # channels; other safetensors reasoning/tools/preserve-thinking + # knobs are not forwarded to tokenizer.apply_chat_template yet, so + # we only advertise support for the Harmony case here. + _sf_supports_reasoning = False + _sf_reasoning_style = "enable_thinking" + if hasattr(backend, "_is_gpt_oss_model"): + try: + if backend._is_gpt_oss_model(): + _sf_supports_reasoning = True + _sf_reasoning_style = "reasoning_effort" + except Exception: + pass + return LoadResponse( status = "loaded", model = config.identifier, @@ -559,6 +597,11 @@ async def load_model( requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), + supports_reasoning = _sf_supports_reasoning, + reasoning_style = _sf_reasoning_style, + reasoning_always_on = False, + supports_preserve_thinking = False, + supports_tools = False, chat_template = _chat_template, ) @@ -766,7 +809,9 @@ async def get_status( (_inference_cfg or {}).get("trust_remote_code", False) ), supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, supports_tools = llama_backend.supports_tools, context_length = llama_backend.context_length, max_context_length = llama_backend.max_context_length, @@ -788,10 +833,18 @@ async def get_status( audio_type = model_info.get("audio_type") has_audio_input = model_info.get("has_audio_input", False) - # gpt-oss safetensors models support reasoning via harmony channels + # Non-GGUF: only gpt-oss Harmony is wired through the transformers + # generation path. Other template-level reasoning / tool kwargs + # are not yet forwarded, so we do not advertise them here. supports_reasoning = False + reasoning_style = "enable_thinking" if backend.active_model_name and hasattr(backend, "_is_gpt_oss_model"): - supports_reasoning = backend._is_gpt_oss_model() + try: + if backend._is_gpt_oss_model(): + supports_reasoning = True + reasoning_style = "reasoning_effort" + except Exception: + pass inference_config = ( load_inference_config(backend.active_model_name) if backend.active_model_name @@ -812,6 +865,10 @@ async def get_status( (inference_config or {}).get("trust_remote_code", False) ), supports_reasoning = supports_reasoning, + reasoning_style = reasoning_style, + reasoning_always_on = False, + supports_preserve_thinking = False, + supports_tools = False, ) except Exception as e: @@ -1393,6 +1450,8 @@ async def openai_chat_completions( presence_penalty = payload.presence_penalty, cancel_event = cancel_event, enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, auto_heal_tool_calls = payload.auto_heal_tool_calls if payload.auto_heal_tool_calls is not None else True, @@ -1562,6 +1621,8 @@ async def openai_chat_completions( presence_penalty = payload.presence_penalty, cancel_event = cancel_event, enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, ) _gguf_sentinel = object() diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index 0b97d98dfd..3f67b11b51 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -24,8 +24,15 @@ import { useScrollThreadToBottom, } from "@/components/assistant-ui/use-intent-aware-autoscroll"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { sentAudioNames } from "@/features/chat/api/chat-adapter"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; +import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; import { deleteThreadMessage } from "@/features/chat/utils/delete-thread-message"; import { AUDIO_ACCEPT, MAX_AUDIO_SIZE, fileToBase64 } from "@/lib/audio-utils"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; @@ -373,18 +380,6 @@ const ComposerAudioUpload: FC = () => { ); }; -/** Qwen3/3.5 recommended params differ between thinking on/off. */ -function applyQwenThinkingParams(thinkingOn: boolean): void { - const store = useChatRuntimeStore.getState(); - const checkpoint = store.params.checkpoint?.toLowerCase() ?? ""; - if (!checkpoint.includes("qwen3")) { - return; - } - const params = thinkingOn - ? { temperature: 0.6, topP: 0.95, topK: 20, minP: 0.0 } - : { temperature: 0.7, topP: 0.8, topK: 20, minP: 0.0 }; - store.setParams({ ...store.params, ...params }); -} const ReasoningToggle: FC = () => { const modelLoaded = useChatRuntimeStore( @@ -393,8 +388,49 @@ const ReasoningToggle: FC = () => { const supportsReasoning = useChatRuntimeStore((s) => s.supportsReasoning); const reasoningEnabled = useChatRuntimeStore((s) => s.reasoningEnabled); const setReasoningEnabled = useChatRuntimeStore((s) => s.setReasoningEnabled); + const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle); + const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort); + const setReasoningEffort = useChatRuntimeStore((s) => s.setReasoningEffort); const disabled = !(modelLoaded && supportsReasoning); + if (reasoningStyle === "reasoning_effort") { + return ( + + + + + + {(["low", "medium", "high"] as const).map((level) => ( + setReasoningEffort(level)} + > + {level.charAt(0).toUpperCase() + level.slice(1)} + {reasoningEffort === level ? " \u2713" : ""} + + ))} + + + ); + } + return ( + ); +}; + const WebSearchToggle: FC = () => { const modelLoaded = useChatRuntimeStore( (s) => !!s.params.checkpoint && !s.modelLoading, @@ -551,6 +625,7 @@ const ComposerAction: FC<{ disabled?: boolean }> = ({ disabled }) => { + diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 53a935cf45..a93120397e 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -385,6 +385,8 @@ async function autoLoadSmallestModel(): Promise<{ supportsReasoning: loadResp.supports_reasoning ?? false, reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, + reasoningStyle: loadResp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: loadResp.supports_preserve_thinking ?? false, supportsTools: loadResp.supports_tools ?? false, toolsEnabled: loadResp.supports_tools ?? false, codeToolsEnabled: loadResp.supports_tools ?? false, @@ -433,6 +435,14 @@ async function autoLoadSmallestModel(): Promise<{ sfLoadResp.requires_trust_remote_code ?? false, ); store.setParams({ ...store.params, maxTokens: 4096 }); + useChatRuntimeStore.setState({ + supportsReasoning: sfLoadResp.supports_reasoning ?? false, + reasoningAlwaysOn: sfLoadResp.reasoning_always_on ?? false, + reasoningEnabled: sfLoadResp.supports_reasoning ?? false, + reasoningStyle: sfLoadResp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: sfLoadResp.supports_preserve_thinking ?? false, + supportsTools: sfLoadResp.supports_tools ?? false, + }); const sfModel: ChatModelSummary = { id: repo.repo_id, name: sfLoadResp.display_name ?? repo.repo_id, @@ -501,6 +511,8 @@ async function autoLoadSmallestModel(): Promise<{ supportsReasoning: loadResp.supports_reasoning ?? false, reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, + reasoningStyle: loadResp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: loadResp.supports_preserve_thinking ?? false, supportsTools: loadResp.supports_tools ?? false, toolsEnabled: loadResp.supports_tools ?? false, codeToolsEnabled: loadResp.supports_tools ?? false, @@ -696,7 +708,14 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { let serverMetadata: { usage?: ServerUsage; timings?: ServerTimings } | null = null; try { - const { supportsReasoning, reasoningEnabled } = runtime; + const { + supportsReasoning, + reasoningEnabled, + reasoningStyle, + reasoningEffort, + supportsPreserveThinking, + preserveThinking, + } = runtime; const stream = streamChatCompletions( { model: params.checkpoint, @@ -712,7 +731,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { image_base64: imageBase64, audio_base64: audioBase64, ...(useAdapter === undefined ? {} : { use_adapter: useAdapter }), - ...(supportsReasoning ? { enable_thinking: reasoningEnabled } : {}), + ...(supportsReasoning + ? reasoningStyle === "reasoning_effort" + ? { reasoning_effort: reasoningEffort } + : { enable_thinking: reasoningEnabled } + : {}), + ...(supportsPreserveThinking ? { preserve_thinking: preserveThinking } : {}), ...(supportsTools && (toolsEnabled || codeToolsEnabled) ? { enable_tools: true, diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 6c870d74a9..c9dcd911a2 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -253,7 +253,9 @@ export function useChatModelRuntime() { max_context_length: statusRes.max_context_length, native_context_length: statusRes.native_context_length, supports_reasoning: statusRes.supports_reasoning, + reasoning_style: statusRes.reasoning_style, reasoning_always_on: statusRes.reasoning_always_on, + supports_preserve_thinking: statusRes.supports_preserve_thinking, supports_tools: statusRes.supports_tools, speculative_type: statusRes.speculative_type, }; @@ -265,6 +267,8 @@ export function useChatModelRuntime() { // Restore reasoning/tools support flags and context length const supportsReasoning = statusRes.supports_reasoning ?? false; const reasoningAlwaysOn = statusRes.reasoning_always_on ?? false; + const reasoningStyle = statusRes.reasoning_style ?? "enable_thinking"; + const supportsPreserveThinking = statusRes.supports_preserve_thinking ?? false; const supportsTools = statusRes.supports_tools ?? false; const currentGgufContextLength = statusRes.is_gguf ? (statusRes.context_length ?? null) @@ -279,7 +283,14 @@ export function useChatModelRuntime() { useChatRuntimeStore.setState({ supportsReasoning, reasoningAlwaysOn, + reasoningStyle, + supportsPreserveThinking, supportsTools, + // Reset per-turn reasoning flag so models that do not support + // reasoning do not inherit a stale off state from a prior model. + reasoningEnabled: supportsReasoning + ? useChatRuntimeStore.getState().reasoningEnabled + : true, ggufContextLength: currentGgufContextLength, ggufMaxContextLength, ggufNativeContextLength, @@ -498,6 +509,8 @@ export function useChatModelRuntime() { supportsReasoning: loadResponse.supports_reasoning ?? false, reasoningAlwaysOn, reasoningEnabled: reasoningAlwaysOn ? true : reasoningDefault, + reasoningStyle: loadResponse.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: loadResponse.supports_preserve_thinking ?? false, supportsTools: loadResponse.supports_tools ?? false, toolsEnabled: loadResponse.supports_tools ?? false, codeToolsEnabled: loadResponse.supports_tools ?? false, diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index e133669a1c..3ba2f8b6f5 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -4,6 +4,13 @@ import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { CodeToggleIcon } from "@/components/assistant-ui/code-toggle-icon"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; import { AUDIO_ACCEPT, MAX_AUDIO_SIZE, fileToBase64 } from "@/lib/audio-utils"; import { useAui } from "@assistant-ui/react"; import { cn } from "@/lib/utils"; @@ -245,6 +252,12 @@ export function SharedComposer({ const reasoningAlwaysOn = useChatRuntimeStore((s) => s.reasoningAlwaysOn); const reasoningEnabled = useChatRuntimeStore((s) => s.reasoningEnabled); const setReasoningEnabled = useChatRuntimeStore((s) => s.setReasoningEnabled); + const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle); + const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort); + const setReasoningEffort = useChatRuntimeStore((s) => s.setReasoningEffort); + const supportsPreserveThinking = useChatRuntimeStore((s) => s.supportsPreserveThinking); + const preserveThinking = useChatRuntimeStore((s) => s.preserveThinking); + const setPreserveThinking = useChatRuntimeStore((s) => s.setPreserveThinking); const supportsTools = useChatRuntimeStore((s) => s.supportsTools); const toolsEnabled = useChatRuntimeStore((s) => s.toolsEnabled); const setToolsEnabled = useChatRuntimeStore((s) => s.setToolsEnabled); @@ -391,6 +404,13 @@ export function SharedComposer({ store.setModelRequiresTrustRemoteCode( resp.requires_trust_remote_code ?? false, ); + useChatRuntimeStore.setState({ + supportsReasoning: resp.supports_reasoning ?? false, + reasoningAlwaysOn: resp.reasoning_always_on ?? false, + reasoningStyle: resp.reasoning_style ?? "enable_thinking", + supportsPreserveThinking: resp.supports_preserve_thinking ?? false, + supportsTools: resp.supports_tools ?? false, + }); return resp.status; } @@ -566,41 +586,93 @@ export function SharedComposer({ )} - + + + {(["low", "medium", "high"] as const).map((level) => ( + setReasoningEffort(level)} + > + {level.charAt(0).toUpperCase() + level.slice(1)} + {reasoningEffort === level ? " \u2713" : ""} + + ))} + + + ) : ( + + )} + {supportsPreserveThinking && ( + + > + {preserveThinking && modelLoaded ? ( + + ) : ( + + )} + Preserve Thinking + + )}