From 44dcf30b9b07362d5bc0e7242877ce43f1d1b7fe Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 16 Mar 2026 06:37:55 -0700 Subject: [PATCH] studio: per-model inference defaults, GGUF slider fix, reasoning toggle (#4325) * studio: extract param count from model name as fallback When HuggingFace API doesn't return totalParams for a model, extract the param count from the model name (e.g. "Qwen3-0.6B" -> "0.6B", "Llama-3.2-1B-Instruct" -> "1B"). Applied to both the recommended list and HF search results. * studio: read GGUF context_length via fast header parser, set max tokens - Fast GGUF metadata reader (~30-55ms) parses only KV header, skips tensor data and large arrays (tokenizer vocab etc) - Extracts context_length and chat_template from GGUF metadata - Returns context_length in LoadResponse for frontend to use - Frontend sets maxTokens to actual context_length for GGUFs (e.g. 262144 for Qwen3.5-9B, 131072 for Qwen2.5-7B) - Max Tokens slider shows "Max" and is locked for GGUFs - Auto-load path also uses actual context_length from load response - Toast auto-dismiss (5s) and close button for auto-load toast * studio: GGUF TTS audio support (from PR #4318) Add GGUF TTS audio generation via llama-server. When a GGUF model loads, the backend probes its vocabulary to detect audio codecs (SNAC/BiCodec/DAC/CSM/Whisper). If detected, the codec is pre-loaded and the model is reported as audio to the frontend. During chat, TTS models route to the audio generation path which sends a per-codec prompt to llama-server's /completion endpoint, extracts generated tokens/text, and decodes to WAV using AudioCodecManager. Also strips base64 audio data from prior assistant messages to prevent context overflow. Co-authored-by: Manan Shah * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove package-lock.json from tracking * studio: per-model inference defaults, GGUF max tokens fix, reasoning toggle - Add inference_defaults.json with per-model-family sampling parameters for ~50 families (Qwen3.5, Qwen3, Gemma-3, Llama-3, DeepSeek, etc.). Values sourced from unslothai/docs and Ollama params blobs. - Family-based lookup in inference_config.py: extracts model family from identifier, matches against patterns (longest match first), merges with priority: model-specific YAML > family JSON > default.yaml. - Fix GGUF Max Tokens slider locked at "Max": store ggufContextLength separately from maxTokens so the slider is adjustable (step=64). - Fix Ministral YAML: top_p was literal string "default", now 0.95. - Add reasoning toggle for thinking models (Qwen3.5, Qwen3, DeepSeek-R1, DeepSeek-V3.1, etc.): detect enable_thinking support from GGUF chat template metadata, pass --jinja to llama-server, send chat_template_kwargs per-request. Frontend shows "Reasoning is ON/OFF" pill button next to attachment button in composer. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * studio: remove default system prompt injection Backend was injecting "You are a helpful AI assistant." when no system prompt was provided. Neither unslothai/docs nor Ollama specify a default system prompt for most models. Now defaults to empty string, letting the model's own chat template handle system behavior. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * studio: use lightbulb icons and "Think" label for reasoning toggle Lightbulb on when thinking enabled, lightbulb-off when disabled. Label is just "Think" in both states; grayed out styling when off. * studio: fix HTML file upload breaking chat Replace SimpleTextAttachmentAdapter with custom TextAttachmentAdapter (excludes text/html) and HtmlAttachmentAdapter that strips tags via DOMParser, removing scripts/styles and extracting readable text content instead of dumping raw HTML markup into the conversation. * studio: show chat template in Configuration panel Display the model's Jinja2 chat template in a new "Chat Template" section under Settings (now open by default). For GGUFs, reads from GGUF metadata; for safetensors, reads from tokenizer.chat_template. Template is editable with a "Restore default chat template" button that appears when modified. Section only shows when a model with a chat template is loaded. * studio: editable chat template with Apply & Reload Chat template section now functional: - Editing the template shows "Apply & Reload" (reloads model with custom template) and "Revert changes" buttons - For GGUFs: writes template to temp .jinja file, passes --chat-template-file to llama-server on reload - For non-GGUF: passes chat_template_override in load request - Settings section now open by default - selectModel supports forceReload to reload same model * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * studio: fix DeepSeek reasoning detection and auto-load metadata - Set _model_identifier before _read_gguf_metadata so DeepSeek "thinking" template detection works (was always None before) - Populate ggufContextLength, supportsReasoning, reasoningEnabled, defaultChatTemplate in autoLoadSmallestModel GGUF path * studio: add spacing before BETA badge in navbar Add gap-1.5 on the logo Link container to space the BETA label from the wordmark. Co-authored-by: Imagineer99 * studio: vertically center BETA badge with logo --------- Co-authored-by: Manan Shah Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Imagineer99 --- .../assets/configs/inference_defaults.json | 381 ++++++++++++++++++ .../unsloth_Ministral-3-3B-Instruct-2512.yaml | 2 +- studio/backend/core/inference/audio_codecs.py | 22 + studio/backend/core/inference/inference.py | 2 +- studio/backend/core/inference/llama_cpp.py | 333 +++++++++++++++ studio/backend/models/inference.py | 23 +- studio/backend/routes/inference.py | 146 ++++--- studio/backend/routes/models.py | 16 + .../utils/inference/inference_config.py | 127 +++++- .../assistant-ui/model-selector/pickers.tsx | 12 +- .../src/components/assistant-ui/thread.tsx | 32 ++ studio/frontend/src/components/navbar.tsx | 4 +- .../src/features/chat/api/chat-adapter.ts | 37 +- .../frontend/src/features/chat/chat-page.tsx | 12 + .../src/features/chat/chat-settings-sheet.tsx | 67 ++- .../chat/hooks/use-chat-model-runtime.ts | 24 +- .../src/features/chat/runtime-provider.tsx | 75 +++- .../src/features/chat/shared-composer.tsx | 26 +- .../chat/stores/chat-runtime-store.ts | 19 + .../frontend/src/features/chat/types/api.ts | 5 + 20 files changed, 1273 insertions(+), 92 deletions(-) create mode 100644 studio/backend/assets/configs/inference_defaults.json diff --git a/studio/backend/assets/configs/inference_defaults.json b/studio/backend/assets/configs/inference_defaults.json new file mode 100644 index 0000000000..a8fea07bf7 --- /dev/null +++ b/studio/backend/assets/configs/inference_defaults.json @@ -0,0 +1,381 @@ +{ + "_comment": "Per-model-family inference parameter defaults. Sources: (1) Ollama params blobs, (2) Existing Unsloth Studio YAML configs. Patterns ordered longest-match-first.", + "families": { + "qwen3.5": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen3-coder": { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen3-next": { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen3-vl": { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen3": { + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen2.5-coder": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "qwen2.5-vl": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "qwen2.5-omni": { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen2.5-math": { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen2.5": { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwen2-vl": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "qwen2": { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "qwq": { + "temperature": 0.6, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "gemma-3n": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 64, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "gemma-3": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 64, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "medgemma": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 64, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "gemma-2": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 64, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "llama-4": { + "temperature": 1.0, + "top_p": 0.9, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "llama-3.3": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "llama-3.2": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "llama-3.1": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "llama-3": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "phi-4": { + "temperature": 0.8, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.0, + "repetition_penalty": 1.0 + }, + "phi-3": { + "temperature": 0.7, + "top_p": 0.9, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "mistral-nemo": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "mistral-small": { + "temperature": 0.15, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "mistral-large": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "magistral": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "ministral": { + "temperature": 0.15, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "devstral": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "pixtral": { + "temperature": 1.5, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.1, + "repetition_penalty": 1.0 + }, + "deepseek-r1": { + "temperature": 0.6, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "deepseek-v3": { + "temperature": 0.6, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "deepseek-ocr": { + "temperature": 0.0, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "glm-5": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "glm-4": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "nemotron": { + "temperature": 1.0, + "top_p": 1.0, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "minimax-m2.5": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "minimax": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "gpt-oss": { + "temperature": 1.0, + "top_p": 1.0, + "top_k": 0, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "granite-4": { + "temperature": 0.0, + "top_p": 1.0, + "top_k": 0, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "kimi-k2": { + "temperature": 0.6, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "kimi": { + "temperature": 0.6, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "lfm2": { + "temperature": 0.1, + "top_p": 0.1, + "top_k": 50, + "min_p": 0.15, + "repetition_penalty": 1.05 + }, + "smollm": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "olmo": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "falcon": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "ernie": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "seed": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "grok": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + }, + "mimo": { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "repetition_penalty": 1.0 + } + }, + "patterns": [ + "qwen3.5", + "qwen3-coder", "qwen3-next", "qwen3-vl", "qwen3", + "qwen2.5-coder", "qwen2.5-vl", "qwen2.5-omni", "qwen2.5-math", "qwen2.5", + "qwen2-vl", "qwen2", + "qwq", + "gemma-3n", "gemma-3", "medgemma", "gemma-2", + "llama-4", "llama-3.3", "llama-3.2", "llama-3.1", "llama-3", + "phi-4", "phi-3", + "mistral-nemo", "mistral-small", "mistral-large", "magistral", "ministral", + "devstral", "pixtral", + "deepseek-r1", "deepseek-v3", "deepseek-ocr", + "glm-5", "glm-4", + "nemotron", + "minimax-m2.5", "minimax", + "gpt-oss", "granite-4", + "kimi-k2", "kimi", + "lfm2", "smollm", "olmo", "falcon", "ernie", "seed", "grok", "mimo" + ] +} diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml index c9f771fd23..b84f7e1abb 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml @@ -51,5 +51,5 @@ logging: inference: trust_remote_code: false temperature: 0.15 - top_p: default + top_p: 0.95 diff --git a/studio/backend/core/inference/audio_codecs.py b/studio/backend/core/inference/audio_codecs.py index 3a418d921d..bcf3ec2937 100644 --- a/studio/backend/core/inference/audio_codecs.py +++ b/studio/backend/core/inference/audio_codecs.py @@ -302,6 +302,28 @@ class AudioCodecManager: waveform = audio.squeeze().cpu().numpy() return _numpy_to_wav_bytes(waveform, 24000), 24000 + def decode( + self, + audio_type: str, + device: str, + token_ids: Optional[list] = None, + text: Optional[str] = None, + ) -> Tuple[bytes, int]: + """Unified decode — dispatches to the right codec decoder.""" + if audio_type == "snac": + if not token_ids: + raise ValueError("SNAC decoding requires token_ids") + return self.decode_snac(torch.tensor([token_ids], dtype = torch.long), device) + elif audio_type == "bicodec": + if not text: + raise ValueError("BiCodec decoding requires text") + return self.decode_bicodec(text, device) + elif audio_type == "dac": + if not text: + raise ValueError("DAC decoding requires text") + return self.decode_dac(text, device) + raise ValueError(f"Cannot decode audio_type: {audio_type}") + # ── Cleanup ────────────────────────────────────────────────── def unload(self) -> None: diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 21bf815695..6f04588fbc 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -950,7 +950,7 @@ class InferenceBackend: break # Use ASR-specific system prompt if user hasn't set a custom one - if not system_prompt or system_prompt == "You are a helpful AI assistant.": + if not system_prompt: system_prompt = "You are an assistant that transcribes speech accurately." # Build messages in Gemma 3n format — audio goes INTO apply_chat_template diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 5d67500928..3a137b8b63 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -10,6 +10,7 @@ through its OpenAI-compatible /v1/chat/completions endpoint. import atexit import json +import struct import structlog from loggers import get_logger import shutil @@ -45,6 +46,9 @@ class LlamaCppBackend: self._hf_variant: Optional[str] = None self._is_vision: bool = False self._healthy = False + self._context_length: Optional[int] = None + self._chat_template: Optional[str] = None + self._supports_reasoning: bool = False self._lock = threading.Lock() self._stdout_lines: list[str] = [] self._stdout_thread: Optional[threading.Thread] = None @@ -80,6 +84,18 @@ class LlamaCppBackend: def hf_variant(self) -> Optional[str]: return self._hf_variant + @property + def context_length(self) -> Optional[int]: + return self._context_length + + @property + def chat_template(self) -> Optional[str]: + return self._chat_template + + @property + def supports_reasoning(self) -> bool: + return self._supports_reasoning + # ── Binary discovery ────────────────────────────────────────── @staticmethod @@ -371,6 +387,114 @@ class LlamaCppBackend: # Pipe closed — process is terminating pass + # GGUF KV type sizes for fast skipping + _GGUF_TYPE_SIZE = { + 0: 1, + 1: 1, + 2: 2, + 3: 2, + 4: 4, + 5: 4, + 6: 4, + 7: 1, + 10: 8, + 11: 8, + 12: 8, + } + + @staticmethod + def _gguf_skip_value(f, vtype: int) -> None: + """Skip a GGUF KV value without reading it.""" + sz = LlamaCppBackend._GGUF_TYPE_SIZE.get(vtype) + if sz is not None: + f.seek(sz, 1) + elif vtype == 8: # STRING + slen = struct.unpack(" None: + """Read context_length and chat_template from a GGUF file's KV header. + + Parses only the KV pairs we need (~30ms even for multi-GB files). + For split GGUFs, metadata is always in shard 1. + """ + try: + WANTED = {"general.architecture", "tokenizer.chat_template"} + arch = None + ctx_key = None + + with open(gguf_path, "rb") as f: + magic = struct.unpack(" bool: @@ -647,6 +772,12 @@ class LlamaCppBackend: else: raise ValueError("Either gguf_path or hf_repo must be provided") + # Set identifier early so _read_gguf_metadata can use it for DeepSeek detection + self._model_identifier = model_identifier + + # Read GGUF metadata (context_length, chat_template) -- fast, header only + self._read_gguf_metadata(model_path) + # Check cancel after download if self._cancel_event.is_set(): logger.info("Load cancelled after download phase") @@ -694,6 +825,36 @@ class LlamaCppBackend: if n_threads is not None: cmd.extend(["--threads", str(n_threads)]) + # Always enable Jinja chat template rendering for proper template support + cmd.extend(["--jinja"]) + + # Apply custom chat template override if provided + if chat_template_override: + import tempfile + + self._chat_template_file = tempfile.NamedTemporaryFile( + mode = "w", + suffix = ".jinja", + delete = False, + prefix = "unsloth_chat_template_", + ) + self._chat_template_file.write(chat_template_override) + self._chat_template_file.close() + cmd.extend(["--chat-template-file", self._chat_template_file.name]) + logger.info( + f"Using custom chat template file: {self._chat_template_file.name}" + ) + + # For reasoning models, default to thinking ON (user can toggle per-request) + if self._supports_reasoning: + cmd.extend( + [ + "--chat-template-kwargs", + json.dumps({"enable_thinking": True}), + ] + ) + logger.info("Reasoning model: enabled enable_thinking=true by default") + if mmproj_path: if not Path(mmproj_path).is_file(): logger.warning(f"mmproj file not found: {mmproj_path}") @@ -784,8 +945,30 @@ class LlamaCppBackend: self._hf_repo = None self._hf_variant = None self._is_vision = False + self._is_audio = False + self._audio_type = None self._port = None self._healthy = False + self._context_length = None + self._chat_template = None + self._supports_reasoning = False + # Clean up temp chat template file + if hasattr(self, "_chat_template_file") and self._chat_template_file: + try: + import os + + os.unlink(self._chat_template_file.name) + except Exception: + pass + self._chat_template_file = None + # Free audio codec GPU memory + if LlamaCppBackend._codec_mgr is not None: + LlamaCppBackend._codec_mgr.unload() + LlamaCppBackend._codec_mgr = None + import torch + + if torch.cuda.is_available(): + torch.cuda.empty_cache() return True def _kill_process(self): @@ -937,6 +1120,7 @@ class LlamaCppBackend: repetition_penalty: float = 1.0, stop: Optional[list[str]] = None, cancel_event: Optional[threading.Event] = None, + enable_thinking: Optional[bool] = None, ) -> Generator[str, None, None]: """ Send a chat completion request to llama-server and stream tokens back. @@ -960,6 +1144,9 @@ class LlamaCppBackend: "min_p": min_p, "repeat_penalty": repetition_penalty, } + # Pass enable_thinking per-request for reasoning models + if self._supports_reasoning and enable_thinking is not None: + payload["chat_template_kwargs"] = {"enable_thinking": enable_thinking} if max_tokens is not None: payload["max_tokens"] = max_tokens if stop: @@ -1033,3 +1220,149 @@ class LlamaCppBackend: if cancel_event is not None and cancel_event.is_set(): return raise + + # ── TTS support ──────────────────────────────────────────── + + def detect_audio_type(self) -> Optional[str]: + """Detect audio/TTS codec by probing the loaded model's vocabulary.""" + if not self.is_loaded: + return None + try: + with httpx.Client(timeout = 10) as client: + + def _detok(tid: int) -> str: + r = client.post( + f"{self.base_url}/detokenize", json = {"tokens": [tid]} + ) + return r.json().get("content", "") if r.status_code == 200 else "" + + def _tok(text: str) -> list[int]: + r = client.post( + f"{self.base_url}/tokenize", + json = {"content": text, "add_special": False}, + ) + return r.json().get("tokens", []) if r.status_code == 200 else [] + + # Check codec-specific tokens (not generic ones that may exist in non-audio models) + if "")) == 1 and len(_tok("<|audio_eos|>")) == 1: + return "csm" + if len(_tok("<|startoftranscript|>")) == 1: + return "whisper" + if ( + len(_tok("<|bicodec_semantic_0|>")) == 1 + and len(_tok("<|bicodec_global_0|>")) == 1 + ): + return "bicodec" + if len(_tok("<|c1_0|>")) == 1 and len(_tok("<|c2_0|>")) == 1: + return "dac" + except Exception as e: + logger.debug(f"Audio type detection failed: {e}") + return None + + # Prompt format per codec: (template, stop_tokens, needs_token_ids) + # Matches prompts in InferenceBackend._generate_snac/bicodec/dac + _TTS_PROMPTS = { + "snac": ( + "{text}<|eot_id|>", + [""], + True, + ), + "bicodec": ( + "<|task_tts|><|start_content|>{text}<|end_content|><|start_global_token|>", + ["<|im_end|>", ""], + False, + ), + "dac": ( + "<|im_start|>\n<|text_start|>{text}<|text_end|>\n<|audio_start|><|global_features_start|>\n", + ["<|im_end|>", "<|audio_end|>"], + False, + ), + } + + _codec_mgr = None # Shared AudioCodecManager instance + + def init_audio_codec(self, audio_type: str) -> None: + """Load the audio codec at model load time (mirrors non-GGUF path).""" + import torch + from core.inference.audio_codecs import AudioCodecManager + + if LlamaCppBackend._codec_mgr is None: + LlamaCppBackend._codec_mgr = AudioCodecManager() + + device = "cuda" if torch.cuda.is_available() else "cpu" + model_repo_path = None + + # BiCodec needs a repo with BiCodec/ weights — download canonical SparkTTS + if audio_type == "bicodec": + from huggingface_hub import snapshot_download + import os + + repo_path = snapshot_download( + "unsloth/Spark-TTS-0.5B", local_dir = "Spark-TTS-0.5B" + ) + model_repo_path = os.path.abspath(repo_path) + + LlamaCppBackend._codec_mgr.load_codec( + audio_type, device, model_repo_path = model_repo_path + ) + logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}") + + def generate_audio_response( + self, + text: str, + audio_type: str, + temperature: float = 0.6, + top_p: float = 0.95, + top_k: int = 50, + min_p: float = 0.0, + max_new_tokens: int = 2048, + repetition_penalty: float = 1.1, + ) -> tuple: + """ + Generate TTS audio via llama-server /completion + codec decoding. + Returns (wav_bytes, sample_rate). + """ + if audio_type not in self._TTS_PROMPTS: + raise RuntimeError(f"GGUF TTS does not support '{audio_type}' codec.") + + tpl, stop, need_ids = self._TTS_PROMPTS[audio_type] + + payload: dict = { + "prompt": tpl.format(text = text), + "stream": False, + "n_predict": max_new_tokens, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k if top_k >= 0 else 0, + "min_p": min_p, + "repeat_penalty": repetition_penalty, + } + if stop: + payload["stop"] = stop + if need_ids: + payload["n_probs"] = 1 + + with httpx.Client(timeout = httpx.Timeout(300, connect = 10)) as client: + resp = client.post(f"{self.base_url}/completion", json = payload) + if resp.status_code != 200: + raise RuntimeError( + f"llama-server returned {resp.status_code}: {resp.text}" + ) + + data = resp.json() + token_ids = ( + [p["id"] for p in data.get("completion_probabilities", []) if "id" in p] + if need_ids + else None + ) + + import torch + + device = "cuda" if torch.cuda.is_available() else "cpu" + return LlamaCppBackend._codec_mgr.decode( + audio_type, device, token_ids = token_ids, text = data.get("content", "") + ) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 620304f37a..0014ebfdd0 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -33,6 +33,10 @@ class LoadRequest(BaseModel): False, description = "Allow loading models with custom code (e.g. NVIDIA Nemotron). Only enable for repos you trust.", ) + chat_template_override: Optional[str] = Field( + None, + description = "Custom Jinja2 chat template to use instead of the model's default", + ) class UnloadRequest(BaseModel): @@ -81,9 +85,7 @@ class GenerateRequest(BaseModel): """Request for text generation (legacy /generate/stream endpoint)""" messages: List[dict] = Field(..., description = "Chat messages in OpenAI format") - system_prompt: str = Field( - "You are a helpful AI assistant.", description = "System prompt" - ) + system_prompt: str = Field("", description = "System prompt") temperature: float = Field(0.7, ge = 0.0, le = 2.0, description = "Sampling temperature") top_p: float = Field(0.9, ge = 0.0, le = 1.0, description = "Top-p sampling") top_k: int = Field(40, ge = -1, le = 100, description = "Top-k sampling") @@ -119,6 +121,17 @@ class LoadResponse(BaseModel): inference: dict = Field( ..., description = "Inference parameters (temperature, top_p, top_k, min_p)" ) + context_length: Optional[int] = Field( + None, description = "Model's native context length (from GGUF metadata)" + ) + supports_reasoning: bool = Field( + False, + description = "Whether model supports thinking/reasoning mode (enable_thinking)", + ) + chat_template: Optional[str] = Field( + None, + description = "Jinja2 chat template string (from GGUF metadata or tokenizer)", + ) class UnloadResponse(BaseModel): @@ -267,6 +280,10 @@ class ChatCompletionRequest(BaseModel): "string = enable a specific adapter by name." ), ) + enable_thinking: Optional[bool] = Field( + None, + description = "[x-unsloth] Enable/disable thinking/reasoning mode for supported models", + ) # ── Streaming response chunks ──────────────────────────────────── diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 55bd9710a5..e93ba1d6d6 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -135,6 +135,7 @@ async def load_model( model_identifier = config.identifier, is_vision = config.is_vision, n_ctx = request.max_seq_length, + chat_template_override = request.chat_template_override, ) else: # Local mode: llama-server loads via -m @@ -145,6 +146,7 @@ async def load_model( model_identifier = config.identifier, is_vision = config.is_vision, n_ctx = request.max_seq_length, + chat_template_override = request.chat_template_override, ) if not success: @@ -155,6 +157,17 @@ async def load_model( logger.info(f"Loaded GGUF model via llama-server: {config.identifier}") + # Detect TTS audio by probing the loaded model's vocabulary + from utils.models import is_audio_input_type + + _gguf_audio = llama_backend.detect_audio_type() + _gguf_is_audio = _gguf_audio in ("snac", "bicodec", "dac") + llama_backend._is_audio = _gguf_is_audio + llama_backend._audio_type = _gguf_audio + if _gguf_is_audio: + logger.info(f"GGUF model detected as audio: audio_type={_gguf_audio}") + await asyncio.to_thread(llama_backend.init_audio_codec, _gguf_audio) + inference_config = load_inference_config(config.identifier) return LoadResponse( @@ -164,7 +177,13 @@ async def load_model( is_vision = config.is_vision, is_lora = False, is_gguf = True, + is_audio = _gguf_is_audio, + audio_type = _gguf_audio, + has_audio_input = is_audio_input_type(_gguf_audio), inference = inference_config, + context_length = llama_backend.context_length, + supports_reasoning = llama_backend.supports_reasoning, + chat_template = llama_backend.chat_template, ) # ── Standard path: load via Unsloth/transformers ────────── @@ -273,6 +292,15 @@ async def load_model( # Load inference configuration parameters inference_config = load_inference_config(config.identifier) + # Get chat template from tokenizer + _chat_template = None + try: + _model_info = backend.models.get(config.identifier, {}) + _tpl_info = _model_info.get("chat_template_info", {}) + _chat_template = _tpl_info.get("template") + except Exception: + pass + return LoadResponse( status = "loaded", model = config.identifier, @@ -284,6 +312,7 @@ async def load_model( audio_type = config.audio_type, has_audio_input = config.has_audio_input, inference = inference_config, + chat_template = _chat_template, ) except HTTPException: @@ -473,6 +502,8 @@ async def get_status( is_vision = llama_backend.is_vision, is_gguf = True, gguf_variant = llama_backend.hf_variant, + is_audio = getattr(llama_backend, "_is_audio", False), + audio_type = getattr(llama_backend, "_audio_type", None), loading = [], loaded = [llama_backend.model_identifier], ) @@ -521,78 +552,84 @@ async def generate_audio( """ Generate audio (TTS) from the latest user message. Returns a JSON response with base64-encoded WAV audio. - Only works when an audio model is loaded. + Works with both GGUF (llama-server) and Unsloth/transformers backends. """ import base64 - backend = get_inference_backend() - if not backend.active_model_name: - raise HTTPException(status_code = 400, detail = "No model loaded.") - - model_info = backend.models.get(backend.active_model_name, {}) - if not model_info.get("is_audio"): - raise HTTPException( - status_code = 400, detail = "Active model is not an audio model." - ) - # Extract text from the last user message _, chat_messages, _ = _extract_content_parts(payload.messages) if not chat_messages: raise HTTPException(status_code = 400, detail = "No messages provided.") - last_user_msg = next( (m for m in reversed(chat_messages) if m["role"] == "user"), None ) if not last_user_msg: raise HTTPException(status_code = 400, detail = "No user message found.") - text = last_user_msg["content"] + # Pick backend — both return (wav_bytes, sample_rate) + llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded and getattr(llama_backend, "_is_audio", False): + model_name = llama_backend.model_identifier + gen = lambda: llama_backend.generate_audio_response( + text = text, + audio_type = llama_backend._audio_type, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_new_tokens = payload.max_tokens or 2048, + repetition_penalty = payload.repetition_penalty, + ) + else: + backend = get_inference_backend() + if not backend.active_model_name: + raise HTTPException(status_code = 400, detail = "No model loaded.") + model_info = backend.models.get(backend.active_model_name, {}) + if not model_info.get("is_audio"): + raise HTTPException( + status_code = 400, detail = "Active model is not an audio model." + ) + model_name = backend.active_model_name + gen = lambda: backend.generate_audio_response( + text = text, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_new_tokens = payload.max_tokens or 2048, + repetition_penalty = payload.repetition_penalty, + use_adapter = payload.use_adapter, + ) + try: wav_bytes, sample_rate = await asyncio.get_event_loop().run_in_executor( - None, - lambda: backend.generate_audio_response( - text = text, - temperature = payload.temperature, - top_p = payload.top_p, - top_k = payload.top_k, - min_p = payload.min_p, - max_new_tokens = payload.max_tokens or 2048, - repetition_penalty = payload.repetition_penalty, - use_adapter = payload.use_adapter, - ), + None, gen ) - - audio_b64 = base64.b64encode(wav_bytes).decode("ascii") - completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" - - return JSONResponse( - content = { - "id": completion_id, - "object": "chat.completion.audio", - "model": backend.active_model_name, - "audio": { - "data": audio_b64, - "format": "wav", - "sample_rate": sample_rate, - }, - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": f'[Generated audio from: "{text[:100]}"]', - }, - "finish_reason": "stop", - } - ], - } - ) - except Exception as e: logger.error(f"Audio generation error: {e}", exc_info = True) raise HTTPException(status_code = 500, detail = str(e)) + audio_b64 = base64.b64encode(wav_bytes).decode("ascii") + return JSONResponse( + content = { + "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", + "object": "chat.completion.audio", + "model": model_name, + "audio": {"data": audio_b64, "format": "wav", "sample_rate": sample_rate}, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": f'[Generated audio from: "{text[:100]}"]', + }, + "finish_reason": "stop", + } + ], + } + ) + # ===================================================================== # OpenAI-Compatible Chat Completions (/chat/completions) @@ -644,11 +681,11 @@ def _extract_content_parts( (``[{type: "text", ...}, {type: "image_url", ...}]``). Returns: - system_prompt: The system message text (or a default). + system_prompt: The system message text (empty string if none provided). chat_messages: Non-system messages with content flattened to strings. image_base64: Base64 data of the *first* image found, or ``None``. """ - system_prompt = "You are a helpful AI assistant." + system_prompt = "" chat_messages: list[dict] = [] first_image_b64: Optional[str] = None @@ -714,6 +751,8 @@ async def openai_chat_completions( # ── Determine which backend is active ───────────────────── if using_gguf: model_name = llama_backend.model_identifier or payload.model + if getattr(llama_backend, "_is_audio", False): + return await generate_audio(payload, request) else: backend = get_inference_backend() if not backend.active_model_name: @@ -903,6 +942,7 @@ async def openai_chat_completions( max_tokens = payload.max_tokens, repetition_penalty = payload.repetition_penalty, cancel_event = cancel_event, + enable_thinking = payload.enable_thinking, ) if payload.stream: diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 868f3aaf33..fa0daea7fd 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -304,6 +304,22 @@ async def list_models( ) loaded_models.append(model_info) + # Include active GGUF model (loaded via llama-server) + from routes.inference import get_llama_cpp_backend + + llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded and llama_backend.model_identifier: + loaded_models.append( + ModelDetails( + id = llama_backend.model_identifier, + name = llama_backend.model_identifier.split("/")[-1], + is_gguf = True, + is_vision = llama_backend.is_vision, + is_audio = getattr(llama_backend, "_is_audio", False), + audio_type = getattr(llama_backend, "_audio_type", None), + ) + ) + # Combine default and loaded models all_models = [] seen_ids = set() diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py index 5b92efce6e..add51bce9b 100644 --- a/studio/backend/utils/inference/inference_config.py +++ b/studio/backend/utils/inference/inference_config.py @@ -6,10 +6,12 @@ Inference configuration loading utilities. This module provides functions to load inference parameters (temperature, top_p, top_k, min_p) from model YAML configuration files, with fallback to default.yaml. +Includes family-based lookup from inference_defaults.json for GGUF models. """ from pathlib import Path -from typing import Dict, Any +from typing import Dict, Any, Optional +import json import yaml import structlog from loggers import get_logger @@ -18,15 +20,96 @@ from utils.models.model_config import load_model_defaults logger = get_logger(__name__) +# ── Family-based inference defaults (loaded once, cached) ────────────── + +_FAMILY_DEFAULTS: Optional[Dict[str, Any]] = None +_FAMILY_PATTERNS: Optional[list] = None + + +def _load_family_defaults(): + """Load and cache inference_defaults.json.""" + global _FAMILY_DEFAULTS, _FAMILY_PATTERNS + if _FAMILY_DEFAULTS is not None: + return + + json_path = ( + Path(__file__).parent.parent.parent + / "assets" + / "configs" + / "inference_defaults.json" + ) + try: + with open(json_path, "r", encoding = "utf-8") as f: + data = json.load(f) + _FAMILY_DEFAULTS = data.get("families", {}) + _FAMILY_PATTERNS = data.get("patterns", []) + except Exception as e: + logger.warning(f"Failed to load inference_defaults.json: {e}") + _FAMILY_DEFAULTS = {} + _FAMILY_PATTERNS = [] + + +def get_family_inference_params(model_id: str) -> Dict[str, Any]: + """ + Look up recommended inference parameters by model family. + + Extracts the model family from the identifier (e.g. "unsloth/Qwen3.5-9B-GGUF" -> "qwen3.5") + and returns the matching parameters from inference_defaults.json. + + Args: + model_id: Model identifier (e.g. "unsloth/Qwen3.5-9B-GGUF") + + Returns: + Dict with inference params, or empty dict if no family match. + """ + _load_family_defaults() + + if not _FAMILY_PATTERNS or not _FAMILY_DEFAULTS: + return {} + + # Normalize: lowercase, strip org prefix + normalized = model_id.lower() + if "/" in normalized: + normalized = normalized.split("/", 1)[1] + + # Match against patterns (ordered longest-match-first in the JSON) + for pattern in _FAMILY_PATTERNS: + if pattern in normalized: + params = _FAMILY_DEFAULTS.get(pattern, {}) + if params: + return dict(params) + + return {} + + +def _has_specific_yaml(model_identifier: str) -> bool: + """Check if a model has its own YAML config (not just default.yaml).""" + from utils.models.model_config import _REVERSE_MODEL_MAPPING + + script_dir = Path(__file__).parent.parent.parent + defaults_dir = script_dir / "assets" / "configs" / "model_defaults" + + # Check the mapping + if model_identifier.lower() in _REVERSE_MODEL_MAPPING: + return True + + # Check for exact filename match + model_filename = model_identifier.replace("/", "_") + ".yaml" + for config_path in defaults_dir.rglob(model_filename): + if config_path.is_file(): + return True + + return False + def load_inference_config(model_identifier: str) -> Dict[str, Any]: """ Load inference configuration parameters for a model. - This function loads inference parameters (temperature, top_p, top_k, min_p) from the - model's YAML configuration file using the same mapping logic as the /config endpoint. - If a parameter is missing from the model's config, it falls back to the value in - default.yaml. + Priority chain: + 1. Model-specific YAML (if it exists and has inference params) + 2. Family-based defaults from inference_defaults.json + 3. default.yaml fallback Args: model_identifier: Model identifier (e.g., "unsloth/llama-3-8b-bnb-4bit") @@ -57,15 +140,35 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]: except Exception as e: logger.warning(f"Failed to load default.yaml: {e}") - # Extract inference parameters from model config, fallback to defaults + # Family-based defaults from inference_defaults.json + family_params = get_family_inference_params(model_identifier) + model_inference = model_defaults.get("inference", {}) + + # If the model has its own YAML config, those values take priority over family defaults. + # If it only fell back to default.yaml, family defaults take priority. + has_own_yaml = _has_specific_yaml(model_identifier) + + def _get_param(key, hardcoded_default): + if has_own_yaml: + # Model-specific YAML wins, then family fills gaps, then default.yaml + val = model_inference.get(key) + if val is not None and isinstance(val, (int, float)): + return val + if key in family_params: + return family_params[key] + return default_inference.get(key, hardcoded_default) + else: + # No model-specific YAML: family wins, then default.yaml + if key in family_params: + return family_params[key] + return default_inference.get(key, hardcoded_default) + inference_config = { - "temperature": model_inference.get( - "temperature", default_inference.get("temperature", 0.7) - ), - "top_p": model_inference.get("top_p", default_inference.get("top_p", 0.95)), - "top_k": model_inference.get("top_k", default_inference.get("top_k", -1)), - "min_p": model_inference.get("min_p", default_inference.get("min_p", 0.01)), + "temperature": _get_param("temperature", 0.7), + "top_p": _get_param("top_p", 0.95), + "top_k": _get_param("top_k", -1), + "min_p": _get_param("min_p", 0.01), "trust_remote_code": model_inference.get( "trust_remote_code", default_inference.get("trust_remote_code", False) ), diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index c8f8881808..2284cd194f 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -340,6 +340,14 @@ function isGgufRepo(id: string): boolean { return id.toUpperCase().includes("-GGUF"); } +/** Extract param count label from model name (e.g. "Qwen3-0.6B" -> "0.6B"). */ +function extractParamLabel(id: string): string | undefined { + // Match patterns like "0.6B", "1B", "4B", "3.5B", "70B", "1.5B" etc. + const name = id.split("/").pop() ?? id; + const match = name.match(/(?:^|[-_])(\d+(?:\.\d+)?)[Bb](?:[-_]|$)/); + return match ? `${match[1]}B` : undefined; +} + // Module-level caches so re-mounting the popover shows results instantly let _cachedGgufCache: CachedGgufRepo[] = []; let _cachedModelsCache: CachedModelRepo[] = []; @@ -558,7 +566,7 @@ export function HubModelPicker({ meta={ isGgufRepo(id) ? "GGUF" - : vram?.detail ?? undefined + : vram?.detail ?? extractParamLabel(id) } selected={value === id} onClick={() => handleModelClick(id)} @@ -593,7 +601,7 @@ export function HubModelPicker({ meta={ isGgufRepo(id) ? "GGUF" - : metricsById.get(id) + : metricsById.get(id) ?? extractParamLabel(id) } selected={value === id} onClick={() => handleModelClick(id)} diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index 0111c2085a..d593d52d54 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -40,6 +40,8 @@ import { CopyIcon, DownloadIcon, HeadphonesIcon, + LightbulbIcon, + LightbulbOffIcon, MicIcon, MoreHorizontalIcon, PencilIcon, @@ -262,12 +264,42 @@ const ComposerAudioUpload: FC = () => { ); }; +const ReasoningToggle: FC = () => { + const supportsReasoning = useChatRuntimeStore((s) => s.supportsReasoning); + const reasoningEnabled = useChatRuntimeStore((s) => s.reasoningEnabled); + const setReasoningEnabled = useChatRuntimeStore((s) => s.setReasoningEnabled); + + if (!supportsReasoning) return null; + + return ( + + ); +}; + const ComposerAction: FC = () => { return (
+
diff --git a/studio/frontend/src/components/navbar.tsx b/studio/frontend/src/components/navbar.tsx index e778b013f2..00693fa064 100644 --- a/studio/frontend/src/components/navbar.tsx +++ b/studio/frontend/src/components/navbar.tsx @@ -66,7 +66,7 @@ export function Navbar() {
{/* Left: logo */} - + Unsloth - + BETA diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 30a8d97461..3710cd795c 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -86,10 +86,17 @@ function toOpenAIMessage(message: RunMessage): { return null; } - return { - role: message.role, - content: collectTextParts(message).join("\n"), - }; + let content = collectTextParts(message).join("\n"); + // Strip inline audio base64 from prior assistant messages to avoid + // inflating token counts (e.g. audio-player responses with embedded WAV). + if (message.role === "assistant") { + content = content.replace( + /data:audio\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g, + "[audio]", + ); + } + + return { role: message.role, content }; } function extractImageBase64(input: string): string | undefined { @@ -194,7 +201,8 @@ function waitForModelReady(abortSignal?: AbortSignal): Promise { async function autoLoadSmallestModel(): Promise { const toastId = toast("Loading a model…", { description: "Auto-selecting the smallest downloaded model.", - duration: Infinity, + duration: 5000, + closeButton: true, }); try { const [ggufRepos, modelRepos] = await Promise.all([ @@ -214,7 +222,7 @@ async function autoLoadSmallestModel(): Promise { .sort((a, b) => a.size_bytes - b.size_bytes); if (downloaded.length > 0) { const variant = downloaded[0]; - await loadModel({ + const loadResp = await loadModel({ model_path: repo.repo_id, hf_token: null, max_seq_length: 4096, @@ -223,7 +231,16 @@ async function autoLoadSmallestModel(): Promise { gguf_variant: variant.quant, trust_remote_code: false, }); - useChatRuntimeStore.getState().setCheckpoint(repo.repo_id, variant.quant); + const store = useChatRuntimeStore.getState(); + store.setCheckpoint(repo.repo_id, variant.quant); + store.setParams({ ...store.params, maxTokens: loadResp.context_length ?? 131072 }); + useChatRuntimeStore.setState({ + ggufContextLength: loadResp.context_length ?? 131072, + supportsReasoning: loadResp.supports_reasoning ?? false, + reasoningEnabled: loadResp.supports_reasoning ?? false, + defaultChatTemplate: loadResp.chat_template ?? null, + chatTemplateOverride: null, + }); toast.success(`Loaded ${repo.repo_id} (${variant.quant})`, { id: toastId }); return true; } @@ -247,7 +264,9 @@ async function autoLoadSmallestModel(): Promise { gguf_variant: null, trust_remote_code: false, }); - useChatRuntimeStore.getState().setCheckpoint(repo.repo_id); + const store = useChatRuntimeStore.getState(); + store.setCheckpoint(repo.repo_id); + store.setParams({ ...store.params, maxTokens: 4096 }); toast.success(`Loaded ${repo.repo_id}`, { id: toastId }); return true; } catch { @@ -411,6 +430,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { let reasoningDuration = 0; try { + const { supportsReasoning, reasoningEnabled } = runtime; const stream = streamChatCompletions( { model: params.checkpoint, @@ -425,6 +445,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { image_base64: imageBase64, audio_base64: audioBase64, ...(useAdapter === undefined ? {} : { use_adapter: useAdapter }), + ...(supportsReasoning ? { enable_thinking: reasoningEnabled } : {}), }, abortSignal, ); diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index b95a3a8643..8c7f1de6bc 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -650,6 +650,18 @@ export function ChatPage(): ReactElement { onParamsChange={setInferenceParams} autoTitle={autoTitle} onAutoTitleChange={setAutoTitle} + onReloadModel={() => { + const state = useChatRuntimeStore.getState(); + if (state.params.checkpoint) { + selectModel({ + id: state.params.checkpoint, + ggufVariant: state.activeGgufVariant ?? undefined, + forceReload: true, + isDownloaded: true, + loadingDescription: "Reloading with updated chat template.", + }); + } + }} />
diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index bb2b05f9da..170429a44b 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -12,6 +12,7 @@ import { Slider } from "@/components/ui/slider"; import { Textarea } from "@/components/ui/textarea"; import { ArrowDown01Icon, + CodeIcon, Delete02Icon, FloppyDiskIcon, PencilEdit01Icon, @@ -152,6 +153,7 @@ interface ChatSettingsPanelProps { onParamsChange: (params: InferenceParams) => void; autoTitle: boolean; onAutoTitleChange: (enabled: boolean) => void; + onReloadModel?: () => void; } export function ChatSettingsPanel({ @@ -160,8 +162,10 @@ export function ChatSettingsPanel({ onParamsChange, autoTitle, onAutoTitleChange, + onReloadModel, }: ChatSettingsPanelProps) { const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; + const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); const [presets, setPresets] = useState(BUILTIN_PRESETS); const [activePreset, setActivePreset] = useState("Default"); const isBuiltinPreset = BUILTIN_PRESETS.some((p) => p.name === activePreset); @@ -340,15 +344,19 @@ export function ChatSettingsPanel({ label="Max Tokens" value={params.maxTokens} min={64} - max={isGguf ? 131072 : 32768} + max={isGguf && ggufContextLength ? ggufContextLength : 32768} step={64} onChange={set("maxTokens")} - displayValue={isGguf && params.maxTokens >= 131072 ? "Max" : undefined} + displayValue={ + isGguf && ggufContextLength && params.maxTokens >= ggufContextLength + ? "Max" + : undefined + } />
- +
@@ -376,8 +384,61 @@ export function ChatSettingsPanel({
+ +
); } + +function ChatTemplateSection({ + onReloadModel, +}: { + onReloadModel?: () => void; +}) { + const defaultTemplate = useChatRuntimeStore((s) => s.defaultChatTemplate); + const override = useChatRuntimeStore((s) => s.chatTemplateOverride); + const setOverride = useChatRuntimeStore((s) => s.setChatTemplateOverride); + + if (!defaultTemplate) return null; + + const displayValue = override ?? defaultTemplate; + const isModified = override !== null; + + return ( + +
+