diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 8f19973cda..5622fed574 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -652,6 +652,10 @@ _TOOL_TEMPLATE_MARKERS = ( "{%- if tools -%}", "{% if tools %}", "{% if tools -%}", + # Defensive templates guard with `tools is defined` before truth-testing + # (e.g. Inkling: `{%- if tools is defined and tools -%}`). + "{%- if tools is defined", + "{% if tools is defined", '"role" == "tool"', "'role' == 'tool'", 'message.role == "tool"', @@ -667,9 +671,10 @@ _TOOL_TEMPLATE_MARKERS = ( # Canonical reasoning_effort levels, weakest -> strongest. Used to read the -# discrete set a template branches on (e.g. GLM-5.2 uses 'high' | 'max') so we -# only ever offer levels the template actually understands. -_REASONING_EFFORT_SCALE = ("minimal", "low", "medium", "high", "max") +# discrete set a template branches on (e.g. GLM-5.2 uses 'high' | 'max', Inkling +# uses the full 'none'..'max' ladder) so we only ever offer levels the template +# actually understands. +_REASONING_EFFORT_SCALE = ("none", "minimal", "low", "medium", "high", "xhigh", "max") def _extract_reasoning_effort_levels(chat_template: str) -> list: @@ -751,10 +756,22 @@ def detect_reasoning_flags( logger.info(f"{prefix}model supports reasoning (enable_thinking)") elif "reasoning_effort" in tpl: # gpt-oss / Harmony use reasoning_effort - # ("low" | "medium" | "high"), not a boolean. + # ("low" | "medium" | "high"), not a boolean. Inkling maps a wider + # named ladder ('none'..'max'); surface the levels the template + # actually branches on so the Think menu offers exactly those and + # nothing more ('none' renders as thinking off: effort 0). Guard: + # trust the scan only when it includes the core 'low' and 'high' + # literals, so a template that quotes e.g. 'none' for an unrelated + # comparison keeps the default low/medium/high set. flags["supports_reasoning"] = True flags["reasoning_style"] = "reasoning_effort" - logger.info(f"{prefix}model supports reasoning (reasoning_effort)") + scanned = _extract_reasoning_effort_levels(tpl) + if "low" in scanned and "high" in scanned: + flags["reasoning_effort_levels"] = scanned + logger.info( + f"{prefix}model supports reasoning " + f"(reasoning_effort: {flags['reasoning_effort_levels'] or 'default levels'})" + ) elif "thinking" in tpl: # DeepSeek uses 'thinking', not 'enable_thinking' normalized_id = (model_identifier or "").lower() @@ -1524,6 +1541,25 @@ def _is_external_link(path: Path) -> bool: return False +# Inkling's template takes a numeric thinking-effort dial (0..0.99) and its +# float() coercion turns unrecognized named levels into 0, i.e. no thinking. +# Map OpenAI-style names to the values the model was trained on. Module-level +# so duck-typed engine stand-ins in tests do not need the attribute. +_INKLING_REASONING_EFFORT = { + "none": 0.0, "minimal": 0.2, "low": 0.2, "medium": 0.7, + "high": 0.9, "xhigh": 0.99, "max": 0.99, +} + +def _coerce_reasoning_effort(architecture, kwargs: dict) -> dict: + if architecture == "inkling": + effort = kwargs.get("reasoning_effort") + if isinstance(effort, str): + mapped = _INKLING_REASONING_EFFORT.get(effort.strip().lower()) + if mapped is not None: + kwargs["reasoning_effort"] = mapped + return kwargs + + class LlamaCppBackend: """Manages a llama-server subprocess for GGUF model inference. @@ -1930,7 +1966,9 @@ class LlamaCppBackend: # the template's default effort (max) in place. return {"enable_thinking": enable_thinking} if self._reasoning_style == "reasoning_effort": - return {"reasoning_effort": "high" if enable_thinking else "low"} + return _coerce_reasoning_effort( + getattr(self, "_architecture", None), + {"reasoning_effort": "high" if enable_thinking else "low"}) return {"enable_thinking": enable_thinking} def _request_reasoning_kwargs( @@ -1978,6 +2016,7 @@ class LlamaCppBackend: kwargs["enable_thinking"] = enable_thinking if self._supports_preserve_thinking and preserve_thinking is not None: kwargs["preserve_thinking"] = preserve_thinking + _coerce_reasoning_effort(getattr(self, "_architecture", None), kwargs) return kwargs or None @property @@ -3505,6 +3544,19 @@ class LlamaCppBackend: _DSV4_CTX_COMPUTE_FLAT_BYTES = 2 * 1024**3 # ctx-independent indexer scratch _DSV4_CTX_COMPUTE_BYTES_PER_TOK = 72000 # per token at ub=512 (~72 GiB at 1M) + # Inkling (inkling): with the reserve fix in the bundled llama.cpp (full-cache + # reserve context reports the whole cache as position-contiguous, so the + # worst-case graph is the banded flash path instead of the dense-bias + # fallback), the ctx-linear compute term is just the banded KQ-mask cont: + # measured on UD-IQ1_S (ub=512) 64K -> 1M total-VRAM slope of 50.6 KiB/tok, + # of which 45,056 B/tok is KV -> ~5.6 KiB/tok compute. 8192 adds ~1.5x + # headroom. (Pre-fix builds reserved the dense fallback at ~402 KiB/tok and + # could not load large contexts at all.) + _INKLING_CTX_COMPUTE_BYTES_PER_TOK = 8192 # per token at ub=512 + # Dense relative-bias fallback rate (quantized KV cache disables the banded + # path): measured 402.5 GiB reserve at 1M ctx pre-fix, ~402 KiB per token. + _INKLING_CTX_COMPUTE_DENSE_BYTES_PER_TOK = 402470 # per token at ub=512 + def _estimate_compute_buffer_bytes( self, *, @@ -3561,6 +3613,17 @@ class LlamaCppBackend: self._DSV4_CTX_COMPUTE_FLAT_BYTES + self._DSV4_CTX_COMPUTE_BYTES_PER_TOK * n_ctx * ub_scale ) + if getattr(self, "_architecture", None) == "inkling": + ub_scale = ub / self._DEFAULT_N_UBATCH + # The fused banded path requires an f32/f16/bf16 KV cache. A quantized + # cache forces the dense relative-bias fallback, whose compute buffer is + # [n_kv, ub, n_head] f32 (~402 KiB per context token at ub=512); size the + # fit for that so a q8_0 cache gets a small honest context instead of an + # unloadable one that crash-loops the server. + if cache_type_kv and _kv_bytes_per_elem(cache_type_kv) < 2.0: + return int(self._INKLING_CTX_COMPUTE_DENSE_BYTES_PER_TOK * n_ctx * ub_scale) + # Banded flash path (see constants): linear, ub-scaled. + return int(self._INKLING_CTX_COMPUTE_BYTES_PER_TOK * n_ctx * ub_scale) if _kv_bytes_per_elem(cache_type_kv) < 2.0: # Quantized cache: the dequant scratch dominates and scales with n_embd. # MLA (compressed KV) needs far less of it: measured 0.94 x n_embd on diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.py index a444431f8d..ed7c7ecfcf 100644 --- a/studio/backend/core/inference/passthrough_healing.py +++ b/studio/backend/core/inference/passthrough_healing.py @@ -41,6 +41,9 @@ _HEAL_SIGNALS = ( "<|tool_call>", "", ) diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.py index 1ab1142eba..9b6b0a7773 100644 --- a/studio/backend/core/inference/tool_call_parser.py +++ b/studio/backend/core/inference/tool_call_parser.py @@ -54,6 +54,8 @@ TOOL_XML_SIGNALS = ( # Kimi K2 / Moonshot. "<|tool_calls_section_begin|>", "<|tool_call_begin|>", + # TML Inkling native call marker. + "<|content_invoke_tool_json|>", ) diff --git a/studio/backend/core/tool_healing.py b/studio/backend/core/tool_healing.py index 1b6b05768a..94d0e40ea7 100644 --- a/studio/backend/core/tool_healing.py +++ b/studio/backend/core/tool_healing.py @@ -124,10 +124,12 @@ def apply_tool_strip_patterns( # Pre-compiled patterns for tool-call XML parsing. -_TC_JSON_START_RE = re.compile(r"\s*\{") +# <|content_invoke_tool_json|> is TML Inkling's native call marker; its JSON uses +# an ``args`` key and the block closes with <|end_message|>. +_TC_JSON_START_RE = re.compile(r"(?:|<\|content_invoke_tool_json\|>)\s*\{") _TC_GEMMA_START_RE = re.compile(r"<\|tool_call>\s*call\s*:\s*([\w.\-]+)\s*\{") _TC_FUNC_START_RE = re.compile(r"\s*") -_TC_END_TAG_RE = re.compile(r"") +_TC_END_TAG_RE = re.compile(r"|<\|end_message\|>") _TC_GEMMA_END_TAG_RE = re.compile(r"") _TC_FUNC_CLOSE_RE = re.compile(r"\s*\s*$") # Horizontal-whitespace trailing class keeps the wrapping newline; _trim_param_value trims it. @@ -686,12 +688,24 @@ def parse_tool_calls_from_text( if kind == "json": obj = json.loads(content[m.end() - 1 : brace_end + 1]) name = obj.get("name", "") - # Accept ``parameters`` alias for ``arguments`` (Llama-3.2 drift inside Hermes). + # Accept ``parameters`` alias for ``arguments`` (Llama-3.2 drift inside + # Hermes) and ``args`` (TML Inkling native calls). arguments = obj.get("arguments") if arguments is None: - arguments = obj.get("parameters", {}) + arguments = obj.get("parameters") + if arguments is None: + arguments = obj.get("args", {}) if isinstance(arguments, dict): arguments = json.dumps(arguments) + # Inkling echoes the bare tool name (and a role opener) before the + # marker: <|message_model|>NAME<|content_invoke_tool_json|>{...}. + # Fold that echo into the markup span so promotion removes it too. + if name and content.startswith("<|content_invoke_tool_json|>", start): + pre = content[:start] + if pre.endswith(name): + start -= len(name) + if content[:start].endswith("<|message_model|>"): + start -= len("<|message_model|>") else: name = m.group(1) arguments = json.dumps(_gemma_arguments_to_json(content[m.end() : brace_end])) diff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.py index 44c39337fb..50e20fc13f 100644 --- a/studio/backend/hub/services/download_lifecycle.py +++ b/studio/backend/hub/services/download_lifecycle.py @@ -75,6 +75,10 @@ def spawn_worker( env["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" env["HF_HUB_DISABLE_TELEMETRY"] = "1" env["HF_HUB_DISABLE_XET"] = "0" if use_xet else "1" + # No token in Studio settings: fall back to the backend's own HF_TOKEN so + # private repos stay downloadable (needed while inkling repos are private). + if not hf_token: + hf_token = os.environ.get("HF_TOKEN") or None env["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "0" if hf_token else "1" # hf_transfer's parallel Range chunks can leave sparse partials even in # "http" mode; disable so the worker's writer is always sequential. diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py index 9b22ab8b05..da261e8d0d 100644 --- a/studio/backend/tests/test_passthrough_healing.py +++ b/studio/backend/tests/test_passthrough_healing.py @@ -1422,6 +1422,7 @@ class TestHealerSignalAlignment: "<|tool_call>", "", } def test_prose_with_bare_args_marker_streams_through(self): diff --git a/studio/backend/tests/test_tool_call_parser_strict.py b/studio/backend/tests/test_tool_call_parser_strict.py index c6da1e90e7..c8556b19d7 100644 --- a/studio/backend/tests/test_tool_call_parser_strict.py +++ b/studio/backend/tests/test_tool_call_parser_strict.py @@ -874,7 +874,7 @@ class TestHealerSignalAlignment: def test_heal_signals_subset_of_promotable_formats(self): from core.inference.passthrough_healing import _HEAL_SIGNALS - assert set(_HEAL_SIGNALS) == {"", "<|tool_call>", "", "<|tool_call>", ""} def test_stream_healer_does_not_hold_llama_python_tag_text(self): from core.inference.passthrough_healing import StreamToolCallHealer diff --git a/studio/frontend/src/components/assistant-ui/attachment.tsx b/studio/frontend/src/components/assistant-ui/attachment.tsx index 0a59d75964..98ebe5ab5f 100644 --- a/studio/frontend/src/components/assistant-ui/attachment.tsx +++ b/studio/frontend/src/components/assistant-ui/attachment.tsx @@ -24,7 +24,7 @@ import { useAui, useAuiState, } from "@assistant-ui/react"; -import { File02Icon } from "@hugeicons/core-free-icons"; +import { AudioWave01Icon, File02Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { PlusIcon, XIcon } from "lucide-react"; import { @@ -120,9 +120,20 @@ const AttachmentPreviewDialog: FC = ({ children }) => { ); }; +const AUDIO_ATTACHMENT_RE = /\.(wav|mp3|m4a|ogg|oga|flac|webm|mp4|aac)$/i; + +const isAudioAttachment = (name: string | undefined, contentType: string) => + /^audio\//i.test(contentType) || AUDIO_ATTACHMENT_RE.test(name ?? ""); + const AttachmentThumb: FC = () => { const src = useAttachmentSrc(); const name = useAuiState(({ attachment }) => attachment.name); + const contentType = useAuiState( + ({ attachment }) => + (attachment as { file?: File }).file?.type ?? + (attachment as { contentType?: string }).contentType ?? + "", + ); if (src) { return ( @@ -137,7 +148,7 @@ const AttachmentThumb: FC = () => { return (
@@ -159,7 +170,12 @@ const AttachmentUI: FC = () => { case "document": return "Document"; case "file": - return "File"; + return isAudioAttachment( + attachment.name, + (attachment as { file?: File }).file?.type ?? "", + ) + ? "Audio" + : "File"; default: throw new Error(`Unknown attachment type: ${type as string}`); } diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index 11d91cc952..7f1527d326 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -2315,7 +2315,13 @@ const ReasoningToggle: FC<{ side?: "top" | "bottom" }> = ({ )} {effectiveReasoningEffortLevels - .filter((level) => level !== "none") + // 'none' is a real template level for models like Inkling + // (effort 0 = thinking off); show it as a pick unless the + // dedicated off item above already covers it. + .filter( + (level) => + level !== "none" || !effectiveSupportsReasoningOff, + ) .map((level) => (