From a5d6e6928d4b7bbffb1b8516c07bdeda8a294cb4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 11 Jun 2026 06:10:17 -0700 Subject: [PATCH] Studio: surface the llama.cpp update affordance when MTP is disabled (#6192) * Studio: surface the llama.cpp update affordance when MTP is disabled When a model asks for MTP (auto on an MTP model, or forced mtp / mtp+ngram) but it gets disabled, the load already degrades gracefully and serves without speculative decoding. Until now the UI gave no hint why, or that an update would fix it. Record why MTP was dropped on the backend (spec_fallback_reason): the probe found no mtp token (binary_no_mtp), the spawn aborted with an outdated-arch / context-build error such as a prebuilt that predates the Gemma drafter (binary_outdated), or the current build could not run it, e.g. a CUDA kernel limit (runtime_error). Expose it in the inference status. In the chat Speculative Decoding section, show a short note and, for the two update-fixable reasons, an inline Update llama.cpp button that reuses the existing update flow. A runtime_error gets the note without an update push, since a newer build may not fix it. Backend tests cover the reason being set / cleared. Frontend typechecks. * Address review: tighten the update hint to genuinely outdated binaries Reserve binary_outdated (which surfaces the Update llama.cpp affordance) for an unknown-architecture abort, which proves the prebuilt predates the model; classify the generic memory/context build failures as runtime_error, where an update may not help. Frontend: only append the "Update llama.cpp to enable it" sentence when an update is actually available, so the text never points at an action the UI is not offering. --- studio/backend/core/inference/llama_cpp.py | 24 +++++++- studio/backend/models/inference.py | 11 ++++ studio/backend/routes/inference.py | 1 + .../tests/test_llama_cpp_mtp_detection.py | 59 +++++++++++++++++++ .../src/features/chat/chat-settings-sheet.tsx | 50 +++++++++++++++- .../chat/hooks/use-chat-model-runtime.ts | 1 + .../chat/stores/chat-runtime-store.ts | 7 +++ .../frontend/src/features/chat/types/api.ts | 6 ++ 8 files changed, 157 insertions(+), 2 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7d67097d09..e46d464784 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -676,6 +676,11 @@ class LlamaCppBackend: # Separate MTP drafter launched with the current model; reload-dedup # key so a drafter that appears next to the weights forces a reload. self._mtp_draft_path: Optional[str] = None + # Why MTP was disabled on the last load that asked for it (auto on an + # MTP model, or forced mtp / mtp+ngram), else None. Drives the "update + # llama.cpp" hint in the UI. "binary_no_mtp" / "binary_outdated" -> + # a newer prebuilt would help; "runtime_error" -> it may not. + self._spec_fallback_reason: Optional[str] = None self._hf_variant: Optional[str] = None self._is_vision: bool = False self._healthy = False @@ -794,6 +799,11 @@ class LlamaCppBackend: def mtp_draft_path(self) -> Optional[str]: return self._mtp_draft_path + @property + def spec_fallback_reason(self) -> Optional[str]: + """Why MTP was disabled on the last MTP-requesting load, else None.""" + return self._spec_fallback_reason + @property def extra_args(self) -> Optional[List[str]]: """Extra llama-server flags from the last load (a copy). None = @@ -3594,8 +3604,13 @@ class LlamaCppBackend: # failing (unknown arch / draft or context build); an # unrelated crash (e.g. OOM) gets a neutral message. _lo = "\n".join(self._stdout_lines).lower() + # Only an unknown architecture proves the prebuilt predates + # this MTP model (an update fixes it). The memory/context + # build failures are generic (VRAM / ctx pressure), where an + # update may not help, so classify those as runtime_error. + _arch_unsupported = "unknown model architecture" in _lo if ( - "unknown model architecture" in _lo + _arch_unsupported or "failed to measure draft model memory" in _lo or "failed to measure mtp context memory" in _lo or "failed to create llama_context" in _lo @@ -3605,10 +3620,14 @@ class LlamaCppBackend: "speculative decoding -- run `unsloth studio " "update` for MTP" ) + self._spec_fallback_reason = ( + "binary_outdated" if _arch_unsupported else "runtime_error" + ) else: _retry_reason = ( "retrying without speculative decoding in case MTP is the cause" ) + self._spec_fallback_reason = "runtime_error" _drafter = ( Path(launch_mtp_draft_path).name if launch_mtp_draft_path @@ -3804,6 +3823,7 @@ class LlamaCppBackend: # Reset; emit branches re-set on the resolved emission. self._spec_draft_n_max = None self._speculative_type = None + self._spec_fallback_reason = None # Canonical UI-facing requested mode (legacy values mapped via # _canonicalize_spec_mode). @@ -3853,6 +3873,7 @@ class LlamaCppBackend: "run `unsloth studio update`. Loading without " "speculative decoding." ) + self._spec_fallback_reason = "binary_no_mtp" return False draft_n_max = _resolved_draft_n_max() n_max_flag = caps.get("spec_draft_n_max_flag") or "--spec-draft-n-max" @@ -4118,6 +4139,7 @@ class LlamaCppBackend: self._gguf_path = None self._hf_repo = None self._mtp_draft_path = None + self._spec_fallback_reason = None self._hf_variant = None self._is_vision = False self._is_audio = False diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index c83c8ecc1b..f58a76d16e 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -346,6 +346,17 @@ class InferenceStatusResponse(BaseModel): "False -> recommend `unsloth studio update`." ), ) + spec_fallback_reason: Optional[str] = Field( + None, + description = ( + "Why MTP was disabled on the loaded model despite being requested " + "(auto on an MTP model, or forced mtp / mtp+ngram). " + "'binary_no_mtp' / 'binary_outdated' -> a newer prebuilt would " + "re-enable it (show the update affordance); 'runtime_error' -> the " + "current build could not run it. None when MTP engaged or was not " + "requested." + ), + ) llama_cpp_prebuilt_stale: bool = Field( False, description = ( diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index ef67f8bb59..0244956b72 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1906,6 +1906,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)): speculative_type = llama_backend.requested_spec_mode, spec_draft_n_max = llama_backend.spec_draft_n_max, llama_cpp_supports_mtp = _supports_mtp, + spec_fallback_reason = llama_backend.spec_fallback_reason, llama_cpp_prebuilt_stale = _stale, llama_cpp_installed_tag = _installed_tag, llama_cpp_latest_tag = _latest_tag, diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.py index a7f30d2951..b00cd7169e 100644 --- a/studio/backend/tests/test_llama_cpp_mtp_detection.py +++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py @@ -1399,3 +1399,62 @@ def test_auto_mode_drops_mtp_exempts_separate_drafter(): assert _auto_mode_drops_mtp("auto", 2.0, has_separate_drafter = True) is False assert _auto_mode_drops_mtp("auto", 4.0) is False assert _auto_mode_drops_mtp("mtp", 2.0) is False # forced engages regardless + + +# ── spec_fallback_reason (drives the "update llama.cpp" UI hint) ─────── + + +def test_spec_fallback_reason_set_when_binary_lacks_mtp(monkeypatch): + # Outdated llama-server with no mtp token: a forced MTP request can't emit + # draft-mtp, so record the reason for the UI update affordance. + backend = _resolver_backend(monkeypatch, mtp_token = None) + backend._build_speculative_flags( + speculative_type = "mtp", + spec_draft_n_max = None, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + assert backend.spec_fallback_reason == "binary_no_mtp" + + +def test_spec_fallback_reason_none_when_mtp_engages(monkeypatch): + backend = _resolver_backend(monkeypatch) + backend._build_speculative_flags( + speculative_type = "auto", + spec_draft_n_max = None, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + assert backend.speculative_type == "draft-mtp" + assert backend.spec_fallback_reason is None + + +def test_spec_fallback_reason_reset_on_off(monkeypatch): + # A subsequent off load must clear a stale reason. + backend = _resolver_backend(monkeypatch, mtp_token = None) + backend._build_speculative_flags( + speculative_type = "mtp", + spec_draft_n_max = None, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + assert backend.spec_fallback_reason == "binary_no_mtp" + backend._build_speculative_flags( + speculative_type = "off", + spec_draft_n_max = None, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + assert backend.spec_fallback_reason is None diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 0bd9d8eee6..03c183fe76 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -50,6 +50,7 @@ import { TooltipTrigger, } from "@/components/ui/tooltip"; import { useIsMobile } from "@/hooks/use-mobile"; +import { useLlamaUpdateCheck } from "@/hooks/use-llama-update-check"; import { cn } from "@/lib/utils"; import { ArrowDown01Icon, @@ -62,7 +63,7 @@ import { HugeiconsIcon } from "@hugeicons/react"; import { ChevronDown, ExternalLink } from "lucide-react"; import { Tooltip as TooltipPrimitive } from "radix-ui"; import { Fragment, type ReactNode } from "react"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "@/lib/toast"; import { OpenAICodeExecSection } from "./components/openai-code-exec-section"; import { @@ -487,6 +488,27 @@ export function ChatSettingsPanel({ const loadedSpeculativeType = useChatRuntimeStore( (s) => s.loadedSpeculativeType, ); + const specFallbackReason = useChatRuntimeStore((s) => s.specFallbackReason); + // "binary_no_mtp" / "binary_outdated" mean a newer prebuilt would re-enable + // MTP; "runtime_error" means the current build cannot run it (no update push). + const mtpUpdatable = + specFallbackReason === "binary_no_mtp" || + specFallbackReason === "binary_outdated"; + const { + status: llamaUpdateStatus, + applying: llamaUpdating, + apply: applyLlamaUpdate, + } = useLlamaUpdateCheck({ enabled: mtpUpdatable }); + const handleMtpUpdate = useCallback(async () => { + const result = await applyLlamaUpdate(); + if (result.ok) { + toast.success( + `llama.cpp updated to ${result.tag ?? "the latest build"}. Reload your model to enable MTP.`, + ); + } else { + toast.error(`llama.cpp update failed: ${result.error ?? "unknown error"}`); + } + }, [applyLlamaUpdate]); const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax); const setSpecDraftNMax = useChatRuntimeStore((s) => s.setSpecDraftNMax); const loadedSpecDraftNMax = useChatRuntimeStore( @@ -923,6 +945,32 @@ export function ChatSettingsPanel({ + {specFallbackReason && + (speculativeType === "auto" || + speculativeType === "mtp" || + speculativeType === "mtp+ngram") && ( +
+

+ {specFallbackReason === "runtime_error" + ? "MTP could not start for this model on the installed llama.cpp build, so it is running without speculative decoding." + : "MTP is not available in the installed llama.cpp build, so this model is running without it." + + (llamaUpdateStatus?.update_available + ? " Update llama.cpp to enable it." + : "")} +

+ {mtpUpdatable && llamaUpdateStatus?.update_available && ( + + )} +
+ )} {(speculativeType === "mtp" || speculativeType === "mtp+ngram") && (
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 66ee2d04e4..4bd36e00d4 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -412,6 +412,7 @@ export function useChatModelRuntime() { statusRes.requires_trust_remote_code ?? false, defaultChatTemplate: nextDefaultChatTemplate, loadedIsMultimodal: isMultimodalResponse(statusRes), + specFallbackReason: statusRes.spec_fallback_reason ?? null, ...(prevState.loadedSpeculativeType === null && { speculativeType: currentSpecType, loadedSpeculativeType: currentSpecType, diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index aabf1b29a6..3ef63d2945 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -416,6 +416,11 @@ type ChatRuntimeStore = { loadedKvCacheDtype: string | null; speculativeType: string | null; loadedSpeculativeType: string | null; + /** + * Why MTP was disabled on the loaded model despite being requested, or null. + * Mirrors InferenceStatusResponse.spec_fallback_reason. + */ + specFallbackReason: string | null; /** User --spec-draft-n-max override (null = platform default). */ specDraftNMax: number | null; loadedSpecDraftNMax: number | null; @@ -765,6 +770,7 @@ export const useChatRuntimeStore = create((set, get) => ({ loadedKvCacheDtype: null, speculativeType: "auto", loadedSpeculativeType: null, + specFallbackReason: null, specDraftNMax: null, loadedSpecDraftNMax: null, loadedIsMultimodal: false, @@ -977,6 +983,7 @@ export const useChatRuntimeStore = create((set, get) => ({ loadedKvCacheDtype: null, speculativeType: "auto", loadedSpeculativeType: null, + specFallbackReason: null, specDraftNMax: null, loadedSpecDraftNMax: null, loadedIsMultimodal: false, diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 49524c17b4..a0e5d5355e 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -174,6 +174,12 @@ export interface InferenceStatusResponse { /** Canonical UI-facing mode currently active. See LoadModelRequest. */ speculative_type?: string | null; spec_draft_n_max?: number | null; + /** + * Why MTP was disabled on the loaded model despite being requested. + * "binary_no_mtp" / "binary_outdated" -> updating llama.cpp would re-enable + * it; "runtime_error" -> the current build could not run it. Null otherwise. + */ + spec_fallback_reason?: string | null; } export interface AudioGenerationResponse {