diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c320f03b2c..800d7fdd8c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1574,11 +1574,21 @@ class LlamaCppBackend: # ref: https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md # ref: https://github.com/ggml-org/llama.cpp/pull/19164 # ref: https://github.com/ggml-org/llama.cpp/pull/18471 + # ``"default"`` -> let llama-server pick a sensible spec + # config via ``--spec-default``. Explicit type names are + # passed through with the manual draft tuning we've shipped + # historically so power users keep their overrides. _valid_spec_types = {"ngram-simple", "ngram-mod"} - if speculative_type and speculative_type in _valid_spec_types: - if not is_vision: # spec decoding disabled for vision models - cmd.extend(["--spec-type", speculative_type]) - if speculative_type == "ngram-mod": + normalized_spec = ( + speculative_type.lower().strip() if speculative_type else None + ) + if normalized_spec and normalized_spec != "off" and not is_vision: + if normalized_spec == "default": + cmd.append("--spec-default") + self._speculative_type = "default" + elif normalized_spec in _valid_spec_types: + cmd.extend(["--spec-type", normalized_spec]) + if normalized_spec == "ngram-mod": cmd.extend( [ "--spec-ngram-size-n", @@ -1589,7 +1599,7 @@ class LlamaCppBackend: "64", ] ) - self._speculative_type = speculative_type + self._speculative_type = normalized_spec else: self._speculative_type = None else: @@ -1750,6 +1760,12 @@ class LlamaCppBackend: if gpu_indices is not None: env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_indices) + # Defensive kill: if a concurrent load slipped past Phase 1 + # (because its `self._process` was None at the time) and + # already stored a Popen handle here, drop that orphan + # before we overwrite the reference. See issue #5161. + self._kill_process() + self._stdout_lines = [] self._process = subprocess.Popen( cmd, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index e4574dab74..fc5f097969 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -1037,7 +1037,7 @@ export function ChatSettingsPanel({ - On + On Off diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 1281592168..e90cab73fa 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -24,6 +24,19 @@ import type { InferenceParams, } from "../types/runtime"; +// The simplified Speculative Decoding control surfaces "default" (which +// maps to llama.cpp's --spec-default) and "off". A backend status / load +// response can still report the older manual modes (ngram-mod, +// ngram-simple) when a model is loaded via the API or carried over from an +// older Studio version. The Select would render an empty trigger for those +// values, so coerce them to "default" -- llama.cpp's own --spec-default +// picks an equivalent strategy and keeps the dropdown coherent. +function normalizeSpeculativeType(v: string | null | undefined): string | null { + if (v == null) return null; + if (v === "default" || v === "off") return v; + return "default"; +} + type SelectedModelInput = { id: string; isLora?: boolean; @@ -279,7 +292,7 @@ export function useChatModelRuntime() { const ggufNativeContextLength = statusRes.is_gguf ? (statusRes.native_context_length ?? null) : null; - const currentSpecType = statusRes.speculative_type ?? null; + const currentSpecType = normalizeSpeculativeType(statusRes.speculative_type); useChatRuntimeStore.setState({ supportsReasoning, reasoningAlwaysOn, @@ -492,7 +505,7 @@ export function useChatModelRuntime() { } } const loadedKv = loadResponse.cache_type_kv ?? null; - const loadedSpec = loadResponse.speculative_type ?? null; + const loadedSpec = normalizeSpeculativeType(loadResponse.speculative_type); const nativeCtx = loadResponse.is_gguf ? (loadResponse.context_length ?? 131072) : null; diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 2a1249aea0..ce69d8f6dc 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -270,7 +270,7 @@ export const useChatRuntimeStore = create((set) => ({ toolCallTimeout: loadInt(TOOL_CALL_TIMEOUT_KEY, 5), kvCacheDtype: null, loadedKvCacheDtype: null, - speculativeType: "ngram-mod", + speculativeType: "default", loadedSpeculativeType: null, customContextLength: null, defaultChatTemplate: null, @@ -365,7 +365,7 @@ export const useChatRuntimeStore = create((set) => ({ toolStatus: null, kvCacheDtype: null, loadedKvCacheDtype: null, - speculativeType: "ngram-mod", + speculativeType: "default", loadedSpeculativeType: null, customContextLength: null, defaultChatTemplate: null,