diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 1d5643ac09..3f89cd3e5d 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -848,7 +848,7 @@ class LlamaCppBackend: "--port", str(self._port), "-c", - "0", # 0 = use model's native context size + str(n_ctx) if n_ctx > 0 else "0", # 0 = model's native context size "--parallel", "1", # Single-user studio, saves VRAM "--flash-attn", diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index b0498319ca..b4e496b051 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -22,7 +22,10 @@ class LoadRequest(BaseModel): None, description = "HuggingFace token for gated models" ) max_seq_length: int = Field( - 4096, ge = 128, le = 32768, description = "Maximum sequence length" + 0, + ge = 0, + le = 1048576, + description = "Maximum sequence length (0 = model default for GGUF)", ) load_in_4bit: bool = Field(True, description = "Load model in 4-bit quantization") is_lora: bool = Field(False, description = "Whether this is a LoRA adapter") diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index aa8c34a3c5..78d95fedbd 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -19,6 +19,27 @@ import asyncio import threading +import re as _re + + +def _friendly_error(exc: Exception) -> str: + """Extract a user-friendly message from known llama-server errors.""" + msg = str(exc) + m = _re.search( + r"request \((\d+) tokens?\) exceeds the available context size \((\d+) tokens?\)", + msg, + ) + if m: + return ( + f"Message too long: {m.group(1)} tokens exceeds the {m.group(2)}-token " + f"context window. Try increasing the Context Length in Model settings, " + f"or shorten the conversation." + ) + if "Lost connection to llama-server" in msg: + return "Lost connection to the model server. It may have crashed -- try reloading the model." + return "An internal error occurred" + + # Add backend directory to path backend_path = Path(__file__).parent.parent.parent if str(backend_path) not in sys.path: @@ -550,7 +571,7 @@ async def generate_stream( except Exception as e: backend.reset_generation_state() logger.error(f"Error during generation: {e}", exc_info = True) - yield f"data: {json.dumps({'error': 'An internal error occurred'})}\n\n" + yield f"data: {json.dumps({'error': _friendly_error(e)})}\n\n" return StreamingResponse( stream(), @@ -944,7 +965,7 @@ async def openai_chat_completions( logger.error( f"Error during audio input streaming: {e}", exc_info = True ) - yield f"data: {json.dumps({'error': {'message': 'An internal error occurred', 'type': 'server_error'}})}\n\n" + yield f"data: {json.dumps({'error': {'message': _friendly_error(e), 'type': 'server_error'}})}\n\n" return StreamingResponse( audio_input_stream(), @@ -1176,7 +1197,7 @@ async def openai_chat_completions( logger.error(f"Error during GGUF tool streaming: {e}\n{tb}") error_chunk = { "error": { - "message": "An internal error occurred", + "message": _friendly_error(e), "type": "server_error", }, } @@ -1314,7 +1335,7 @@ async def openai_chat_completions( logger.error(f"Error during GGUF streaming: {e}", exc_info = True) error_chunk = { "error": { - "message": "An internal error occurred", + "message": _friendly_error(e), "type": "server_error", }, } @@ -1495,7 +1516,7 @@ async def openai_chat_completions( logger.error(f"Error during OpenAI streaming: {e}", exc_info = True) error_chunk = { "error": { - "message": "An internal error occurred", + "message": _friendly_error(e), "type": "server_error", }, } diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 328cba3acd..3ca9aadb1f 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -454,7 +454,8 @@ export function HubModelPicker({ const recommendedIds = useMemo(() => { const all = dedupe([...models.map((model) => model.id), value ?? ""]) .filter((id) => !downloadedSet.has(id.toLowerCase())) - .filter((id) => !chatOnly || isGgufRepo(id)); + .filter((id) => !chatOnly || isGgufRepo(id)) + .filter((id) => !/-FP8[-.]|FP8-Dynamic/i.test(id)); // Sort: GGUFs first, then hub models const gguf: string[] = []; const hub: string[] = []; @@ -498,7 +499,8 @@ export function HubModelPicker({ return results .map((result) => result.id) .filter((id) => !recommendedSet.has(id)) - .filter((id) => !chatOnly || isGgufRepo(id)); + .filter((id) => !chatOnly || isGgufRepo(id)) + .filter((id) => !/-FP8[-.]|FP8-Dynamic/i.test(id)); }, [recommendedSet, results, showHfSection, chatOnly]); const metricsById = useMemo( diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 15ac416b1f..95af560305 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -253,6 +253,7 @@ function waitForModelReady(abortSignal?: AbortSignal): Promise { * falls back to smallest cached safetensors model. */ async function autoLoadSmallestModel(): Promise { + const hfToken = useChatRuntimeStore.getState().hfToken || null; const toastId = toast("Loading a model…", { description: "Auto-selecting the smallest downloaded model.", duration: 5000, @@ -278,8 +279,8 @@ async function autoLoadSmallestModel(): Promise { const variant = downloaded[0]; const loadResp = await loadModel({ model_path: repo.repo_id, - hf_token: null, - max_seq_length: 4096, + hf_token: hfToken, + max_seq_length: 0, load_in_4bit: true, is_lora: false, gguf_variant: variant.quant, @@ -308,8 +309,10 @@ async function autoLoadSmallestModel(): Promise { supportsReasoning: loadResp.supports_reasoning ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, supportsTools: loadResp.supports_tools ?? false, - toolsEnabled: false, - codeToolsEnabled: false, + toolsEnabled: loadResp.supports_tools ?? false, + codeToolsEnabled: loadResp.supports_tools ?? false, + kvCacheDtype: loadResp.cache_type_kv ?? null, + loadedKvCacheDtype: loadResp.cache_type_kv ?? null, defaultChatTemplate: loadResp.chat_template ?? null, chatTemplateOverride: null, }); @@ -329,7 +332,7 @@ async function autoLoadSmallestModel(): Promise { try { const sfLoadResp = await loadModel({ model_path: repo.repo_id, - hf_token: null, + hf_token: hfToken, max_seq_length: 4096, load_in_4bit: true, is_lora: false, @@ -366,8 +369,8 @@ async function autoLoadSmallestModel(): Promise { try { const loadResp = await loadModel({ model_path: "unsloth/Qwen3.5-4B-GGUF", - hf_token: null, - max_seq_length: 4096, + hf_token: hfToken, + max_seq_length: 0, load_in_4bit: true, is_lora: false, gguf_variant: "UD-Q4_K_XL", @@ -391,7 +394,10 @@ async function autoLoadSmallestModel(): Promise { supportsReasoning: loadResp.supports_reasoning ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, supportsTools: loadResp.supports_tools ?? false, - toolsEnabled: false, + toolsEnabled: loadResp.supports_tools ?? false, + codeToolsEnabled: loadResp.supports_tools ?? false, + kvCacheDtype: loadResp.cache_type_kv ?? null, + loadedKvCacheDtype: loadResp.cache_type_kv ?? null, defaultChatTemplate: loadResp.chat_template ?? null, chatTemplateOverride: null, }); @@ -410,8 +416,7 @@ async function autoLoadSmallestModel(): Promise { export function createOpenAIStreamAdapter(): ChatModelAdapter { return { async *run({ messages, abortSignal, unstable_threadId }) { - const runtime = useChatRuntimeStore.getState(); - const { params } = runtime; + let runtime = useChatRuntimeStore.getState(); // Wait for in-progress model load to finish before inferring if (runtime.modelLoading) { @@ -430,6 +435,9 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { } } + // Re-read store after potential auto-load / model ready wait + runtime = useChatRuntimeStore.getState(); + const { params } = runtime; const { supportsTools, toolsEnabled, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 081e3efc26..a315aeb005 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -279,6 +279,14 @@ export function ChatSettingsPanel({ const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype); const setKvCacheDtype = useChatRuntimeStore((s) => s.setKvCacheDtype); + const loadedKvCacheDtype = useChatRuntimeStore((s) => s.loadedKvCacheDtype); + const customContextLength = useChatRuntimeStore((s) => s.customContextLength); + const setCustomContextLength = useChatRuntimeStore((s) => s.setCustomContextLength); + + const ctxDisplayValue = customContextLength ?? ggufContextLength ?? ""; + const kvDirty = kvCacheDtype !== loadedKvCacheDtype; + const ctxDirty = customContextLength !== null; + const modelSettingsDirty = kvDirty || ctxDirty; const [customPresets, setCustomPresets] = useState(() => loadSavedCustomPresets(), ); @@ -467,32 +475,53 @@ export function ChatSettingsPanel({
{isGguf && ( <> -
-
-
Context Length
-
- Reported by the loaded GGUF model. -
+
+
+ Context Length + { + const raw = e.target.value; + if (raw === "") { + setCustomContextLength(null); + return; + } + const v = parseInt(raw, 10); + if (!Number.isNaN(v) && v >= 0) { + const maxCtx = ggufContextLength ?? Infinity; + const clamped = Math.min(v, maxCtx); + setCustomContextLength(clamped === (ggufContextLength ?? 0) ? null : clamped); + } + }} + />
- { + setCustomContextLength(v === (ggufContextLength ?? 0) ? null : v); + }} />
KV Cache Dtype
- Quantize KV cache to reduce VRAM. Reload to apply. + Quantize KV cache to reduce VRAM.
+ {modelSettingsDirty && ( +
+ + +
+ )} )} - {!isGguf && ( + {!isGguf && params.checkpoint && (
-
Trust remote code
+
Enable custom code
- Allow models with custom code (e.g. Nemotron). Only enable for repos you trust. + Allow models with custom code (e.g. Nemotron). Only enable if sure.
+
@@ -775,6 +826,29 @@ function AutoHealToolCallsToggle() { ); } +function HfTokenField() { + const hfToken = useChatRuntimeStore((s) => s.hfToken); + const setHfToken = useChatRuntimeStore((s) => s.setHfToken); + + return ( +
+
+
Hugging Face Token
+
+ For downloading gated or private models. +
+
+ setHfToken(e.target.value)} + /> +
+ ); +} + function ChatTemplateSection({ onReloadModel, }: { diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index cfdd4e774a..25c776948f 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -354,12 +354,13 @@ export function useChatModelRuntime() { useChatRuntimeStore.getState().params.checkpoint; const paramsBeforeLoad = useChatRuntimeStore.getState().params; const maxSeqLength = paramsBeforeLoad.maxSeqLength; + const hfToken = useChatRuntimeStore.getState().hfToken || null; try { // Lightweight pre-flight validation: avoid unloading a working model // if the new identifier is clearly invalid (e.g. bad HF id / path). await validateModel({ model_path: modelId, - hf_token: null, + hf_token: hfToken, max_seq_length: maxSeqLength, load_in_4bit: true, is_lora: isLora, @@ -371,11 +372,16 @@ export function useChatModelRuntime() { previousWasUnloaded = true; } - const { chatTemplateOverride, kvCacheDtype } = useChatRuntimeStore.getState(); + const { chatTemplateOverride, kvCacheDtype, customContextLength, ggufContextLength } = useChatRuntimeStore.getState(); + // GGUF: use custom context length, or 0 = model's native context + // Non-GGUF: use the Max Seq Length slider value + const effectiveMaxSeqLength = customContextLength != null + ? customContextLength + : ggufVariant != null ? (ggufContextLength ?? 0) : maxSeqLength; const loadResponse = await loadModel({ model_path: modelId, - hf_token: null, - max_seq_length: maxSeqLength, + hf_token: hfToken, + max_seq_length: effectiveMaxSeqLength, load_in_4bit: true, is_lora: isLora, gguf_variant: ggufVariant ?? null, @@ -403,15 +409,27 @@ export function useChatModelRuntime() { } } } + const loadedKv = loadResponse.cache_type_kv ?? null; + const nativeCtx = loadResponse.is_gguf + ? (loadResponse.context_length ?? 131072) + : null; + // Keep customContextLength if the user set one and it differs + // from the model's native context; otherwise clear it so the + // display shows the native value without a dirty marker. + const keepCustomCtx = customContextLength != null + && customContextLength !== nativeCtx + ? customContextLength + : null; useChatRuntimeStore.setState({ - ggufContextLength: loadResponse.is_gguf - ? (loadResponse.context_length ?? 131072) - : null, + ggufContextLength: nativeCtx, supportsReasoning: loadResponse.supports_reasoning ?? false, reasoningEnabled: reasoningDefault, supportsTools: loadResponse.supports_tools ?? false, - toolsEnabled: false, - kvCacheDtype: loadResponse.cache_type_kv ?? null, + toolsEnabled: loadResponse.supports_tools ?? false, + codeToolsEnabled: loadResponse.supports_tools ?? false, + kvCacheDtype: loadedKv, + loadedKvCacheDtype: loadedKv, + customContextLength: keepCustomCtx, defaultChatTemplate: loadResponse.chat_template ?? null, chatTemplateOverride: null, }); @@ -432,7 +450,7 @@ export function useChatModelRuntime() { try { await loadModel({ model_path: previousCheckpoint, - hf_token: null, + hf_token: hfToken, max_seq_length: maxSeqLength, load_in_4bit: true, is_lora: previousIsLora, diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index 78cfcc66d2..5ac8c79160 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -337,7 +337,7 @@ export function SharedComposer({ async function ensureModelLoaded(sel: CompareModelSelection): Promise { const resp = await loadModel({ model_path: sel.id, - hf_token: null, + hf_token: useChatRuntimeStore.getState().hfToken || null, max_seq_length: maxSeqLength, load_in_4bit: true, is_lora: sel.isLora, diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 920737a279..2d60d52043 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -14,6 +14,7 @@ const AUTO_TITLE_KEY = "unsloth_chat_auto_title"; const AUTO_HEAL_TOOL_CALLS_KEY = "unsloth_auto_heal_tool_calls"; const MAX_TOOL_CALLS_KEY = "unsloth_max_tool_calls_per_message"; const TOOL_CALL_TIMEOUT_KEY = "unsloth_tool_call_timeout"; +const HF_TOKEN_KEY = "unsloth_hf_token"; const INFERENCE_PARAMS_KEY = "unsloth_chat_inference_params"; let hasShownInferencePersistenceWarning = false; @@ -62,6 +63,24 @@ function saveInt(key: string, value: number): void { } } +function loadString(key: string, fallback: string): string { + if (!canUseStorage()) return fallback; + try { + return localStorage.getItem(key) ?? fallback; + } catch { + return fallback; + } +} + +function saveString(key: string, value: string): void { + if (!canUseStorage()) return; + try { + localStorage.setItem(key, value); + } catch { + // ignore + } +} + function asFiniteNumber(value: unknown, fallback: number): number { return typeof value === "number" && Number.isFinite(value) ? value : fallback; } @@ -127,6 +146,7 @@ type ChatRuntimeStore = { loras: ChatLoraSummary[]; runningByThreadId: Record; autoTitle: boolean; + hfToken: string; modelsError: string | null; activeGgufVariant: string | null; ggufContextLength: number | null; @@ -141,6 +161,8 @@ type ChatRuntimeStore = { maxToolCallsPerMessage: number; toolCallTimeout: number; kvCacheDtype: string | null; + loadedKvCacheDtype: string | null; + customContextLength: number | null; defaultChatTemplate: string | null; chatTemplateOverride: string | null; activeThreadId: string | null; @@ -159,6 +181,7 @@ type ChatRuntimeStore = { setLoras: (loras: ChatLoraSummary[]) => void; setThreadRunning: (threadId: string, running: boolean) => void; setAutoTitle: (enabled: boolean) => void; + setHfToken: (token: string) => void; setModelsError: (error: string | null) => void; setCheckpoint: (modelId: string, ggufVariant?: string | null) => void; setActiveThreadId: (threadId: string | null) => void; @@ -172,6 +195,7 @@ type ChatRuntimeStore = { setMaxToolCallsPerMessage: (value: number) => void; setToolCallTimeout: (value: number) => void; setKvCacheDtype: (dtype: string | null) => void; + setCustomContextLength: (v: number | null) => void; setChatTemplateOverride: (template: string | null) => void; setPendingAudio: (base64: string, name: string) => void; clearPendingAudio: () => void; @@ -184,6 +208,7 @@ export const useChatRuntimeStore = create((set) => ({ loras: [], runningByThreadId: {}, autoTitle: loadBool(AUTO_TITLE_KEY, false), + hfToken: loadString(HF_TOKEN_KEY, ""), modelsError: null, activeGgufVariant: null, ggufContextLength: null, @@ -198,6 +223,8 @@ export const useChatRuntimeStore = create((set) => ({ maxToolCallsPerMessage: loadInt(MAX_TOOL_CALLS_KEY, 10), toolCallTimeout: loadInt(TOOL_CALL_TIMEOUT_KEY, 5), kvCacheDtype: null, + loadedKvCacheDtype: null, + customContextLength: null, defaultChatTemplate: null, chatTemplateOverride: null, activeThreadId: null, @@ -235,6 +262,11 @@ export const useChatRuntimeStore = create((set) => ({ saveBool(AUTO_TITLE_KEY, autoTitle); return { autoTitle }; }), + setHfToken: (hfToken) => + set(() => { + saveString(HF_TOKEN_KEY, hfToken); + return { hfToken }; + }), setModelsError: (modelsError) => set({ modelsError }), setCheckpoint: (modelId, ggufVariant) => set((state) => ({ @@ -261,6 +293,8 @@ export const useChatRuntimeStore = create((set) => ({ codeToolsEnabled: false, toolStatus: null, kvCacheDtype: null, + loadedKvCacheDtype: null, + customContextLength: null, defaultChatTemplate: null, chatTemplateOverride: null, })), @@ -285,6 +319,7 @@ export const useChatRuntimeStore = create((set) => ({ return { toolCallTimeout }; }), setKvCacheDtype: (kvCacheDtype) => set({ kvCacheDtype }), + setCustomContextLength: (customContextLength) => set({ customContextLength }), setChatTemplateOverride: (chatTemplateOverride) => set({ chatTemplateOverride }), setPendingAudio: (base64, name) => set({ pendingAudioBase64: base64, pendingAudioName: name }), diff --git a/studio/frontend/src/features/chat/thread-sidebar.tsx b/studio/frontend/src/features/chat/thread-sidebar.tsx index 53c5521dc7..ba97d2ee6e 100644 --- a/studio/frontend/src/features/chat/thread-sidebar.tsx +++ b/studio/frontend/src/features/chat/thread-sidebar.tsx @@ -172,7 +172,7 @@ export function ThreadSidebar({ Learn more in docs