diff --git a/studio/backend/core/inference/defaults.py b/studio/backend/core/inference/defaults.py index 53718c1294..f14c03dad2 100644 --- a/studio/backend/core/inference/defaults.py +++ b/studio/backend/core/inference/defaults.py @@ -6,15 +6,16 @@ import utils.hardware.hardware as hw DEFAULT_MODELS_GGUF = [ + "unsloth/Qwen3.6-27B-MTP-GGUF", + "unsloth/Qwen3.6-35B-A3B-MTP-GGUF", "unsloth/gemma-4-E2B-it-GGUF", "unsloth/gemma-4-E4B-it-GGUF", "unsloth/gemma-4-31B-it-GGUF", "unsloth/gemma-4-26B-A4B-it-GGUF", - "unsloth/Qwen3.6-35B-A3B-GGUF", - "unsloth/Qwen3.5-4B-GGUF", - "unsloth/Qwen3.5-9B-GGUF", - "unsloth/Qwen3.5-35B-A3B-GGUF", - "unsloth/Qwen3.5-0.8B-GGUF", + "unsloth/Qwen3.5-4B-MTP-GGUF", + "unsloth/Qwen3.5-9B-MTP-GGUF", + "unsloth/Qwen3.5-35B-A3B-MTP-GGUF", + "unsloth/Qwen3.5-0.8B-MTP-GGUF", "unsloth/Llama-3.2-1B-Instruct-GGUF", "unsloth/Llama-3.2-3B-Instruct-GGUF", "unsloth/Llama-3.1-8B-Instruct-GGUF", @@ -24,15 +25,16 @@ DEFAULT_MODELS_GGUF = [ ] DEFAULT_MODELS_STANDARD = [ + "unsloth/Qwen3.6-27B-MTP-GGUF", + "unsloth/Qwen3.6-35B-A3B-MTP-GGUF", "unsloth/gemma-4-E2B-it-GGUF", "unsloth/gemma-4-E4B-it-GGUF", "unsloth/gemma-4-31B-it-GGUF", "unsloth/gemma-4-26B-A4B-it-GGUF", - "unsloth/Qwen3.6-35B-A3B-GGUF", - "unsloth/Qwen3.5-4B-GGUF", - "unsloth/Qwen3.5-9B-GGUF", - "unsloth/Qwen3.5-35B-A3B-GGUF", - "unsloth/Qwen3.5-0.8B-GGUF", + "unsloth/Qwen3.5-4B-MTP-GGUF", + "unsloth/Qwen3.5-9B-MTP-GGUF", + "unsloth/Qwen3.5-35B-A3B-MTP-GGUF", + "unsloth/Qwen3.5-0.8B-MTP-GGUF", "unsloth/gemma-4-E2B-it", "unsloth/gemma-4-E4B-it", "unsloth/gemma-4-31B-it", diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 286fddda11..21f2fe71b5 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2651,9 +2651,10 @@ class LlamaCppBackend: ) user_owns_spec_type = _extra_args_set_spec_type(extra_args) # Auto-promote unset/"default" to draft-mtp on MTP GGUFs. + # llama.cpp #22673: MTP is compatible with mmproj, so the + # vision gate previously here was wrong. if ( is_mtp_model - and not effective_is_vision and not user_owns_spec_type and normalized_spec in (None, "", "default") ): @@ -2662,11 +2663,7 @@ class LlamaCppBackend: # User --spec-type wins (it accumulates if repeated). normalized_spec = None self._speculative_type = None - if ( - normalized_spec - and normalized_spec != "off" - and not effective_is_vision - ): + if normalized_spec and normalized_spec != "off": if normalized_spec == "default": cmd.append("--spec-default") self._speculative_type = "default" @@ -3112,22 +3109,16 @@ class LlamaCppBackend: if _norm(self._cache_type_kv) != _norm(cache_type_kv): return False - # Vision GGUFs silently drop speculative decoding in - # load_model (the spec gate is "not is_vision"); treat the - # request's value as "off" so a vision load with - # speculative_type="default" still matches. - if self._is_vision or is_vision: - req_spec = "off" - else: - raw_spec = _norm(speculative_type) - req_spec = raw_spec or "off" - # Mirror load_model's auto-promotion so repeat /load matches. - if ( - raw_spec in (None, "default") - and _is_mtp_model_name(model_identifier, gguf_path) - and not _extra_args_set_spec_type(extra_args) - ): - req_spec = "draft-mtp" + # Mirror load_model's auto-promotion. Vision is no longer a + # spec blocker (llama.cpp #22673: MTP is compatible with mmproj). + raw_spec = _norm(speculative_type) + req_spec = raw_spec or "off" + if ( + raw_spec in (None, "default") + and _is_mtp_model_name(model_identifier, gguf_path) + and not _extra_args_set_spec_type(extra_args) + ): + req_spec = "draft-mtp" backend_spec = _norm(self._speculative_type) or "off" if req_spec != backend_spec: return False diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.py index c6a170fa0a..7da633201f 100644 --- a/studio/backend/tests/test_llama_cpp_mtp_detection.py +++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py @@ -351,6 +351,67 @@ def test_already_in_target_state_local_file_mtp_match(tmp_path): ) +def test_already_in_target_state_vision_mtp_match(): + # llama.cpp #22673: MTP is compatible with mmproj. A vision MTP load + # with auto/default spec must match a backend already running draft-mtp. + backend = _mtp_backend(_is_vision = True) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.6-27B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + chat_template_override = None, + extra_args = None, + is_vision = True, + ) + is True + ) + + +def test_already_in_target_state_vision_mtp_default_matches(): + backend = _mtp_backend(_is_vision = True) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.6-27B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = "default", + chat_template_override = None, + extra_args = None, + is_vision = True, + ) + is True + ) + + +def test_already_in_target_state_vision_non_mtp_unaffected(): + # Vision non-MTP repo (no -MTP marker) must still mismatch req=None + # against a backend running draft-mtp. + backend = _mtp_backend( + _model_identifier = "unsloth/Qwen3-VL-4B-Instruct-GGUF", + _is_vision = True, + ) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3-VL-4B-Instruct-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + chat_template_override = None, + extra_args = None, + is_vision = True, + ) + is False + ) + + # GGUF-metadata-based detection (nextn_predict_layers). diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 9705beea62..3703beff0a 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -979,26 +979,25 @@ export function ChatSettingsPanel({ - {!currentModelIsMultimodal && ( -
-
- - Speculative Decoding - - - N-gram speculation; faster generation with negligible - VRAM overhead. Text-only models. - -
- { - setSpeculativeType(checked ? "default" : null); - }} - /> +
+
+ + Speculative Decoding + + + Faster generation with 0% accuracy hit. +
- )} + { + setSpeculativeType(checked ? "default" : "off"); + }} + /> +
)} {!isGguf && params.checkpoint && ( diff --git a/studio/frontend/src/hooks/use-gpu-info.ts b/studio/frontend/src/hooks/use-gpu-info.ts index 64caf06b50..a7ee416112 100644 --- a/studio/frontend/src/hooks/use-gpu-info.ts +++ b/studio/frontend/src/hooks/use-gpu-info.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { apiUrl } from "@/lib/api-base"; +import { authFetch } from "@/features/auth"; import { useEffect, useState } from "react"; export interface GpuInfo { @@ -28,7 +28,7 @@ async function fetchGpuOnce(): Promise { fetchPromise = (async () => { try { - const res = await fetch(apiUrl("/api/system")); + const res = await authFetch("/api/system"); if (!res.ok) throw new Error(`HTTP ${res.status}`); const data = await res.json(); const gpuData = data?.gpu; diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 410da60a13..a46d1f0c0e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2026.5.3" +__version__ = "2026.5.4" __all__ = [ "SUPPORTS_BFLOAT16",