ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: persist speculative decoding preference across restart and model switch (#6169)

Browse source 
* Studio: persist speculative decoding preference across restart and model switch

* Studio: persist speculative preference on apply, not on edit

* Fix/adjust speculative decoding persistence for PR #6169

* Fix speculative ngram alias for PR #6169

* Fix compare speculative preference for PR #6169

---------

Co-authored-by: wasimysaid <wasimysdev@gmail.com>
This commit is contained in:
oobabooga 2026-06-12 09:37:44 -03:00 • committed by GitHub
commit f4fc06b5bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 144 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

16
studio/frontend/src/features/chat/api/chat-adapter.ts
View file

@ -36,7 +36,10 @@ import {
import {
type PendingImageEditReference,
type RagAutoInject,
resolveLoadedSpeculativeSettings,
resolveSpeculativeSettingsForLoad,
resolveToolsEnabledOnLoad,
saveSpeculativeType,
useChatRuntimeStore,
} from "../stores/chat-runtime-store";
import { useExternalProvidersStore } from "../stores/external-providers-store";
@ -1137,6 +1140,7 @@ async function autoLoadSmallestModel(): Promise<{
const store = useChatRuntimeStore.getState();
const hfToken = store.hfToken || null;
const trustRemoteCode = store.params.trustRemoteCode ?? false;
const specSettings = resolveSpeculativeSettingsForLoad();
const toastId = toast("Loading a model…", {
description: "Auto-selecting the smallest downloaded model.",
duration: 5000,
@ -1201,7 +1205,10 @@ async function autoLoadSmallestModel(): Promise<{
is_lora: false,
gguf_variant: variant.quant,
trust_remote_code: trustRemoteCode,
speculative_type: specSettings.speculativeType,
spec_draft_n_max: specSettings.specDraftNMax,
});
saveSpeculativeType(specSettings.speculativeType);
useChatRuntimeStore
.getState()
.setCheckpoint(repo.repo_id, variant.quant);
@ -1250,6 +1257,7 @@ async function autoLoadSmallestModel(): Promise<{
chatTemplateOverride: null,
loadedChatTemplateOverride: null,
loadedIsMultimodal: isMultimodalResponse(loadResp),
...resolveLoadedSpeculativeSettings(loadResp),
});
toast.success(`Loaded ${repo.repo_id} (${variant.quant})`, {
id: toastId,
@ -1290,7 +1298,10 @@ async function autoLoadSmallestModel(): Promise<{
is_lora: false,
gguf_variant: null,
trust_remote_code: trustRemoteCode,
speculative_type: specSettings.speculativeType,
spec_draft_n_max: specSettings.specDraftNMax,
});
saveSpeculativeType(specSettings.speculativeType);
useChatRuntimeStore.getState().setCheckpoint(repo.repo_id);
const store = useChatRuntimeStore.getState();
store.setModelRequiresTrustRemoteCode(
@ -1310,6 +1321,7 @@ async function autoLoadSmallestModel(): Promise<{
defaultChatTemplate: sfLoadResp.chat_template ?? null,
chatTemplateOverride: null,
loadedChatTemplateOverride: null,
...resolveLoadedSpeculativeSettings(sfLoadResp),
});
const sfModel: ChatModelSummary = {
id: repo.repo_id,
@ -1372,7 +1384,10 @@ async function autoLoadSmallestModel(): Promise<{
is_lora: false,
gguf_variant: "UD-Q4_K_XL",
trust_remote_code: trustRemoteCode,
speculative_type: specSettings.speculativeType,
spec_draft_n_max: specSettings.specDraftNMax,
});
saveSpeculativeType(specSettings.speculativeType);
useChatRuntimeStore
.getState()
.setCheckpoint("unsloth/Qwen3.5-4B-MTP-GGUF", "UD-Q4_K_XL");
@ -1412,6 +1427,7 @@ async function autoLoadSmallestModel(): Promise<{
defaultChatTemplate: loadResp.chat_template ?? null,
chatTemplateOverride: null,
loadedIsMultimodal: isMultimodalResponse(loadResp),
...resolveLoadedSpeculativeSettings(loadResp),
});
toast.success("Loaded Qwen3.5-4B-MTP (UD-Q4_K_XL)", { id: toastId });
return { loaded: true, blockedByTrustRemoteCode: false };