ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into studio-fla-tilelang-qwen3.5

Browse source
This commit is contained in:
Daniel Han 2026-05-15 05:36:19 -07:00 • committed by GitHub
commit 9e9c3ac59e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1277 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

17
.github/workflows/consolidated-tests-ci.yml vendored
View file

@ -1032,16 +1032,28 @@ jobs:
"""Spot-check on the three production-relevant families that
the compile_every sweep also covers; this case verifies the
emitted cache file has the model-specific RMSNorm class
attribute, not just that the file parses + imports."""
attribute, not just that the file parses + imports.
Note on test isolation: ``unsloth_compile_transformers``
early-returns when ``modeling.__UNSLOTH_PATCHED__`` is set,
so once an earlier test in the same collection patches the
module the next call won't re-emit the cache file. Drop the
marker (and any stale cache file) before invoking so this
test is order-independent."""
import importlib as _il
try:
_il.import_module(
modeling = _il.import_module(
f"transformers.models.{model_type}.modeling_{model_type}"
)
except ModuleNotFoundError:
pytest.skip(
f"transformers build lacks model_type={model_type}"
)
if hasattr(modeling, "__UNSLOTH_PATCHED__"):
delattr(modeling, "__UNSLOTH_PATCHED__")
combined = _CACHE / f"unsloth_compiled_module_{model_type}.py"
if combined.exists():
combined.unlink()
unsloth_compile_transformers(
model_type=model_type, fast_lora_forwards=False,
)
@ -1049,7 +1061,6 @@ jobs:
f"transformers.models.{model_type}.modeling_{model_type}"
)
assert getattr(modeling, "__UNSLOTH_PATCHED__", False) is True
combined = _CACHE / f"unsloth_compiled_module_{model_type}.py"
_verify_file(combined, must_expose=[rms_class])

6
.github/workflows/notebooks-ci.yml vendored
View file

@ -200,7 +200,7 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
with: { path: unsloth }
path: unsloth
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: unslothai/notebooks
@ -246,7 +246,7 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
with: { path: unsloth }
path: unsloth
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: unslothai/notebooks
@ -352,7 +352,7 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
with: { path: unsloth }
path: unsloth
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: unslothai/notebooks

2
.github/workflows/version-compat-ci.yml vendored
View file

@ -214,7 +214,7 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
with: { path: unsloth }
path: unsloth
- name: Clone unsloth-zoo @ main
run: |
# github.com occasionally 500s on the git fetch; retry so a

1007
studio/backend/core/inference/external_provider.py
View file

File diff suppressed because it is too large Load diff

1
studio/backend/routes/inference.py
View file

@ -1595,6 +1595,7 @@ async def _proxy_to_external_provider(
top_k = payload.top_k,
enable_thinking = payload.enable_thinking,
reasoning_effort = payload.reasoning_effort,
enabled_tools = payload.enabled_tools,
stream = payload.stream,
)
try:

45
studio/frontend/src/components/assistant-ui/thread.tsx
View file

@ -496,6 +496,9 @@ const ReasoningToggle: FC = () => {
externalSelection != null
? externalProviders.find((p) => p.id === externalSelection.providerId)
: undefined;
const isKimiExternal = selectedExternalProvider?.providerType === "kimi";
const toolsEnabled = useChatRuntimeStore((s) => s.toolsEnabled);
const setToolsEnabled = useChatRuntimeStore((s) => s.setToolsEnabled);
const effectiveExternalModelId =
selectedExternalProvider?.providerType === "openrouter" &&
externalSelection?.modelId === "openrouter/free" &&
@ -587,6 +590,11 @@ const ReasoningToggle: FC = () => {
setReasoningEffort(level);
setReasoningEnabled(true);
applyQwenThinkingParams(true);
// Kimi's $web_search builtin forbids thinking, so
// enabling thinking flips the Search pill off.
if (isKimiExternal && toolsEnabled) {
setToolsEnabled(false);
}
}}
>
{formatEffortLabel(level)}
@ -613,6 +621,11 @@ const ReasoningToggle: FC = () => {
const next = !reasoningEnabled;
setReasoningEnabled(next);
applyQwenThinkingParams(next);
// Mutual exclusion with the Search pill on Kimi — see the
// dropdown branch above and shared-composer for the same rule.
if (isKimiExternal && next && toolsEnabled) {
setToolsEnabled(false);
}
}}
className="composer-pill-btn"
data-active={
@ -680,16 +693,44 @@ const WebSearchToggle: FC = () => {
const modelLoaded = useChatRuntimeStore(
(s) => !!s.params.checkpoint && !s.modelLoading,
);
const checkpoint = useChatRuntimeStore((s) => s.params.checkpoint);
const supportsTools = useChatRuntimeStore((s) => s.supportsTools);
// External providers (OpenAI today) expose a server-side web_search tool
// even when the local tool runtime is unavailable — gate the Search pill
// on either source so it lights up on external models too. Mirror of
// shared-composer's searchDisabled.
const supportsBuiltinWebSearch = useChatRuntimeStore(
(s) => s.supportsBuiltinWebSearch,
);
const toolsEnabled = useChatRuntimeStore((s) => s.toolsEnabled);
const setToolsEnabled = useChatRuntimeStore((s) => s.setToolsEnabled);
const disabled = !(modelLoaded && supportsTools);
const setReasoningEnabled = useChatRuntimeStore((s) => s.setReasoningEnabled);
const externalProviders = useExternalProvidersStore((s) => s.providers);
const externalSelection = parseExternalModelId(checkpoint);
const selectedExternalProvider =
externalSelection != null
? externalProviders.find((p) => p.id === externalSelection.providerId)
: undefined;
const isKimiExternal = selectedExternalProvider?.providerType === "kimi";
const disabled =
!modelLoaded || !(supportsTools || supportsBuiltinWebSearch);
return (
<button
type="button"
disabled={disabled}
onClick={() => setToolsEnabled(!toolsEnabled)}
onClick={() => {
const next = !toolsEnabled;
setToolsEnabled(next);
// Kimi's $web_search builtin requires thinking=disabled (see
// https://platform.kimi.ai/docs/guide/use-web-search). Keep
// the two pills mutually exclusive so the visible state always
// matches what the backend ends up sending.
if (isKimiExternal) {
setReasoningEnabled(!next);
applyQwenThinkingParams(!next);
}
}}
className="composer-pill-btn"
data-active={toolsEnabled && !disabled ? "true" : "false"}
aria-label={toolsEnabled ? "Disable web search" : "Enable web search"}

15
studio/frontend/src/features/chat/api/chat-adapter.ts
View file

@ -35,6 +35,7 @@ import {
getExternalMinOutputTokens,
getExternalReasoningCapabilities,
getProviderCapabilities,
providerSupportsBuiltinWebSearch,
} from "../provider-capabilities";
import { useChatRuntimeStore } from "../stores/chat-runtime-store";
import { isMultimodalResponse } from "../types/api";
@ -1005,6 +1006,20 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
...(externalCapabilities?.presencePenalty
? { presence_penalty: params.presencePenalty }
: {}),
// Built-in web search: when the user has the Search toggle
// on AND the active provider supports a server-side
// web_search tool (currently OpenAI's /v1/responses), pass
// the enable_tools shorthand. Backend translates
// enabled_tools=["web_search"] into the provider's tool
// schema — for OpenAI that's `tools: [{type:"web_search"}]`
// on the Responses body, see _stream_openai_responses.
...(toolsEnabled &&
providerSupportsBuiltinWebSearch(externalProvider.providerType)
? {
enable_tools: true,
enabled_tools: ["web_search"],
}
: {}),
provider_id: externalProvider.id,
provider_type: externalBackendProviderType,
external_model: externalSelection.modelId,

113
studio/frontend/src/features/chat/chat-page.tsx
View file

@ -50,6 +50,7 @@ import {
clampReasoningEffortToLevels,
getExternalReasoningCapabilities,
getProviderCapabilities,
providerSupportsBuiltinWebSearch,
} from "./provider-capabilities";
import { useChatModelRuntime } from "./hooks/use-chat-model-runtime";
import {
@ -678,9 +679,56 @@ export function ChatPage(): ReactElement {
preferredEffort,
effortLevels,
);
// Per-provider default effort. Anthropic gets the highest available
// level (xhigh on 4.6/4.7, high on 4.5) since Claude's adaptive
// thinking adjusts cost per turn — sitting at the top of the dial
// gives users the strongest answers and the model can still skip
// thinking when the turn is trivial. OpenAI gets "high" by default
// — the gpt-5.x reasoning models accept high across the board and
// it's the right cost/quality sweet spot for Responses-API tools
// (web search included). Everyone else gets "medium" as a balanced
// default. Users can pick another level via the Think dropdown.
const isAnthropic = provider?.providerType === "anthropic";
const isOpenAI = provider?.providerType === "openai";
const anthropicTopEffort = effortLevels.includes("xhigh")
? "xhigh"
: effortLevels.includes("high")
? "high"
: clampedEffort;
const openaiDefaultEffort = effortLevels.includes("high")
? "high"
: effortLevels.includes("medium")
? "medium"
: clampedEffort;
const nextReasoningEffort = reasoningCaps.supportsReasoning
? clampedEffort
? isAnthropic
? anthropicTopEffort
: isOpenAI
? openaiDefaultEffort
: effortLevels.includes("medium")
? "medium"
: clampedEffort
: state.reasoningEffort;
const supportsBuiltinWebSearch = providerSupportsBuiltinWebSearch(
provider?.providerType,
);
// Kimi's k2.6/k2.5 default to thinking enabled on the server side
// (per https://platform.kimi.ai/docs/models). Mirror that default
// in the UI so the Think pill comes up clicked when the user picks
// a Kimi model. The Search pill stays off by default; the mutual-
// exclusion handlers in the composer flip the two when needed.
const isKimi = provider?.providerType === "kimi";
// Web search is on by default for the two providers we trust most
// for it: Anthropic (web_search_20250305 server tool, structured
// citations) and OpenAI (/v1/responses web_search, structured
// citations). Other providers stay off-by-default — OpenRouter's
// plugins shape and Kimi's $web_search builtin still work when the
// user opts in via the pill, but they're a notch less reliable so
// we don't pre-enable them.
const searchOnByDefault =
supportsBuiltinWebSearch &&
(provider?.providerType === "anthropic" ||
provider?.providerType === "openai");
useChatRuntimeStore.setState({
supportsReasoning: reasoningCaps.supportsReasoning,
reasoningAlwaysOn: reasoningCaps.reasoningAlwaysOn,
@ -690,10 +738,22 @@ export function ChatPage(): ReactElement {
reasoningEffort: nextReasoningEffort,
reasoningEnabled: reasoningCaps.supportsReasoning
? reasoningCaps.supportsReasoningOff
? state.reasoningEnabled
? isKimi
? true
: state.reasoningEnabled
: true
: state.reasoningEnabled,
supportsPreserveThinking: false,
// External models never give us a local tool runtime (no Code
// execution, no python sandbox), so `supportsTools` must be
// false — that's what gates the Code pill in the composer.
// `supportsBuiltinWebSearch` is the separate flag that lets the
// Search pill light up for providers (currently just OpenAI) who
// run web_search server-side.
supportsTools: false,
supportsBuiltinWebSearch,
toolsEnabled: searchOnByDefault,
codeToolsEnabled: false,
});
}, [externalProviders, inferenceParams.checkpoint]);
const canCompare = useMemo(() => {
@ -810,8 +870,29 @@ export function ChatPage(): ReactElement {
preferredEffort,
effortLevels,
);
// Same per-provider default policy as the useEffect path above:
// Anthropic picks the highest available level, OpenAI picks
// "high", everyone else picks "medium".
const isAnthropic = selectedProvider?.providerType === "anthropic";
const isOpenAI = selectedProvider?.providerType === "openai";
const anthropicTopEffort = effortLevels.includes("xhigh")
? "xhigh"
: effortLevels.includes("high")
? "high"
: clampedEffort;
const openaiDefaultEffort = effortLevels.includes("high")
? "high"
: effortLevels.includes("medium")
? "medium"
: clampedEffort;
const nextReasoningEffort = reasoningCaps.supportsReasoning
? clampedEffort
? isAnthropic
? anthropicTopEffort
: isOpenAI
? openaiDefaultEffort
: effortLevels.includes("medium")
? "medium"
: clampedEffort
: store.reasoningEffort;
// Clear any cached router-picked openrouter/free model unless the
// user is staying on openrouter/free — otherwise the chip would
@ -823,6 +904,20 @@ export function ChatPage(): ReactElement {
...store.params,
checkpoint: value,
});
const supportsBuiltinWebSearch = providerSupportsBuiltinWebSearch(
selectedProvider?.providerType,
);
// See sibling useEffect above: Kimi's k2.x default to thinking
// enabled, so the Think pill comes up clicked. Search pill stays
// off by default; mutual exclusion flips them via the composer.
const isKimi = selectedProvider?.providerType === "kimi";
// Mirror of sibling useEffect: Anthropic and OpenAI get Search
// on-by-default since their server tools emit structured
// citations end-to-end. OpenRouter and Kimi stay off-by-default.
const searchOnByDefault =
supportsBuiltinWebSearch &&
(selectedProvider?.providerType === "anthropic" ||
selectedProvider?.providerType === "openai");
useChatRuntimeStore.setState({
activeGgufVariant: null,
ggufContextLength: null,
@ -837,10 +932,20 @@ export function ChatPage(): ReactElement {
reasoningEffort: nextReasoningEffort,
reasoningEnabled: reasoningCaps.supportsReasoning
? reasoningCaps.supportsReasoningOff
? store.reasoningEnabled
? isKimi
? true
: store.reasoningEnabled
: true
: store.reasoningEnabled,
supportsPreserveThinking: false,
// External models have no local tool runtime → supportsTools=false
// keeps the Code pill greyed out. supportsBuiltinWebSearch is the
// separate flag the composer reads to light up the Search pill
// when the provider offers a server-side web_search tool.
supportsTools: false,
supportsBuiltinWebSearch,
toolsEnabled: searchOnByDefault,
codeToolsEnabled: false,
...(stillOnOpenRouterFree ? {} : { lastOpenRouterChosenModel: null }),
});
return;

40
studio/frontend/src/features/chat/provider-capabilities.ts
View file

@ -83,6 +83,44 @@ export function clampReasoningEffortToLevels(
*/
export const EXTERNAL_MAX_OUTPUT_TOKENS = 32768;
/**
* Whether the external provider offers a built-in web-search tool that the
* model invokes server-side. When `true`, the chat composer's Search button
* is available for that provider and the chat-adapter forwards
* `enable_tools: true, enabled_tools: ["web_search"]` on the request — the
* backend routes the call through the provider's tool schema:
* - OpenAI: `tools: [{type: "web_search"}]` on /v1/responses
* - Anthropic: `tools: [{type: "web_search_20250305", name: "web_search",
* max_uses: 5}]` on /v1/messages
* - OpenRouter: `plugins: [{id: "web"}]` on /v1/chat/completions (the
* router's universal web-search shape; works for every
* underlying model including the `openrouter/free` router).
* - Kimi: `tools: [{type: "builtin_function", function: {name:
* "$web_search"}}]` with `thinking: {type:
* "disabled"}`. Requires a client round-trip:
* the first call returns the search args; the backend
* echoes them back as a role=tool message; the second
* call streams the answer. Handled in
* _stream_kimi_web_search on the backend.
*
* Mistral is intentionally excluded: their `web_search` connector lives on
* the Agents API (`/v1/agents` + `/v1/conversations`), not chat completions,
* and returns `"WebSearchTool connector is not supported"` if injected into
* /v1/chat/completions. Wiring it would require a dedicated Agents streaming
* path. Gemini's grounded-search can be added with the same pattern when
* matching backend translation lands.
*/
export function providerSupportsBuiltinWebSearch(
providerType: string | null | undefined,
): boolean {
return (
providerType === "openai" ||
providerType === "anthropic" ||
providerType === "openrouter" ||
providerType === "kimi"
);
}
/**
* Per-provider minimum on the outbound max_tokens. Kimi's docs require
* `max_tokens >= 16000` whenever a thinking model is in use so the
@ -239,7 +277,7 @@ const NO_REASONING_CAPS: ReasoningCaps = {
const ANTHROPIC_REASONING_MODELS = [
{
prefixes: ["claude-opus-4-7"],
levels: ["none", "low", "medium", "high", "xhigh"],
levels: ["none", "low", "medium", "high", "xhigh", "max"],
},
{
prefixes: ["claude-opus-4-6", "claude-sonnet-4-6"],

56
studio/frontend/src/features/chat/shared-composer.tsx
View file

@ -304,6 +304,9 @@ export function SharedComposer({
const preserveThinking = useChatRuntimeStore((s) => s.preserveThinking);
const setPreserveThinking = useChatRuntimeStore((s) => s.setPreserveThinking);
const supportsTools = useChatRuntimeStore((s) => s.supportsTools);
const supportsBuiltinWebSearch = useChatRuntimeStore(
(s) => s.supportsBuiltinWebSearch,
);
const toolsEnabled = useChatRuntimeStore((s) => s.toolsEnabled);
const setToolsEnabled = useChatRuntimeStore((s) => s.setToolsEnabled);
const codeToolsEnabled = useChatRuntimeStore((s) => s.codeToolsEnabled);
@ -345,13 +348,31 @@ export function SharedComposer({
const reasoningLockedOn =
effectiveSupportsReasoning &&
(effectiveReasoningAlwaysOn || !effectiveSupportsReasoningOff);
// Kimi's $web_search builtin mandates thinking=disabled per the docs at
// https://platform.kimi.ai/docs/guide/use-web-search. Both pills stay
// clickable for Kimi, but turning one on flips the other off — the
// click handlers below enforce this mutual exclusion so the visible
// state always matches what the backend actually sends.
const isKimiExternal = selectedExternalProvider?.providerType === "kimi";
const effectiveReasoningEnabled = reasoningLockedOn ? true : reasoningEnabled;
const effectiveReasoningVisualEnabled =
effectiveReasoningEnabled && reasoningEffort !== "none";
const reasoningDisabled = !modelLoaded || !effectiveSupportsReasoning;
const showReasoningControl =
effectiveSupportsReasoning || effectiveReasoningAlwaysOn;
const toolsDisabled = !modelLoaded || !supportsTools;
// Two-pill gating: Search pill lights up when the runtime has either
// a local tool runtime (supportsTools, gives us our Code/python + local
// web_search) OR a server-side web_search the provider runs for us
// (supportsBuiltinWebSearch, currently just OpenAI's /v1/responses).
// Code pill is gated on `supportsTools` only — external providers
// never give us code execution, so the pill must stay disabled even
// when Search is available.
const searchDisabled =
!modelLoaded || !(supportsTools || supportsBuiltinWebSearch);
const codeDisabled = !modelLoaded || !supportsTools;
// Backwards-compatible alias for any other call site that may still
// reference `toolsDisabled` (rare; both pills used it before).
const toolsDisabled = codeDisabled;
const setPendingAudioStore = useChatRuntimeStore((s) => s.setPendingAudio);
const clearPendingAudioStore = useChatRuntimeStore((s) => s.clearPendingAudio);
@ -766,6 +787,11 @@ export function SharedComposer({
setReasoningEffort(level);
setReasoningEnabled(true);
applyQwenThinkingParams(true);
// Mutual exclusion: turning thinking on for a
// Kimi model forces the web_search builtin off.
if (isKimiExternal && toolsEnabled) {
setToolsEnabled(false);
}
}}
>
{formatReasoningEffortLabel(level, externalSelection?.modelId)}
@ -789,6 +815,12 @@ export function SharedComposer({
const next = !reasoningEnabled;
setReasoningEnabled(next);
applyQwenThinkingParams(next);
// Mutual exclusion: Kimi's $web_search builtin
// requires thinking off, so turning thinking on flips
// the Search pill off (and vice versa).
if (isKimiExternal && next && toolsEnabled) {
setToolsEnabled(false);
}
}}
className={cn(
"flex items-center gap-1.5 rounded-full px-2.5 py-1 text-xs font-medium transition-colors",
@ -845,10 +877,22 @@ export function SharedComposer({
)}
<button
type="button"
disabled={toolsDisabled}
onClick={() => setToolsEnabled(!toolsEnabled)}
disabled={searchDisabled}
onClick={() => {
const next = !toolsEnabled;
setToolsEnabled(next);
// Kimi's $web_search builtin requires thinking=disabled
// (https://platform.kimi.ai/docs/guide/use-web-search).
// Toggle the Think pill off when Search comes on, and
// back on when Search goes off — mutual exclusion that
// mirrors what the backend enforces.
if (isKimiExternal) {
setReasoningEnabled(!next);
applyQwenThinkingParams(!next);
}
}}
className="composer-pill-btn"
data-active={toolsEnabled && !toolsDisabled ? "true" : "false"}
data-active={toolsEnabled && !searchDisabled ? "true" : "false"}
aria-label={toolsEnabled ? "Disable web search" : "Enable web search"}
>
<GlobeIcon className="size-3.5" />
@ -856,10 +900,10 @@ export function SharedComposer({
</button>
<button
type="button"
disabled={toolsDisabled}
disabled={codeDisabled}
onClick={() => setCodeToolsEnabled(!codeToolsEnabled)}
className="composer-pill-btn"
data-active={codeToolsEnabled && !toolsDisabled ? "true" : "false"}
data-active={codeToolsEnabled && !codeDisabled ? "true" : "false"}
aria-label={codeToolsEnabled ? "Disable code execution" : "Enable code execution"}
>
<CodeToggleIcon className="size-3.5" />

12
studio/frontend/src/features/chat/stores/chat-runtime-store.ts
View file

@ -229,6 +229,16 @@ type ChatRuntimeStore = {
supportsPreserveThinking: boolean;
preserveThinking: boolean;
supportsTools: boolean;
/**
* Whether the active external provider exposes a server-side
* web_search tool (OpenAI's /v1/responses today). Distinct from
* `supportsTools` — that flag governs the local tool runtime (Code,
* python sandbox, our DuckDuckGo web_search). This one only enables
* the chat composer's Search pill for external models and leaves
* the Code pill disabled, because external providers do not give
* us code execution. Local models keep `supportsTools` only.
*/
supportsBuiltinWebSearch: boolean;
toolsEnabled: boolean;
codeToolsEnabled: boolean;
toolStatus: string | null;
@ -320,6 +330,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set) => ({
supportsPreserveThinking: false,
preserveThinking: loadBool(PRESERVE_THINKING_KEY, false),
supportsTools: false,
supportsBuiltinWebSearch: false,
toolsEnabled: false,
codeToolsEnabled: false,
toolStatus: null,
@ -430,6 +441,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set) => ({
reasoningEffortLevels: ["low", "medium", "high"],
supportsPreserveThinking: false,
supportsTools: false,
supportsBuiltinWebSearch: false,
toolsEnabled: false,
codeToolsEnabled: false,
toolStatus: null,