From c4908b7929a25fc8f228e7da96164800c4466dbf Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Tue, 19 May 2026 00:55:55 -0700 Subject: [PATCH 1/4] studio: fix toast close-button click and light-mode hover (#5597) Two related issues on the chat toasts: 1. Close X did nothing. The lib/toast.ts wrapper defaulted every toast to `dismissible: false` (originally to keep swipe capture from stealing text selection). In sonner v2, `dismissible: false` makes the close-button onClick a no-op, so the X looked clickable but never dismissed the toast. The Toaster already sets `swipeDirections={[]}` in components/ui/sonner.tsx, so the per-toast swipe workaround is unnecessary and harmful. Replace the wrapper with a thin re-export of sonner. 2. Close X hover collapsed to a near-black circle in light mode. Sonner's default close-button styling uses fixed gray-scale tokens (--gray2 hover, --gray12 text) that ignore the theme attribute. Once the Toaster's inline style overrides --normal-bg with var(--popover), the base background follows the app theme but the hover state does not, so the hover bg lands on a color that has no contrast with the X glyph. Pin both base and hover to theme tokens (--popover, --muted, --popover-foreground, --border) so contrast stays visible in both light and dark modes. Repro: open chat, load any cached model, hover the X on the " loaded" toast in light mode -- before this change the circle turned dark and the click did nothing; after, the circle stays light and the click dismisses the toast. --- studio/frontend/src/index.css | 11 ++++-- studio/frontend/src/lib/toast.ts | 57 ++------------------------------ 2 files changed, 12 insertions(+), 56 deletions(-) diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.css index dc73112994..7afa87f597 100644 --- a/studio/frontend/src/index.css +++ b/studio/frontend/src/index.css @@ -1172,11 +1172,18 @@ mix-blend-mode: normal; } -/* Override sonner's hardcoded top: 0 on the toast close button. */ +/* Override sonner top: 0 and pin to theme tokens (--gray2 hover ignores data-sonner-theme). */ [data-sonner-toast][data-styled="true"] [data-close-button] { top: 8px !important; + background: var(--popover) !important; + color: var(--popover-foreground) !important; + border-color: var(--border) !important; } -/* Bump the X stroke so it stays visible against dark backgrounds. */ [data-sonner-toast][data-styled="true"] [data-close-button] svg { stroke-width: 2.25; } +[data-sonner-toast][data-styled="true"]:hover [data-close-button]:hover { + background: var(--muted) !important; + color: var(--popover-foreground) !important; + border-color: var(--border) !important; +} diff --git a/studio/frontend/src/lib/toast.ts b/studio/frontend/src/lib/toast.ts index 138ceb4c4d..6b1635b42e 100644 --- a/studio/frontend/src/lib/toast.ts +++ b/studio/frontend/src/lib/toast.ts @@ -1,59 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -// sonner `toast` wrapper that defaults `dismissible: false` so swipe -// capture doesn't block text selection. Drop-in for `from "sonner"`. - -import { toast as sonnerToast, type ExternalToast } from "sonner"; - -type AnyFn = (...args: unknown[]) => unknown; - -function withDismissibleFalse(fn: F): F { - return ((...args: unknown[]) => { - // Branch by arity: React-element messages are objects too. - if (args.length <= 1) { - args.push({ dismissible: false } satisfies ExternalToast); - } else { - const lastIdx = args.length - 1; - const last = args[lastIdx]; - if (last && typeof last === "object" && !Array.isArray(last)) { - const opts = last as ExternalToast; - if (!("dismissible" in opts)) { - args[lastIdx] = { dismissible: false, ...opts }; - } - } - } - return fn(...args); - }) as F; -} - -const wrappedCallable = withDismissibleFalse( - sonnerToast as unknown as AnyFn, -) as typeof sonnerToast; - -// `promise(p, data?)` carries `dismissible` at the top of `data`, -// covering loading / success / error states. `dismiss`, `getHistory`, -// `getToasts` take no options. -const wrappedPromise: typeof sonnerToast.promise = ((promise, data) => { - const merged = - data && typeof data === "object" && !("dismissible" in data) - ? { dismissible: false, ...data } - : (data ?? { dismissible: false }); - return sonnerToast.promise(promise, merged); -}) as typeof sonnerToast.promise; - -export const toast: typeof sonnerToast = Object.assign(wrappedCallable, { - success: withDismissibleFalse(sonnerToast.success.bind(sonnerToast) as AnyFn) as typeof sonnerToast.success, - info: withDismissibleFalse(sonnerToast.info.bind(sonnerToast) as AnyFn) as typeof sonnerToast.info, - warning: withDismissibleFalse(sonnerToast.warning.bind(sonnerToast) as AnyFn) as typeof sonnerToast.warning, - error: withDismissibleFalse(sonnerToast.error.bind(sonnerToast) as AnyFn) as typeof sonnerToast.error, - message: withDismissibleFalse(sonnerToast.message.bind(sonnerToast) as AnyFn) as typeof sonnerToast.message, - loading: withDismissibleFalse(sonnerToast.loading.bind(sonnerToast) as AnyFn) as typeof sonnerToast.loading, - custom: withDismissibleFalse(sonnerToast.custom.bind(sonnerToast) as AnyFn) as typeof sonnerToast.custom, - promise: wrappedPromise, - dismiss: sonnerToast.dismiss.bind(sonnerToast) as typeof sonnerToast.dismiss, - getHistory: sonnerToast.getHistory.bind(sonnerToast) as typeof sonnerToast.getHistory, - getToasts: sonnerToast.getToasts.bind(sonnerToast) as typeof sonnerToast.getToasts, -}); +// Re-export of sonner. Swipe blocking lives on the Toaster via +// `swipeDirections={[]}`, so no per-toast dismissible override. +export { toast } from "sonner"; export type { ExternalToast } from "sonner"; From 94026fc8dc8e23423ec6bf232ba9ba6646cba21e Mon Sep 17 00:00:00 2001 From: Junhyuk Lee <58055473+xodn348@users.noreply.github.com> Date: Tue, 19 May 2026 03:05:13 -0500 Subject: [PATCH 2/4] fix(loader): honour HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE in from_pretrained (#5598) Reads HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE in FastLanguageModel.from_pretrained and FastModel.from_pretrained, forcing local_files_only=True so all delegation paths (load_in_4bit, load_in_8bit, full_finetuning, qat_scheme) and direct FastModel callers (FastVisionModel, FastTextModel) honour offline mode. Also gates HF_HUB_ENABLE_HF_TRANSFER in unsloth/dataprep/synthetic.py and adds an early return in get_statistics. Pairs with unslothai/unsloth-zoo#675. Fixes #5316. --- unsloth/dataprep/synthetic.py | 7 ++++++- unsloth/models/_utils.py | 7 +++++++ unsloth/models/loader.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 612c531f47..1be83c3bb9 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -21,7 +21,12 @@ from collections import deque import time import os -os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" +_OFFLINE_VALS = {"1", "true", "yes", "on"} +if not ( + os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _OFFLINE_VALS + or os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _OFFLINE_VALS +): + os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" import requests import torch import gc diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index a46d1f0c0e..f4313c4518 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1771,6 +1771,13 @@ def get_statistics(local_files_only = False): return if local_files_only: return + # Also skip when HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE are set. + _offline_vals = {"1", "true", "yes", "on"} + if ( + os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline_vals + or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline_vals + ): + return from huggingface_hub.utils import ( disable_progress_bars, enable_progress_bars, diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index fc91178d88..c10443e289 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -308,6 +308,16 @@ class FastLanguageModel(FastLlamaModel): if is_dist: device_map = distributed_device_map + # Honour offline env vars BEFORE FastModel delegation so 8bit / + # full-finetuning / qat paths also receive local_files_only. + if not kwargs.get("local_files_only", False): + _offline = {"1", "true", "yes", "on"} + if ( + os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline + or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline + ): + kwargs["local_files_only"] = True + if load_in_8bit or full_finetuning or qat_scheme is not None: return FastModel.from_pretrained( model_name = model_name, @@ -1055,6 +1065,15 @@ class FastModel(FastBaseModel): model_config = None peft_config = None local_files_only = kwargs.get("local_files_only", False) + # Mirror env-var fallback for direct callers (FastVisionModel / FastTextModel). + if not local_files_only: + _offline = {"1", "true", "yes", "on"} + if ( + os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline + or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline + ): + local_files_only = True + kwargs["local_files_only"] = True try: model_config = AutoConfig.from_pretrained( From 5ce4ab4d54d22af53581b0acf512dfee8d5ce987 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 19 May 2026 03:16:05 -0700 Subject: [PATCH 3/4] studio: emit one comma-chained --spec-type for CPU/Mac MTP path (#5575) * studio: emit one comma-chained --spec-type for CPU/Mac MTP path llama-server takes a single --spec-type whose value may be comma-separated to chain implementations (e.g. ngram-mod,draft-mtp). The CPU/Mac MTP branch in LlamaCppBackend.load_model was passing --spec-type twice in the same invocation, which is not the documented chaining mechanism and silently drops one of the two specs depending on llama.cpp's argv handling. Collapse the pair to --spec-type ngram-mod,{mtp_token} and update the stale _extra_args_set_spec_type docstring that claimed llama-server accumulates repeated --spec-type. Update the matching pass-through fixture in test_llama_server_args.py. * studio: align MTP ngram-mod knobs with llama.cpp upstream defaults Two correctness fixes against the llama.cpp server README: 1. The CPU/Mac comma-chained branch was emitting --spec-ngram-mod-n-max 6 with --spec-ngram-mod-n-min 48, which is nonsensical (min > max). Per the upstream default the value is 64. 2. The standalone ngram-mod branch was emitting --spec-ngram-size-n, --draft-min, --draft-max. llama.cpp removed those arg aliases for ngram-mod (they live only on the ngram-simple / map families now); the correct knobs are --spec-ngram-mod-n-match / n-min / n-max. Also refresh the inline comment block to point at the server README rather than the older docs/speculative.md draft- aliases. --- studio/backend/core/inference/llama_cpp.py | 31 +++++++++++-------- .../backend/tests/test_llama_server_args.py | 6 ++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 21f2fe71b5..95a8c26a3a 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -471,8 +471,9 @@ def _is_mtp_model_name( def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: - """User passed --spec-type / --spec-default? llama-server accumulates - repeated --spec-type, so we suppress auto-emit when this is true.""" + """User passed --spec-type / --spec-default? llama-server takes a + single --spec-type (comma-separated to chain), so suppress + auto-emit when this is true.""" if not extra_args: return False for raw in extra_args: @@ -2631,10 +2632,10 @@ class LlamaCppBackend: # Qwen3-235B offloaded | 12 t/s | 21 t/s | 1.8x # gpt-oss-120b repeat (92% accept)| 181 t/s | 814 t/s | 4.5x # - # Params from llama.cpp docs (docs/speculative.md): - # --spec-ngram-size-n 24 (small n not recommended) - # --draft-min 48 --draft-max 64 (MoEs need long drafts; - # dense models can reduce these) + # Params from llama.cpp server README: + # --spec-ngram-mod-n-match 24 (lookup length) + # --spec-ngram-mod-n-min 48 --spec-ngram-mod-n-max 64 + # (MoEs need long drafts; dense models can reduce these) # ref: https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md # ref: https://github.com/ggml-org/llama.cpp/pull/19164 # ref: https://github.com/ggml-org/llama.cpp/pull/18471 @@ -2692,20 +2693,22 @@ class LlamaCppBackend: ] ) else: + # CPU/Mac: chain ngram-mod + MTP in one + # comma-separated --spec-type (not repeated). + # ngram-mod knobs match llama.cpp defaults + # (n-match 24, n-min 48, n-max 64). cmd.extend( [ "--spec-type", - mtp_token, + f"ngram-mod,{mtp_token}", "--spec-draft-n-max", "3", - "--spec-type", - "ngram-mod", "--spec-ngram-mod-n-match", "24", "--spec-ngram-mod-n-min", "48", "--spec-ngram-mod-n-max", - "6", + "64", ] ) self._speculative_type = "draft-mtp" @@ -2715,13 +2718,15 @@ class LlamaCppBackend: elif normalized_spec in _valid_spec_types: cmd.extend(["--spec-type", normalized_spec]) if normalized_spec == "ngram-mod": + # llama.cpp defaults; legacy --spec-ngram-size-n + # / --draft-{min,max} were removed for ngram-mod. cmd.extend( [ - "--spec-ngram-size-n", + "--spec-ngram-mod-n-match", "24", - "--draft-min", + "--spec-ngram-mod-n-min", "48", - "--draft-max", + "--spec-ngram-mod-n-max", "64", ] ) diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py index f4dabfcf08..68a1c870fb 100644 --- a/studio/backend/tests/test_llama_server_args.py +++ b/studio/backend/tests/test_llama_server_args.py @@ -47,17 +47,15 @@ from core.inference.llama_server_args import ( ["--spec-type", "draft-mtp", "--spec-draft-n-max", "6"], [ "--spec-type", - "draft-mtp", + "ngram-mod,draft-mtp", "--spec-draft-n-max", "3", - "--spec-type", - "ngram-mod", "--spec-ngram-mod-n-match", "24", "--spec-ngram-mod-n-min", "48", "--spec-ngram-mod-n-max", - "6", + "64", ], # Reasoning controls ["--reasoning-format", "deepseek"], From 66cfbeac1d6dc26d177244861062b31cee2d65f1 Mon Sep 17 00:00:00 2001 From: swappy <59965507+rycerzes@users.noreply.github.com> Date: Tue, 19 May 2026 16:27:50 +0530 Subject: [PATCH 4/4] Fix loss function not patched for Qwen3.5 models (#5442) * fix: patch loss functions for Qwen3_5ForConditionalGeneration to prevent OOM errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Narrow except scope and simplify LOSS_MAPPING sweep Replace bare except Exception with the only two compatibility errors we actually care about so genuine bugs in the sweep surface. Drop the redundant _key != "ForCausalLM" guard since the __name__ predicate already excludes the patched entry (UnslothForCausalLMLoss != ForCausalLMLoss). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- tests/test_import_fixes_drift.py | 67 +++++++++++++++++++++++++++ unsloth/kernels/cross_entropy_loss.py | 15 ++++++ 2 files changed, 82 insertions(+) diff --git a/tests/test_import_fixes_drift.py b/tests/test_import_fixes_drift.py index f90556bf66..099b65d09a 100644 --- a/tests/test_import_fixes_drift.py +++ b/tests/test_import_fixes_drift.py @@ -545,6 +545,73 @@ def test_transformers_pretrained_model_has_get_input_embeddings(): # =========================================================================== +# =========================================================================== +# transformers LOSS_MAPPING -- patch_loss_functions() coverage +# Regression for https://github.com/unslothai/unsloth/issues/4188: +# Qwen3_5ForConditionalGeneration has loss_type='ForConditionalGeneration', +# a separate LOSS_MAPPING key that was never patched, leaving the model with +# the stock ForCausalLMLoss which does logits.float() and OOMs on <=24 GB GPUs. +# =========================================================================== + + +def _reset_loss_mapping(mapping, saved): + mapping.clear() + mapping.update(saved) + + +def test_patch_loss_functions_covers_conditional_generation(): + """After patch_loss_functions(), every LOSS_MAPPING key that was aliased + to ForCausalLMLoss must also point at the Unsloth kernel -- not just + LOSS_MAPPING['ForCausalLM'].""" + lu = pytest.importorskip("transformers.loss.loss_utils") + cel = pytest.importorskip("unsloth.kernels.cross_entropy_loss") + + saved = dict(lu.LOSS_MAPPING) + try: + cel.patch_loss_functions(torch_compile = False) + + unsloth_loss = lu.LOSS_MAPPING.get("ForCausalLM") + assert unsloth_loss is not None + assert "Unsloth" in str( + unsloth_loss + ), f"LOSS_MAPPING['ForCausalLM'] was not replaced: {unsloth_loss}" + + cg_loss = lu.LOSS_MAPPING.get("ForConditionalGeneration") + assert cg_loss is unsloth_loss, ( + f"LOSS_MAPPING['ForConditionalGeneration'] not patched: {cg_loss}. " + f"Qwen3_5ForConditionalGeneration will silently use the stock " + f"ForCausalLMLoss and OOM at large sequence lengths." + ) + finally: + _reset_loss_mapping(lu.LOSS_MAPPING, saved) + + +def test_patch_loss_functions_does_not_touch_other_loss_types(): + """patch_loss_functions() must not overwrite unrelated loss types + (segmentation, detection, masked-LM, etc.) with the causal-LM kernel.""" + lu = pytest.importorskip("transformers.loss.loss_utils") + cel = pytest.importorskip("unsloth.kernels.cross_entropy_loss") + + non_causal_keys = { + k + for k, v in lu.LOSS_MAPPING.items() + if getattr(v, "__name__", "") != "ForCausalLMLoss" + } + + saved = dict(lu.LOSS_MAPPING) + try: + cel.patch_loss_functions(torch_compile = False) + + unsloth_loss = lu.LOSS_MAPPING.get("ForCausalLM") + for key in non_causal_keys: + assert lu.LOSS_MAPPING.get(key) is not unsloth_loss, ( + f"patch_loss_functions() incorrectly overwrote " + f"LOSS_MAPPING['{key}'] with the Unsloth ForCausalLM kernel." + ) + finally: + _reset_loss_mapping(lu.LOSS_MAPPING, saved) + + def test_accelerate_utils_imports_module_present(): """``disable_broken_wandb`` + ``fix_trl_vllm_ascend`` (import_fixes.py 493-516, 1320-1372). Both reach into accelerate.utils.imports.""" diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py index d92229314f..4a8f83ad04 100644 --- a/unsloth/kernels/cross_entropy_loss.py +++ b/unsloth/kernels/cross_entropy_loss.py @@ -461,3 +461,18 @@ if (Version(torch.__version__) < Version("2.4.0")) and not hasattr( # Patch CE Losses in transformers def patch_loss_functions(torch_compile = True): _patch_loss_functions(fast_cross_entropy_loss, torch_compile = torch_compile) + + # Defense-in-depth sweep for LOSS_MAPPING aliases still pointing at the + # stock ForCausalLMLoss (e.g. ForConditionalGeneration for Qwen3.5, + # CsmForConditionalGeneration). unsloth_zoo also does this; remove once + # the floor pin moves past unslothai/unsloth-zoo#656. + try: + import transformers.loss.loss_utils as _lu + + _unsloth_loss = _lu.LOSS_MAPPING.get("ForCausalLM") + if _unsloth_loss is not None: + for _key, _fn in list(_lu.LOSS_MAPPING.items()): + if getattr(_fn, "__name__", "") == "ForCausalLMLoss": + _lu.LOSS_MAPPING[_key] = _unsloth_loss + except (ImportError, AttributeError): + pass