diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 21f2fe71b5..95a8c26a3a 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -471,8 +471,9 @@ def _is_mtp_model_name( def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: - """User passed --spec-type / --spec-default? llama-server accumulates - repeated --spec-type, so we suppress auto-emit when this is true.""" + """User passed --spec-type / --spec-default? llama-server takes a + single --spec-type (comma-separated to chain), so suppress + auto-emit when this is true.""" if not extra_args: return False for raw in extra_args: @@ -2631,10 +2632,10 @@ class LlamaCppBackend: # Qwen3-235B offloaded | 12 t/s | 21 t/s | 1.8x # gpt-oss-120b repeat (92% accept)| 181 t/s | 814 t/s | 4.5x # - # Params from llama.cpp docs (docs/speculative.md): - # --spec-ngram-size-n 24 (small n not recommended) - # --draft-min 48 --draft-max 64 (MoEs need long drafts; - # dense models can reduce these) + # Params from llama.cpp server README: + # --spec-ngram-mod-n-match 24 (lookup length) + # --spec-ngram-mod-n-min 48 --spec-ngram-mod-n-max 64 + # (MoEs need long drafts; dense models can reduce these) # ref: https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md # ref: https://github.com/ggml-org/llama.cpp/pull/19164 # ref: https://github.com/ggml-org/llama.cpp/pull/18471 @@ -2692,20 +2693,22 @@ class LlamaCppBackend: ] ) else: + # CPU/Mac: chain ngram-mod + MTP in one + # comma-separated --spec-type (not repeated). + # ngram-mod knobs match llama.cpp defaults + # (n-match 24, n-min 48, n-max 64). cmd.extend( [ "--spec-type", - mtp_token, + f"ngram-mod,{mtp_token}", "--spec-draft-n-max", "3", - "--spec-type", - "ngram-mod", "--spec-ngram-mod-n-match", "24", "--spec-ngram-mod-n-min", "48", "--spec-ngram-mod-n-max", - "6", + "64", ] ) self._speculative_type = "draft-mtp" @@ -2715,13 +2718,15 @@ class LlamaCppBackend: elif normalized_spec in _valid_spec_types: cmd.extend(["--spec-type", normalized_spec]) if normalized_spec == "ngram-mod": + # llama.cpp defaults; legacy --spec-ngram-size-n + # / --draft-{min,max} were removed for ngram-mod. cmd.extend( [ - "--spec-ngram-size-n", + "--spec-ngram-mod-n-match", "24", - "--draft-min", + "--spec-ngram-mod-n-min", "48", - "--draft-max", + "--spec-ngram-mod-n-max", "64", ] ) diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py index f4dabfcf08..68a1c870fb 100644 --- a/studio/backend/tests/test_llama_server_args.py +++ b/studio/backend/tests/test_llama_server_args.py @@ -47,17 +47,15 @@ from core.inference.llama_server_args import ( ["--spec-type", "draft-mtp", "--spec-draft-n-max", "6"], [ "--spec-type", - "draft-mtp", + "ngram-mod,draft-mtp", "--spec-draft-n-max", "3", - "--spec-type", - "ngram-mod", "--spec-ngram-mod-n-match", "24", "--spec-ngram-mod-n-min", "48", "--spec-ngram-mod-n-max", - "6", + "64", ], # Reasoning controls ["--reasoning-format", "deepseek"], diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.css index dc73112994..7afa87f597 100644 --- a/studio/frontend/src/index.css +++ b/studio/frontend/src/index.css @@ -1172,11 +1172,18 @@ mix-blend-mode: normal; } -/* Override sonner's hardcoded top: 0 on the toast close button. */ +/* Override sonner top: 0 and pin to theme tokens (--gray2 hover ignores data-sonner-theme). */ [data-sonner-toast][data-styled="true"] [data-close-button] { top: 8px !important; + background: var(--popover) !important; + color: var(--popover-foreground) !important; + border-color: var(--border) !important; } -/* Bump the X stroke so it stays visible against dark backgrounds. */ [data-sonner-toast][data-styled="true"] [data-close-button] svg { stroke-width: 2.25; } +[data-sonner-toast][data-styled="true"]:hover [data-close-button]:hover { + background: var(--muted) !important; + color: var(--popover-foreground) !important; + border-color: var(--border) !important; +} diff --git a/studio/frontend/src/lib/toast.ts b/studio/frontend/src/lib/toast.ts index 138ceb4c4d..6b1635b42e 100644 --- a/studio/frontend/src/lib/toast.ts +++ b/studio/frontend/src/lib/toast.ts @@ -1,59 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -// sonner `toast` wrapper that defaults `dismissible: false` so swipe -// capture doesn't block text selection. Drop-in for `from "sonner"`. - -import { toast as sonnerToast, type ExternalToast } from "sonner"; - -type AnyFn = (...args: unknown[]) => unknown; - -function withDismissibleFalse(fn: F): F { - return ((...args: unknown[]) => { - // Branch by arity: React-element messages are objects too. - if (args.length <= 1) { - args.push({ dismissible: false } satisfies ExternalToast); - } else { - const lastIdx = args.length - 1; - const last = args[lastIdx]; - if (last && typeof last === "object" && !Array.isArray(last)) { - const opts = last as ExternalToast; - if (!("dismissible" in opts)) { - args[lastIdx] = { dismissible: false, ...opts }; - } - } - } - return fn(...args); - }) as F; -} - -const wrappedCallable = withDismissibleFalse( - sonnerToast as unknown as AnyFn, -) as typeof sonnerToast; - -// `promise(p, data?)` carries `dismissible` at the top of `data`, -// covering loading / success / error states. `dismiss`, `getHistory`, -// `getToasts` take no options. -const wrappedPromise: typeof sonnerToast.promise = ((promise, data) => { - const merged = - data && typeof data === "object" && !("dismissible" in data) - ? { dismissible: false, ...data } - : (data ?? { dismissible: false }); - return sonnerToast.promise(promise, merged); -}) as typeof sonnerToast.promise; - -export const toast: typeof sonnerToast = Object.assign(wrappedCallable, { - success: withDismissibleFalse(sonnerToast.success.bind(sonnerToast) as AnyFn) as typeof sonnerToast.success, - info: withDismissibleFalse(sonnerToast.info.bind(sonnerToast) as AnyFn) as typeof sonnerToast.info, - warning: withDismissibleFalse(sonnerToast.warning.bind(sonnerToast) as AnyFn) as typeof sonnerToast.warning, - error: withDismissibleFalse(sonnerToast.error.bind(sonnerToast) as AnyFn) as typeof sonnerToast.error, - message: withDismissibleFalse(sonnerToast.message.bind(sonnerToast) as AnyFn) as typeof sonnerToast.message, - loading: withDismissibleFalse(sonnerToast.loading.bind(sonnerToast) as AnyFn) as typeof sonnerToast.loading, - custom: withDismissibleFalse(sonnerToast.custom.bind(sonnerToast) as AnyFn) as typeof sonnerToast.custom, - promise: wrappedPromise, - dismiss: sonnerToast.dismiss.bind(sonnerToast) as typeof sonnerToast.dismiss, - getHistory: sonnerToast.getHistory.bind(sonnerToast) as typeof sonnerToast.getHistory, - getToasts: sonnerToast.getToasts.bind(sonnerToast) as typeof sonnerToast.getToasts, -}); +// Re-export of sonner. Swipe blocking lives on the Toaster via +// `swipeDirections={[]}`, so no per-toast dismissible override. +export { toast } from "sonner"; export type { ExternalToast } from "sonner"; diff --git a/tests/test_import_fixes_drift.py b/tests/test_import_fixes_drift.py index f90556bf66..099b65d09a 100644 --- a/tests/test_import_fixes_drift.py +++ b/tests/test_import_fixes_drift.py @@ -545,6 +545,73 @@ def test_transformers_pretrained_model_has_get_input_embeddings(): # =========================================================================== +# =========================================================================== +# transformers LOSS_MAPPING -- patch_loss_functions() coverage +# Regression for https://github.com/unslothai/unsloth/issues/4188: +# Qwen3_5ForConditionalGeneration has loss_type='ForConditionalGeneration', +# a separate LOSS_MAPPING key that was never patched, leaving the model with +# the stock ForCausalLMLoss which does logits.float() and OOMs on <=24 GB GPUs. +# =========================================================================== + + +def _reset_loss_mapping(mapping, saved): + mapping.clear() + mapping.update(saved) + + +def test_patch_loss_functions_covers_conditional_generation(): + """After patch_loss_functions(), every LOSS_MAPPING key that was aliased + to ForCausalLMLoss must also point at the Unsloth kernel -- not just + LOSS_MAPPING['ForCausalLM'].""" + lu = pytest.importorskip("transformers.loss.loss_utils") + cel = pytest.importorskip("unsloth.kernels.cross_entropy_loss") + + saved = dict(lu.LOSS_MAPPING) + try: + cel.patch_loss_functions(torch_compile = False) + + unsloth_loss = lu.LOSS_MAPPING.get("ForCausalLM") + assert unsloth_loss is not None + assert "Unsloth" in str( + unsloth_loss + ), f"LOSS_MAPPING['ForCausalLM'] was not replaced: {unsloth_loss}" + + cg_loss = lu.LOSS_MAPPING.get("ForConditionalGeneration") + assert cg_loss is unsloth_loss, ( + f"LOSS_MAPPING['ForConditionalGeneration'] not patched: {cg_loss}. " + f"Qwen3_5ForConditionalGeneration will silently use the stock " + f"ForCausalLMLoss and OOM at large sequence lengths." + ) + finally: + _reset_loss_mapping(lu.LOSS_MAPPING, saved) + + +def test_patch_loss_functions_does_not_touch_other_loss_types(): + """patch_loss_functions() must not overwrite unrelated loss types + (segmentation, detection, masked-LM, etc.) with the causal-LM kernel.""" + lu = pytest.importorskip("transformers.loss.loss_utils") + cel = pytest.importorskip("unsloth.kernels.cross_entropy_loss") + + non_causal_keys = { + k + for k, v in lu.LOSS_MAPPING.items() + if getattr(v, "__name__", "") != "ForCausalLMLoss" + } + + saved = dict(lu.LOSS_MAPPING) + try: + cel.patch_loss_functions(torch_compile = False) + + unsloth_loss = lu.LOSS_MAPPING.get("ForCausalLM") + for key in non_causal_keys: + assert lu.LOSS_MAPPING.get(key) is not unsloth_loss, ( + f"patch_loss_functions() incorrectly overwrote " + f"LOSS_MAPPING['{key}'] with the Unsloth ForCausalLM kernel." + ) + finally: + _reset_loss_mapping(lu.LOSS_MAPPING, saved) + + def test_accelerate_utils_imports_module_present(): """``disable_broken_wandb`` + ``fix_trl_vllm_ascend`` (import_fixes.py 493-516, 1320-1372). Both reach into accelerate.utils.imports.""" diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 612c531f47..1be83c3bb9 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -21,7 +21,12 @@ from collections import deque import time import os -os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" +_OFFLINE_VALS = {"1", "true", "yes", "on"} +if not ( + os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _OFFLINE_VALS + or os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _OFFLINE_VALS +): + os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" import requests import torch import gc diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py index d92229314f..4a8f83ad04 100644 --- a/unsloth/kernels/cross_entropy_loss.py +++ b/unsloth/kernels/cross_entropy_loss.py @@ -461,3 +461,18 @@ if (Version(torch.__version__) < Version("2.4.0")) and not hasattr( # Patch CE Losses in transformers def patch_loss_functions(torch_compile = True): _patch_loss_functions(fast_cross_entropy_loss, torch_compile = torch_compile) + + # Defense-in-depth sweep for LOSS_MAPPING aliases still pointing at the + # stock ForCausalLMLoss (e.g. ForConditionalGeneration for Qwen3.5, + # CsmForConditionalGeneration). unsloth_zoo also does this; remove once + # the floor pin moves past unslothai/unsloth-zoo#656. + try: + import transformers.loss.loss_utils as _lu + + _unsloth_loss = _lu.LOSS_MAPPING.get("ForCausalLM") + if _unsloth_loss is not None: + for _key, _fn in list(_lu.LOSS_MAPPING.items()): + if getattr(_fn, "__name__", "") == "ForCausalLMLoss": + _lu.LOSS_MAPPING[_key] = _unsloth_loss + except (ImportError, AttributeError): + pass diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index a46d1f0c0e..f4313c4518 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1771,6 +1771,13 @@ def get_statistics(local_files_only = False): return if local_files_only: return + # Also skip when HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE are set. + _offline_vals = {"1", "true", "yes", "on"} + if ( + os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline_vals + or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline_vals + ): + return from huggingface_hub.utils import ( disable_progress_bars, enable_progress_bars, diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index fc91178d88..c10443e289 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -308,6 +308,16 @@ class FastLanguageModel(FastLlamaModel): if is_dist: device_map = distributed_device_map + # Honour offline env vars BEFORE FastModel delegation so 8bit / + # full-finetuning / qat paths also receive local_files_only. + if not kwargs.get("local_files_only", False): + _offline = {"1", "true", "yes", "on"} + if ( + os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline + or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline + ): + kwargs["local_files_only"] = True + if load_in_8bit or full_finetuning or qat_scheme is not None: return FastModel.from_pretrained( model_name = model_name, @@ -1055,6 +1065,15 @@ class FastModel(FastBaseModel): model_config = None peft_config = None local_files_only = kwargs.get("local_files_only", False) + # Mirror env-var fallback for direct callers (FastVisionModel / FastTextModel). + if not local_files_only: + _offline = {"1", "true", "yes", "on"} + if ( + os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline + or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline + ): + local_files_only = True + kwargs["local_files_only"] = True try: model_config = AutoConfig.from_pretrained(