ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main' into studio-sandbox-hardening

Browse source
This commit is contained in:
Daniel Han 2026-05-19 10:58:55 +00:00
commit 6efb0f64ea
9 changed files with 146 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

31
studio/backend/core/inference/llama_cpp.py
View file

@ -471,8 +471,9 @@ def _is_mtp_model_name(
def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool:
"""User passed --spec-type / --spec-default? llama-server accumulates
repeated --spec-type, so we suppress auto-emit when this is true."""
"""User passed --spec-type / --spec-default? llama-server takes a
single --spec-type (comma-separated to chain), so suppress
auto-emit when this is true."""
if not extra_args:
return False
for raw in extra_args:
@ -2631,10 +2632,10 @@ class LlamaCppBackend:
# Qwen3-235B offloaded | 12 t/s | 21 t/s | 1.8x
# gpt-oss-120b repeat (92% accept)| 181 t/s | 814 t/s | 4.5x
#
# Params from llama.cpp docs (docs/speculative.md):
# --spec-ngram-size-n 24 (small n not recommended)
# --draft-min 48 --draft-max 64 (MoEs need long drafts;
# dense models can reduce these)
# Params from llama.cpp server README:
# --spec-ngram-mod-n-match 24 (lookup length)
# --spec-ngram-mod-n-min 48 --spec-ngram-mod-n-max 64
# (MoEs need long drafts; dense models can reduce these)
# ref: https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md
# ref: https://github.com/ggml-org/llama.cpp/pull/19164
# ref: https://github.com/ggml-org/llama.cpp/pull/18471
@ -2692,20 +2693,22 @@ class LlamaCppBackend:
]
)
else:
# CPU/Mac: chain ngram-mod + MTP in one
# comma-separated --spec-type (not repeated).
# ngram-mod knobs match llama.cpp defaults
# (n-match 24, n-min 48, n-max 64).
cmd.extend(
[
"--spec-type",
mtp_token,
f"ngram-mod,{mtp_token}",
"--spec-draft-n-max",
"3",
"--spec-type",
"ngram-mod",
"--spec-ngram-mod-n-match",
"24",
"--spec-ngram-mod-n-min",
"48",
"--spec-ngram-mod-n-max",
"6",
"64",
]
)
self._speculative_type = "draft-mtp"
@ -2715,13 +2718,15 @@ class LlamaCppBackend:
elif normalized_spec in _valid_spec_types:
cmd.extend(["--spec-type", normalized_spec])
if normalized_spec == "ngram-mod":
# llama.cpp defaults; legacy --spec-ngram-size-n
# / --draft-{min,max} were removed for ngram-mod.
cmd.extend(
[
"--spec-ngram-size-n",
"--spec-ngram-mod-n-match",
"24",
"--draft-min",
"--spec-ngram-mod-n-min",
"48",
"--draft-max",
"--spec-ngram-mod-n-max",
"64",
]
)

6
studio/backend/tests/test_llama_server_args.py
View file

@ -47,17 +47,15 @@ from core.inference.llama_server_args import (
["--spec-type", "draft-mtp", "--spec-draft-n-max", "6"],
[
"--spec-type",
"draft-mtp",
"ngram-mod,draft-mtp",
"--spec-draft-n-max",
"3",
"--spec-type",
"ngram-mod",
"--spec-ngram-mod-n-match",
"24",
"--spec-ngram-mod-n-min",
"48",
"--spec-ngram-mod-n-max",
"6",
"64",
],
# Reasoning controls
["--reasoning-format", "deepseek"],

11
studio/frontend/src/index.css
View file

@ -1172,11 +1172,18 @@
mix-blend-mode: normal;
}
/* Override sonner's hardcoded top: 0 on the toast close button. */
/* Override sonner top: 0 and pin to theme tokens (--gray2 hover ignores data-sonner-theme). */
[data-sonner-toast][data-styled="true"] [data-close-button] {
top: 8px !important;
background: var(--popover) !important;
color: var(--popover-foreground) !important;
border-color: var(--border) !important;
}
/* Bump the X stroke so it stays visible against dark backgrounds. */
[data-sonner-toast][data-styled="true"] [data-close-button] svg {
stroke-width: 2.25;
}
[data-sonner-toast][data-styled="true"]:hover [data-close-button]:hover {
background: var(--muted) !important;
color: var(--popover-foreground) !important;
border-color: var(--border) !important;
}

57
studio/frontend/src/lib/toast.ts
View file

@ -1,59 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
// sonner `toast` wrapper that defaults `dismissible: false` so swipe
// capture doesn't block text selection. Drop-in for `from "sonner"`.
import { toast as sonnerToast, type ExternalToast } from "sonner";
type AnyFn = (...args: unknown[]) => unknown;
function withDismissibleFalse<F extends AnyFn>(fn: F): F {
return ((...args: unknown[]) => {
// Branch by arity: React-element messages are objects too.
if (args.length <= 1) {
args.push({ dismissible: false } satisfies ExternalToast);
} else {
const lastIdx = args.length - 1;
const last = args[lastIdx];
if (last && typeof last === "object" && !Array.isArray(last)) {
const opts = last as ExternalToast;
if (!("dismissible" in opts)) {
args[lastIdx] = { dismissible: false, ...opts };
}
}
}
return fn(...args);
}) as F;
}
const wrappedCallable = withDismissibleFalse(
sonnerToast as unknown as AnyFn,
) as typeof sonnerToast;
// `promise(p, data?)` carries `dismissible` at the top of `data`,
// covering loading / success / error states. `dismiss`, `getHistory`,
// `getToasts` take no options.
const wrappedPromise: typeof sonnerToast.promise = ((promise, data) => {
const merged =
data && typeof data === "object" && !("dismissible" in data)
? { dismissible: false, ...data }
: (data ?? { dismissible: false });
return sonnerToast.promise(promise, merged);
}) as typeof sonnerToast.promise;
export const toast: typeof sonnerToast = Object.assign(wrappedCallable, {
success: withDismissibleFalse(sonnerToast.success.bind(sonnerToast) as AnyFn) as typeof sonnerToast.success,
info: withDismissibleFalse(sonnerToast.info.bind(sonnerToast) as AnyFn) as typeof sonnerToast.info,
warning: withDismissibleFalse(sonnerToast.warning.bind(sonnerToast) as AnyFn) as typeof sonnerToast.warning,
error: withDismissibleFalse(sonnerToast.error.bind(sonnerToast) as AnyFn) as typeof sonnerToast.error,
message: withDismissibleFalse(sonnerToast.message.bind(sonnerToast) as AnyFn) as typeof sonnerToast.message,
loading: withDismissibleFalse(sonnerToast.loading.bind(sonnerToast) as AnyFn) as typeof sonnerToast.loading,
custom: withDismissibleFalse(sonnerToast.custom.bind(sonnerToast) as AnyFn) as typeof sonnerToast.custom,
promise: wrappedPromise,
dismiss: sonnerToast.dismiss.bind(sonnerToast) as typeof sonnerToast.dismiss,
getHistory: sonnerToast.getHistory.bind(sonnerToast) as typeof sonnerToast.getHistory,
getToasts: sonnerToast.getToasts.bind(sonnerToast) as typeof sonnerToast.getToasts,
});
// Re-export of sonner. Swipe blocking lives on the Toaster via
// `swipeDirections={[]}`, so no per-toast dismissible override.
export { toast } from "sonner";
export type { ExternalToast } from "sonner";

67
tests/test_import_fixes_drift.py
View file

@ -545,6 +545,73 @@ def test_transformers_pretrained_model_has_get_input_embeddings():
# ===========================================================================
# ===========================================================================
# transformers LOSS_MAPPING -- patch_loss_functions() coverage
# Regression for https://github.com/unslothai/unsloth/issues/4188:
# Qwen3_5ForConditionalGeneration has loss_type='ForConditionalGeneration',
# a separate LOSS_MAPPING key that was never patched, leaving the model with
# the stock ForCausalLMLoss which does logits.float() and OOMs on <=24 GB GPUs.
# ===========================================================================
def _reset_loss_mapping(mapping, saved):
mapping.clear()
mapping.update(saved)
def test_patch_loss_functions_covers_conditional_generation():
"""After patch_loss_functions(), every LOSS_MAPPING key that was aliased
to ForCausalLMLoss must also point at the Unsloth kernel -- not just
LOSS_MAPPING['ForCausalLM']."""
lu = pytest.importorskip("transformers.loss.loss_utils")
cel = pytest.importorskip("unsloth.kernels.cross_entropy_loss")
saved = dict(lu.LOSS_MAPPING)
try:
cel.patch_loss_functions(torch_compile = False)
unsloth_loss = lu.LOSS_MAPPING.get("ForCausalLM")
assert unsloth_loss is not None
assert "Unsloth" in str(
unsloth_loss
), f"LOSS_MAPPING['ForCausalLM'] was not replaced: {unsloth_loss}"
cg_loss = lu.LOSS_MAPPING.get("ForConditionalGeneration")
assert cg_loss is unsloth_loss, (
f"LOSS_MAPPING['ForConditionalGeneration'] not patched: {cg_loss}. "
f"Qwen3_5ForConditionalGeneration will silently use the stock "
f"ForCausalLMLoss and OOM at large sequence lengths."
)
finally:
_reset_loss_mapping(lu.LOSS_MAPPING, saved)
def test_patch_loss_functions_does_not_touch_other_loss_types():
"""patch_loss_functions() must not overwrite unrelated loss types
(segmentation, detection, masked-LM, etc.) with the causal-LM kernel."""
lu = pytest.importorskip("transformers.loss.loss_utils")
cel = pytest.importorskip("unsloth.kernels.cross_entropy_loss")
non_causal_keys = {
k
for k, v in lu.LOSS_MAPPING.items()
if getattr(v, "__name__", "") != "ForCausalLMLoss"
}
saved = dict(lu.LOSS_MAPPING)
try:
cel.patch_loss_functions(torch_compile = False)
unsloth_loss = lu.LOSS_MAPPING.get("ForCausalLM")
for key in non_causal_keys:
assert lu.LOSS_MAPPING.get(key) is not unsloth_loss, (
f"patch_loss_functions() incorrectly overwrote "
f"LOSS_MAPPING['{key}'] with the Unsloth ForCausalLM kernel."
)
finally:
_reset_loss_mapping(lu.LOSS_MAPPING, saved)
def test_accelerate_utils_imports_module_present():
"""``disable_broken_wandb`` + ``fix_trl_vllm_ascend`` (import_fixes.py
493-516, 1320-1372). Both reach into accelerate.utils.imports."""

7
unsloth/dataprep/synthetic.py
View file

@ -21,7 +21,12 @@ from collections import deque
import time
import os
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
_OFFLINE_VALS = {"1", "true", "yes", "on"}
if not (
os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _OFFLINE_VALS
or os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _OFFLINE_VALS
):
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
import requests
import torch
import gc

15
unsloth/kernels/cross_entropy_loss.py
View file

@ -461,3 +461,18 @@ if (Version(torch.__version__) < Version("2.4.0")) and not hasattr(
# Patch CE Losses in transformers
def patch_loss_functions(torch_compile = True):
_patch_loss_functions(fast_cross_entropy_loss, torch_compile = torch_compile)
# Defense-in-depth sweep for LOSS_MAPPING aliases still pointing at the
# stock ForCausalLMLoss (e.g. ForConditionalGeneration for Qwen3.5,
# CsmForConditionalGeneration). unsloth_zoo also does this; remove once
# the floor pin moves past unslothai/unsloth-zoo#656.
try:
import transformers.loss.loss_utils as _lu
_unsloth_loss = _lu.LOSS_MAPPING.get("ForCausalLM")
if _unsloth_loss is not None:
for _key, _fn in list(_lu.LOSS_MAPPING.items()):
if getattr(_fn, "__name__", "") == "ForCausalLMLoss":
_lu.LOSS_MAPPING[_key] = _unsloth_loss
except (ImportError, AttributeError):
pass

7
unsloth/models/_utils.py
View file

@ -1771,6 +1771,13 @@ def get_statistics(local_files_only = False):
return
if local_files_only:
return
# Also skip when HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE are set.
_offline_vals = {"1", "true", "yes", "on"}
if (
os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline_vals
or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline_vals
):
return
from huggingface_hub.utils import (
disable_progress_bars,
enable_progress_bars,

19
unsloth/models/loader.py
View file

@ -308,6 +308,16 @@ class FastLanguageModel(FastLlamaModel):
if is_dist:
device_map = distributed_device_map
# Honour offline env vars BEFORE FastModel delegation so 8bit /
# full-finetuning / qat paths also receive local_files_only.
if not kwargs.get("local_files_only", False):
_offline = {"1", "true", "yes", "on"}
if (
os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline
or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline
):
kwargs["local_files_only"] = True
if load_in_8bit or full_finetuning or qat_scheme is not None:
return FastModel.from_pretrained(
model_name = model_name,
@ -1055,6 +1065,15 @@ class FastModel(FastBaseModel):
model_config = None
peft_config = None
local_files_only = kwargs.get("local_files_only", False)
# Mirror env-var fallback for direct callers (FastVisionModel / FastTextModel).
if not local_files_only:
_offline = {"1", "true", "yes", "on"}
if (
os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _offline
or os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _offline
):
local_files_only = True
kwargs["local_files_only"] = True
try:
model_config = AutoConfig.from_pretrained(