ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into pip

Browse source
This commit is contained in:
Daniel Han 2026-06-12 07:36:20 -07:00
commit 667467a315
7 changed files with 252 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

10
install.ps1
View file

@ -1921,7 +1921,7 @@ shell.Run cmd, 0, False
if ($SkipTorch) {
# No-torch: install unsloth + unsloth-zoo with --no-deps, then
# runtime deps (typer, safetensors, transformers, etc.) with --no-deps.
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.6.3" unsloth-zoo }
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.6.4" unsloth-zoo }
if ($baseInstallExit -eq 0) {
# Resolve pydantic WITH deps so pip pins pydantic-core
# to the matching version (no-torch-runtime.txt below
@ -1935,7 +1935,7 @@ shell.Run cmd, 0, False
}
}
} else {
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.6.3" unsloth-zoo }
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.6.4" unsloth-zoo }
}
if ($baseInstallExit -ne 0) {
Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red
@ -1982,7 +1982,7 @@ shell.Run cmd, 0, False
if ($SkipTorch) {
# No-torch: install unsloth + unsloth-zoo with --no-deps, then
# runtime deps (typer, safetensors, transformers, etc.) with --no-deps.
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.6.3" unsloth-zoo }
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.6.4" unsloth-zoo }
if ($baseInstallExit -eq 0) {
# Same pydantic-with-deps trick as the migrated branch.
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython pydantic }
@ -1994,7 +1994,7 @@ shell.Run cmd, 0, False
}
}
} elseif ($StudioLocalInstall) {
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.6.3" unsloth-zoo }
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.6.4" unsloth-zoo }
} else {
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth -- "$PackageName" }
}
@ -2022,7 +2022,7 @@ shell.Run cmd, 0, False
Write-TauriLog "STEP" "Installing unsloth"
substep "installing unsloth (this may take a few minutes)..."
if ($StudioLocalInstall) {
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.6.3" --torch-backend=auto }
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.6.4" --torch-backend=auto }
if ($baseInstallExit -ne 0) {
Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red
return (Exit-InstallFailure "Failed to install unsloth (exit code $baseInstallExit)" $baseInstallExit)

15
install.sh
View file

@ -2422,7 +2422,7 @@ if [ "$_MIGRATED" = true ]; then
# to prevent transitive torch resolution.
run_install_cmd "install unsloth (migrated no-torch)" uv pip install --python "$_VENV_PY" --no-deps \
--reinstall-package unsloth --reinstall-package unsloth-zoo \
"unsloth>=2026.6.3" unsloth-zoo
"unsloth>=2026.6.4" unsloth-zoo
# Resolve pydantic WITH deps so pip pins pydantic-core to the
# matching version (no-torch-runtime.txt below is --no-deps).
# All transitive deps are torch-free.
@ -2435,7 +2435,7 @@ if [ "$_MIGRATED" = true ]; then
else
run_install_cmd "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \
--reinstall-package unsloth --reinstall-package unsloth-zoo \
"unsloth>=2026.6.3" unsloth-zoo
"unsloth>=2026.6.4" unsloth-zoo
fi
if [ "$STUDIO_LOCAL_INSTALL" = true ]; then
substep "overlaying local repo (editable)..."
@ -2639,7 +2639,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then
# runtime deps (typer, safetensors, transformers, etc.) with --no-deps.
run_install_cmd "install unsloth (no-torch)" uv pip install --python "$_VENV_PY" --no-deps \
--upgrade-package unsloth --upgrade-package unsloth-zoo \
"unsloth>=2026.6.3" unsloth-zoo
"unsloth>=2026.6.4" unsloth-zoo
# Same pydantic-with-deps trick as the migrated branch.
run_install_cmd "install pydantic (with deps for compatible core)" \
uv pip install --python "$_VENV_PY" pydantic
@ -2657,7 +2657,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then
fi
elif [ "$STUDIO_LOCAL_INSTALL" = true ]; then
run_install_cmd "install unsloth (local)" uv pip install --python "$_VENV_PY" \
--upgrade-package unsloth "unsloth>=2026.6.3" unsloth-zoo
--upgrade-package unsloth "unsloth>=2026.6.4" unsloth-zoo
substep "overlaying local repo (editable)..."
run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps
substep "overlaying unsloth-zoo from git main..."
@ -2689,7 +2689,7 @@ else
tauri_log "STEP" "Installing Unsloth"
substep "installing unsloth (this may take a few minutes)..."
if [ "$STUDIO_LOCAL_INSTALL" = true ]; then
run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.6.3" --torch-backend=auto
run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.6.4" --torch-backend=auto
substep "overlaying local repo (editable)..."
run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps
substep "overlaying unsloth-zoo from git main..."
@ -2900,7 +2900,10 @@ if [ -t 1 ]; then
case "${_reply:-y}" in
[Yy]*|"")
step "launch" "starting Unsloth Studio..."
"$VENV_DIR/bin/unsloth" studio -p 8888
# Detach stdin from the `curl | sh` pipe: as a foreground server the
# studio would otherwise drain the rest of this piped script, leaving
# the shell to die parsing the now-truncated tail (`unexpected fi`).
"$VENV_DIR/bin/unsloth" studio -p 8888 </dev/null
_LAUNCH_EXIT=$?
if [ "$_LAUNCH_EXIT" -ne 0 ] && [ "$_MIGRATED" = true ]; then
echo ""

59
studio/backend/core/inference/llama_cpp.py
View file

@ -2527,6 +2527,10 @@ class LlamaCppBackend:
]
env = child_env_without_native_path_secret()
# `python -m unsloth_zoo.diffusion_studio.shim` imports unsloth_zoo, which
# refuses to load unless UNSLOTH_IS_PRESENT is set (normally by `import
# unsloth`). The shim never imports unsloth, so set it here as unsloth does.
env["UNSLOTH_IS_PRESENT"] = "1"
env["DG_VISUAL_BIN"] = visual_bin
env["DG_GPU"] = gpu
# The file-override shim imports its sibling visual_engine; put its dir on PYTHONPATH.
@ -2996,6 +3000,16 @@ class LlamaCppBackend:
return str(mmproj)
def _mmproj_vram_bytes(self, launch_mmproj_path: Optional[str]) -> int:
"""Return resolved mmproj VRAM bytes, or 0 when absent/unreadable."""
if not launch_mmproj_path:
return 0
try:
return self._get_gguf_size_bytes(launch_mmproj_path)
except OSError as e:
logger.debug(f"Could not size mmproj {launch_mmproj_path}: {e}")
return 0
def _resolve_launch_mtp_path(self, *, mtp_draft_path: Optional[str]) -> Optional[str]:
"""Return mtp_draft_path iff it exists on disk, else None.
@ -3561,8 +3575,29 @@ class LlamaCppBackend:
effective_ctx = requested_ctx if requested_ctx > 0 else (self._context_length or 0)
max_available_ctx = self._context_length or effective_ctx
gpus: list[tuple[int, int]] = []
# Keep fit-budget and launch-flag mmproj resolution in sync.
launch_mmproj_path = None
if not extra_args_disable_mmproj(extra_args):
launch_mmproj_path = self._resolve_launch_mmproj_path(
model_path = model_path,
mmproj_path = mmproj_path,
)
# Need both a resolved mmproj AND the config vision flag; a stray
# mmproj passing the family-name heuristic must not flip a non-VLM
# GGUF into vision mode.
effective_is_vision = bool(launch_mmproj_path) and bool(is_vision)
if is_vision and not effective_is_vision:
logger.warning(
"Vision-capable GGUF loaded without a usable mmproj; "
"image input will be disabled for this session"
)
try:
model_size = self._get_gguf_size_bytes(model_path)
gguf_size = self._get_gguf_size_bytes(model_path)
# Include GPU-loaded mmproj in the fit budget (#5825).
mmproj_size = (
self._mmproj_vram_bytes(launch_mmproj_path) if effective_is_vision else 0
)
model_size = gguf_size + mmproj_size
gpus = self._get_gpu_free_memory()
# Resolve effective context: 0 means let llama-server use
@ -3802,8 +3837,12 @@ class LlamaCppBackend:
kv_cache_bytes = self._estimate_kv_cache_bytes(
effective_ctx, cache_type_kv, n_parallel = n_parallel
)
mmproj_note = (
f"mmproj: {mmproj_size / (1024**3):.1f} GB, " if mmproj_size else ""
)
logger.info(
f"GGUF size: {model_size / (1024**3):.1f} GB, "
f"GGUF size: {gguf_size / (1024**3):.1f} GB, "
f"{mmproj_note}"
f"est. KV cache: {kv_cache_bytes / (1024**3):.1f} GB, "
f"context: {effective_ctx}, "
f"GPUs free: {gpus}, selected: {gpu_indices}, fit: {use_fit}"
@ -3814,22 +3853,6 @@ class LlamaCppBackend:
tp_tensor_split = None
effective_ctx = requested_ctx # fall back to original
launch_mmproj_path = None
if not extra_args_disable_mmproj(extra_args):
launch_mmproj_path = self._resolve_launch_mmproj_path(
model_path = model_path,
mmproj_path = mmproj_path,
)
# Need both a resolved mmproj AND the config vision flag; a stray
# mmproj passing the family-name heuristic must not flip a non-VLM
# GGUF into vision mode.
effective_is_vision = bool(launch_mmproj_path) and bool(is_vision)
if is_vision and not effective_is_vision:
logger.warning(
"Vision-capable GGUF loaded without a usable mmproj; "
"image input will be disabled for this session"
)
# Audio input straight from the mmproj (clip.has_audio_encoder),
# independent of token names.
self._mmproj_has_audio = False

40
studio/backend/tests/test_mmproj_vram_accounting.py Normal file
View file

@ -0,0 +1,40 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tests for mmproj VRAM accounting in GGUF fit budgeting (#5825)."""
from __future__ import annotations
from pathlib import Path
from core.inference.llama_cpp import LlamaCppBackend
def _write(path: Path, n_bytes: int) -> Path:
path.parent.mkdir(parents = True, exist_ok = True)
path.write_bytes(b"\x00" * n_bytes)
return path
def _backend() -> LlamaCppBackend:
return LlamaCppBackend.__new__(LlamaCppBackend)
def test_counts_resolved_projector_size(tmp_path: Path):
mmproj = _write(tmp_path / "Qwen3.5-9B-BF16-mmproj.gguf", 1024)
got = _backend()._mmproj_vram_bytes(str(mmproj))
assert got == 1024
def test_zero_when_no_projector_resolved(tmp_path: Path):
assert _backend()._mmproj_vram_bytes(None) == 0
def test_zero_when_projector_missing_on_disk(tmp_path: Path):
missing = tmp_path / "Qwen3.5-9B-BF16-mmproj.gguf" # never created
got = _backend()._mmproj_vram_bytes(str(missing))
assert got == 0

94
studio/backend/tests/test_vision_cache.py
View file

@ -30,6 +30,7 @@ _loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
from utils.models.model_config import (
ModelConfig,
is_vision_model,
_is_vision_model_uncached,
_vision_detection_cache,
@ -120,6 +121,99 @@ class TestVisionCacheSubprocessPath:
mock_raw_config.assert_called_once_with("unsloth/gemma-4-E4B-it", hf_token = None)
# ---------------------------------------------------------------------------
# Local GGUF capability path
# ---------------------------------------------------------------------------
class TestLocalGgufVisionDetection:
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_qwen36_gguf_with_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
(tmp_path / "mmproj-F32.gguf").write_bytes(b"")
assert is_vision_model(str(model)) is True
mock_subprocess.assert_not_called()
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_direct_gguf_in_variant_subdir_finds_snapshot_mmproj(self, mock_subprocess, tmp_path):
variant_dir = tmp_path / "BF16"
variant_dir.mkdir()
model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
(tmp_path / "mmproj-F32.gguf").write_bytes(b"")
assert is_vision_model(str(model)) is True
mock_subprocess.assert_not_called()
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_qwen36_gguf_without_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
assert is_vision_model(str(model)) is False
mock_subprocess.assert_not_called()
def test_local_gguf_check_observes_mmproj_added_later(self, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
assert is_vision_model(str(model)) is False
(tmp_path / "mmproj-F32.gguf").write_bytes(b"")
assert is_vision_model(str(model)) is True
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_ui_selection_returns_local_gguf_config(self, mock_subprocess, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
mmproj = tmp_path / "mmproj-F32.gguf"
mmproj.write_bytes(b"")
config = ModelConfig.from_ui_selection(str(model), None)
assert config is not None
assert config.is_gguf is True
assert config.is_vision is True
assert config.gguf_mmproj_file == str(mmproj.resolve())
mock_subprocess.assert_not_called()
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_ui_selection_direct_gguf_in_variant_subdir_keeps_mmproj(
self, mock_subprocess, tmp_path
):
variant_dir = tmp_path / "BF16"
variant_dir.mkdir()
model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
mmproj = tmp_path / "mmproj-F32.gguf"
mmproj.write_bytes(b"")
config = ModelConfig.from_ui_selection(str(model), None)
assert config is not None
assert config.is_gguf is True
assert config.is_vision is True
assert config.gguf_mmproj_file == str(mmproj.resolve())
mock_subprocess.assert_not_called()
# ---------------------------------------------------------------------------
# Exception handling — cache the False fallback

67
studio/backend/utils/models/model_config.py
View file

@ -721,6 +721,25 @@ def is_vision_model(model_name: str, hf_token: Optional[str] = None) -> bool:
model_name: Model identifier (HF repo or local path)
hf_token: Optional HF token for gated/private models
"""
# Local GGUF models are served by llama-server. Their multimodal
# capability comes from a companion mmproj, not a Transformers config.
# Do not cache this lookup: a projector may be added beside an existing
# weight file after it was first inspected.
if is_local_path(model_name):
local_path = normalize_path(model_name)
gguf_file = detect_gguf_model(local_path)
if gguf_file:
companion_root = _local_gguf_companion_search_root(local_path, gguf_file)
mmproj_file = detect_mmproj_file(gguf_file, search_root = companion_root)
is_vision = mmproj_file is not None
logger.debug(
"Local GGUF vision check for '%s': mmproj=%s, is_vision=%s",
gguf_file,
mmproj_file,
is_vision,
)
return is_vision
# Normalize model name so different casings of the same repo share a key
try:
if is_local_path(model_name):
@ -1358,6 +1377,35 @@ def _extract_quant_label(filename: str) -> str:
return stem.split("-")[-1]
def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str:
"""Directory to scan upward from for local GGUF companion files."""
import re
selected = Path(selected_path)
gguf_path = Path(gguf_file)
if selected.suffix.lower() != ".gguf":
return selected_path
gguf_dir = gguf_path.parent
if not gguf_dir.name:
return str(gguf_dir)
quant_dir_re = (
r"(UD-)?("
r"MXFP[0-9]+(?:_[A-Z0-9]+)*"
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
r"|TQ[0-9]+_[0-9]+"
r"|Q[0-9]+_K_[A-Z]+"
r"|Q[0-9]+_[0-9]+"
r"|Q[0-9]+_K"
r"|BF16|F16|F32"
r")"
)
if re.fullmatch(quant_dir_re, gguf_dir.name, re.IGNORECASE):
return str(gguf_dir.parent)
return str(gguf_dir)
def _iter_hf_cache_snapshots(repo_id: str):
"""Yield HF cache snapshot dirs for *repo_id*, newest first.
@ -2275,10 +2323,10 @@ class ModelConfig:
except Exception as e:
logger.debug(f"Could not read export metadata: {e}")
# Pass search_root=path so detect_mmproj_file walks up to the
# snapshot root: the weight may sit in a quant subdir while
# mmproj-*.gguf lives at the root.
mmproj_file = detect_mmproj_file(gguf_file, search_root = path)
# Direct file selections may point into a quant subdir while
# mmproj-*.gguf lives at the snapshot root.
companion_root = _local_gguf_companion_search_root(path, gguf_file)
mmproj_file = detect_mmproj_file(gguf_file, search_root = companion_root)
if mmproj_file:
gguf_is_vision = True
logger.info(f"Detected mmproj for vision: {mmproj_file}")
@ -2286,7 +2334,7 @@ class ModelConfig:
logger.warning(f"Base model is vision but no mmproj file found in {gguf_dir}")
# Separate MTP drafter sibling (Gemma 4), mirroring mmproj.
mtp_file = detect_mtp_file(gguf_file, search_root = path)
mtp_file = detect_mtp_file(gguf_file, search_root = companion_root)
if mtp_file:
logger.info(f"Detected MTP drafter: {mtp_file}")
@ -2476,6 +2524,15 @@ class ModelConfig:
identifier = resolved_identifier
path = resolved_identifier
# Keep existing local GGUF selections on the llama-server path. This
# constructor is still used by older inference helpers and must not
# describe a .gguf weight file as loadable by FastVisionModel.
if is_local and not is_lora and detect_gguf_model(path):
gguf_config = cls.from_identifier(path, hf_token = hf_token)
if gguf_config is not None:
gguf_config.display_name = display_name
return gguf_config
# --- Base Model and Vision Detection ---
base_model = None
is_vision = False

2
unsloth/models/_utils.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "2026.6.4"
__version__ = "2026.6.5"
__all__ = [
"SUPPORTS_BFLOAT16",