ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge origin/main into studio-autoload-on-device-only

Browse source 
Resolves the chat-api import conflict: keep GenerationLengthError from
main and keep the branch's move of listCachedGguf and listCachedModels
to the unified inventory API.
This commit is contained in:
Unsloth 2026-07-26 00:10:41 -07:00
commit 71341a1046
254 changed files with 13624 additions and 1654 deletions
Download patch file Download diff file Expand all files Collapse all files

57
.github/workflows/studio-backend-ci.yml vendored
View file

@ -30,6 +30,13 @@ on:
- 'unsloth/**'
- 'unsloth_cli/**'
- 'tests/**'
# The root installers: tests/sh/*.sh and tests/studio/install/* assert
# against these two files, so a change here must run the suite that
# covers it. Without them an install-only edit (the shape most AMD/ROCm
# routing fixes take) skipped Backend CI entirely.
- 'install.sh'
- 'install.ps1'
- 'scripts/**'
- 'pyproject.toml'
- '.github/workflows/studio-backend-ci.yml'
push:
@ -193,6 +200,7 @@ jobs:
--ignore=tests/sh \
--ignore=tests/studio/test_hardware_dispatch_matrix.py \
--ignore=tests/studio/test_is_mlx_dispatch_gate.py \
--ignore=tests/studio/test_xpu_spoof_pipeline.py \
--ignore=tests/vllm_compat \
--ignore=tests/version_compat \
-m 'not server and not e2e' \
@ -205,36 +213,43 @@ jobs:
env:
PYTHONPATH: ${{ github.workspace }}/studio
UNSLOTH_COMPILE_DISABLE: '1'
# These two files mutate hardware.py module globals at runtime
# via the spoof fixtures, which leaks state into any other test
# that imports hardware. Run them in their own pytest invocation
# so the leak does not cross file boundaries.
# These files mutate hardware.py module globals at runtime via the
# spoof fixtures (CUDA/ROCm/XPU/MLX/CPU), which leaks state into any
# other test that imports hardware. Run them in their own pytest
# invocation so the leak does not cross file boundaries.
run: |
python -m pytest -q --tb=short \
tests/studio/test_hardware_dispatch_matrix.py \
tests/studio/test_is_mlx_dispatch_gate.py
tests/studio/test_is_mlx_dispatch_gate.py \
tests/studio/test_xpu_spoof_pipeline.py
- name: Shell installer tests
# Subset that does not depend on a writable / pristine install.sh
# tree; test_install_host_defaults.sh checks install.ps1 layout
# which has drifted (separate followup).
# Auto-discovered rather than allowlisted. The old hardcoded list had
# silently fallen seven files behind tests/run_all.sh, including
# test_strixhalo_wsl_reroute.sh -- the only shell coverage of the ROCm
# WSL reroute -- so that suite never ran on a PR. Skips are explicit,
# each with a reason, and tests/studio/test_ci_shell_suite_coverage.py
# fails if this step stops discovering the directory or the skip list
# grows without one.
#
# Skipped:
# test_install_host_defaults.sh: asserts an install.ps1 layout that
# has drifted (separate followup).
# test_install_rollback_lifecycle.sh: already runs on both platforms
# in cross-platform-parity-ci.yml.
run: |
set -e
for s in \
tests/sh/test_get_torch_index_url.sh \
tests/sh/test_mac_intel_compat.sh \
tests/sh/test_node_decision.sh \
tests/sh/test_studio_home_node_dir.sh \
tests/sh/test_system_node_readonly.sh \
tests/sh/test_nvcc_meets_llama_minimum.sh \
tests/sh/test_resolve_cuda_archs.sh \
tests/sh/test_tauri_install_exit_order.sh \
tests/sh/test_torch_constraint.sh \
tests/sh/test_torch_flavor.sh \
tests/sh/test_with_llama_cpp_dir_flag.sh \
tests/sh/test_with_llama_cpp_dir_link_behavior.sh; do
skip="test_install_host_defaults.sh test_install_rollback_lifecycle.sh"
found=0
for s in tests/sh/test_*.sh; do
case " $skip " in
*" $(basename "$s") "*) echo "skipping $s (see workflow comment)"; continue ;;
esac
found=$((found + 1))
echo "::group::$s"
bash "$s"
echo "::endgroup::"
done
[ "$found" -gt 0 ] || { echo "::error::no shell tests discovered under tests/sh"; exit 1; }
echo "ran $found shell installer test files"

18
install.ps1
View file

@ -1917,12 +1917,14 @@ exit 0
# (gfx120X/110X/1151/1150/103X); unknown names fall back to CPU.
elseif ($ROCmGpuLabel) {
$nameArchTable = @(
@{ P = "9070 XT|9080"; A = "gfx1201" } # RDNA 4 (RX 9070 XT / 9080)
@{ P = "9070|9060"; A = "gfx1200" } # RDNA 4 (RX 9070 / 9060)
@{ P = "9070|9080"; A = "gfx1201" } # RDNA 4 (Navi 48: RX 9070 XT / 9070 GRE / 9070 / 9080)
@{ P = "9060"; A = "gfx1200" } # RDNA 4 (Navi 44: RX 9060 XT / 9060)
@{ P = "8065S|8060S|8050S|8040S|Strix Halo|Ryzen AI Max|AI Max"; A = "gfx1151" } # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+)
@{ P = "890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33"; A = "gfx1150" } # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375)
@{ P = "RX 7900|RX 7800|RX 7700(?!S)|PRO W7900|PRO W7800|PRO W7700"; A = "gfx1100" } # RDNA 3 desktop/workstation (Navi 31)
@{ P = "RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500|PRO V710"; A = "gfx1102" } # RDNA 3 (Navi 33)
@{ P = "890M|880M|Strix Point|HX 37[05]|AI 9 HX|AI 9 36[05]"; A = "gfx1150" } # RDNA 3.5 (Strix Point: Radeon 890M/880M, Ryzen AI 9 HX 370/375)
@{ P = "860M|840M|Krackan|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33"; A = "gfx1152" } # RDNA 3.5 (Krackan Point: Radeon 860M/840M, Ryzen AI 7 350 / AI 5 340)
@{ P = "RX 7900|PRO W7900|PRO W7800"; A = "gfx1100" } # RDNA 3 desktop/workstation (Navi 31)
@{ P = "RX 7800|RX 7700(?!S)|PRO W7700|PRO V710"; A = "gfx1101" } # RDNA 3 (Navi 32)
@{ P = "RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500"; A = "gfx1102" } # RDNA 3 (Navi 33)
@{ P = "780M|760M|740M|Phoenix|Hawk Point|Z1 Extreme|Z2 Extreme"; A = "gfx1103" } # RDNA 3 iGPU (Phoenix / Hawk Point)
@{ P = "RX 6900|RX 6800|RX 6750|RX 6700|PRO W6800|PRO W6900"; A = "gfx1030" } # RDNA 2 (Navi 21) -- gfx103X family
@{ P = "RX 6650|RX 6600|PRO W6600|PRO W6650"; A = "gfx1032" } # RDNA 2 (Navi 23) -- gfx103X family
@ -2203,6 +2205,7 @@ exit 0
$archFamilyMap = @{
"gfx1201" = "gfx120X-all"; "gfx1200" = "gfx120X-all" # RDNA 4
"gfx1151" = "gfx1151"; "gfx1150" = "gfx1150" # RDNA 3.5 (Strix Halo/Point)
"gfx1152" = "gfx1152" # RDNA 3.5 (Krackan Point)
"gfx1103" = "gfx110X-all"; "gfx1102" = "gfx110X-all" # RDNA 3
"gfx1101" = "gfx110X-all"; "gfx1100" = "gfx110X-all"
"gfx1036" = "gfx103X-all"; "gfx1035" = "gfx103X-all" # RDNA 2 (RX 6000)
@ -2224,6 +2227,7 @@ exit 0
$torchFloorMap = @{
"gfx1201" = "torch>=2.11.0,<2.12.0"; "gfx1200" = "torch>=2.11.0,<2.12.0"
"gfx1151" = "torch>=2.11.0,<2.12.0"; "gfx1150" = "torch>=2.11.0,<2.12.0"
"gfx1152" = "torch>=2.11.0,<2.12.0"
}
# Companion ranges track the torch ceiling so pip resolves a consistent
# trio on AMD's per-arch index (each published independently). Mirrors
@ -2231,10 +2235,12 @@ exit 0
$torchvisionFloorMap = @{
"gfx1201" = "torchvision>=0.26.0,<0.27.0"; "gfx1200" = "torchvision>=0.26.0,<0.27.0"
"gfx1151" = "torchvision>=0.26.0,<0.27.0"; "gfx1150" = "torchvision>=0.26.0,<0.27.0"
"gfx1152" = "torchvision>=0.26.0,<0.27.0"
}
$torchaudioFloorMap = @{
"gfx1201" = "torchaudio>=2.11.0,<2.12.0"; "gfx1200" = "torchaudio>=2.11.0,<2.12.0"
"gfx1151" = "torchaudio>=2.11.0,<2.12.0"; "gfx1150" = "torchaudio>=2.11.0,<2.12.0"
"gfx1152" = "torchaudio>=2.11.0,<2.12.0"
}
$archFamily = if ($ROCmGfxArch -and $archFamilyMap.ContainsKey($ROCmGfxArch)) { $archFamilyMap[$ROCmGfxArch] } else { $null }
if ($archFamily) {
@ -2264,7 +2270,7 @@ exit 0
$_pinRocm211 = ([int]$Matches[1] -eq 7 -and [int]$Matches[2] -eq 2)
}
# Only the 2.11-allowlist gfx arches need the floor; others publish <2.11 and stay bare.
$_pinGfx211 = @('gfx120x-all', 'gfx1151', 'gfx1150') -contains $_pinLeaf
$_pinGfx211 = @('gfx120x-all', 'gfx1151', 'gfx1150', 'gfx1152') -contains $_pinLeaf
if ($_pinGfx211 -or $_pinRocm211) {
$ROCmIndexUrl = $TorchIndexUrl
$ROCmTorchFloor = "torch>=2.11.0,<2.12.0"

72
install.sh
View file

@ -625,6 +625,36 @@ _is_pkg_installed() {
esac
}
# ── Helper: human-readable apt distro label for the sudo package prompt (#6207) ──
# Reads /etc/os-release so the Accept? prompt can say which distro we detected and
# that packages come from that distro's official apt repos (not a tarball).
_apt_distro_description() {
# Plain ( ... ) subshell — not $() — so case/;; stays bash-3.2-safe on macOS.
# Bash 3.2 misparses case arms inside command substitution and errors on `;;`.
(
if [ ! -r /etc/os-release ]; then
printf 'a debian-like system'
exit 0
fi
# shellcheck disable=SC1091
. /etc/os-release 2>/dev/null || true
if [ -n "${NAME:-}" ] && [ -n "${VERSION_ID:-}" ]; then
_ad_label="$NAME $VERSION_ID"
elif [ -n "${PRETTY_NAME:-}" ]; then
_ad_label="$PRETTY_NAME"
elif [ -n "${NAME:-}" ]; then
_ad_label="$NAME"
else
printf 'a debian-like system'
exit 0
fi
case " ${ID:-} ${ID_LIKE:-} " in
*" debian "*|*" ubuntu "*) _ad_label="${_ad_label} (debian-like)" ;;
esac
printf '%s' "$_ad_label"
)
}
# ── Helper: install packages via apt, escalating to sudo only if needed ──
# Usage: _smart_apt_install pkg1 pkg2 pkg3 ...
_smart_apt_install() {
@ -655,11 +685,14 @@ _smart_apt_install() {
# Step 3: Escalate -- need elevated permissions for remaining packages
if command -v sudo >/dev/null 2>&1; then
_ad_desc="$(_apt_distro_description)"
echo ""
echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo " WARNING: We require sudo elevated permissions to install:"
echo " $_STILL_MISSING"
echo " If you accept, we'll run sudo now, and it'll prompt your password."
echo " Detected ${_ad_desc}."
echo " If you accept, we'll run sudo apt-get to install these packages"
echo " from your distro's official repositories (not a third-party tarball)."
echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo ""
printf " Accept? [Y/n] "
@ -2227,6 +2260,7 @@ _amd_arch_index_family_for_gfx() {
gfx1201|gfx1200) echo gfx120X-all ;;
gfx1151) echo gfx1151 ;;
gfx1150) echo gfx1150 ;;
gfx1152) echo gfx1152 ;;
gfx1103|gfx1102|gfx1101|gfx1100) echo gfx110X-all ;;
gfx1036|gfx1035|gfx1034|gfx1033|gfx1032|gfx1031|gfx1030) echo gfx103X-all ;;
gfx90a) echo gfx90a ;;
@ -2238,12 +2272,14 @@ _amd_arch_index_family_for_gfx() {
# Map a GPU marketing name to gfx arch (kept in sync with install.ps1 nameArchTable).
_infer_amd_gfx_arch_from_gpu_name() {
case "$1" in
*"9070 XT"*|*9080*) echo gfx1201 ;;
*9070*|*9060*) echo gfx1200 ;;
*9070*|*9080*) echo gfx1201 ;;
*9060*) echo gfx1200 ;;
*"8065S"*|*"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) echo gfx1151 ;;
*"890M"*|*"880M"*|*"860M"*|*"840M"*|*"Strix Point"*|*"Krackan"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) echo gfx1150 ;;
*"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*|*"PRO V710"*) echo gfx1102 ;;
*"RX 7900"*|*"RX 7800"*|*"RX 7700"*|*"PRO W7900"*|*"PRO W7800"*|*"PRO W7700"*) echo gfx1100 ;;
*"890M"*|*"880M"*|*"Strix Point"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*) echo gfx1150 ;;
*"860M"*|*"840M"*|*"Krackan"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) echo gfx1152 ;;
*"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*) echo gfx1102 ;;
*"RX 7800"*|*"RX 7700"*|*"PRO W7700"*|*"PRO V710"*) echo gfx1101 ;;
*"RX 7900"*|*"PRO W7900"*|*"PRO W7800"*) echo gfx1100 ;;
*"780M"*|*"760M"*|*"740M"*|*"Phoenix"*|*"Hawk Point"*|*"Z1 Extreme"*|*"Z2 Extreme"*) echo gfx1103 ;;
*"RX 6900"*|*"RX 6800"*|*"RX 6750"*|*"RX 6700"*|*"PRO W6800"*|*"PRO W6900"*) echo gfx1030 ;;
*"RX 6650"*|*"RX 6600"*|*"PRO W6600"*|*"PRO W6650"*) echo gfx1032 ;;
@ -2283,10 +2319,14 @@ _infer_linux_amd_gfx_arch() {
echo gfx1151
return 0
fi
if [ -n "$_gpu_evidence" ] && grep -qiE '890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33' /proc/cpuinfo 2>/dev/null; then
if [ -n "$_gpu_evidence" ] && grep -qiE '890M|880M|Strix Point|HX 37[05]|AI 9 HX|AI 9 36[05]' /proc/cpuinfo 2>/dev/null; then
echo gfx1150
return 0
fi
if [ -n "$_gpu_evidence" ] && grep -qiE '860M|840M|Krackan|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33' /proc/cpuinfo 2>/dev/null; then
echo gfx1152
return 0
fi
if command -v lspci >/dev/null 2>&1; then
# A non-AMD controller can enumerate first (Intel/ASPEED before an AMD
# dGPU), so scan every display-class line and take the first AMD one
@ -3022,7 +3062,7 @@ if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \
# whole handoff (a user-set override re-exports unchanged).
export UNSLOTH_ROCM_GFX_ARCH="$_linux_inferred_gfx"
case "$_linux_inferred_gfx" in
gfx1201|gfx1200|gfx1151|gfx1150)
gfx1201|gfx1200|gfx1151|gfx1150|gfx1152)
TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0"
TORCHVISION_CONSTRAINT="torchvision>=0.26.0,<0.27.0"
TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0"
@ -3091,7 +3131,7 @@ fi
# and a bare name can resolve a 2.12 ABI-mismatched wheel. Match on the FINAL leaf so a
# custom mirror with a gfx/rocm7.2 path segment but a cu*/cpu family isn't forced.
case "$_torch_index_leaf" in
rocm7.2|gfx120x-all|gfx1151|gfx1150)
rocm7.2|gfx120x-all|gfx1151|gfx1150|gfx1152)
TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0"
TORCHVISION_CONSTRAINT="torchvision>=0.26.0,<0.27.0"
TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0"
@ -3210,7 +3250,7 @@ case "$_torch_index_leaf" in
fi
_strix_gfx=""
case "$_runtime_gfx" in
gfx1151|gfx1150) _strix_gfx="$_runtime_gfx" ;;
gfx1151|gfx1150|gfx1152) _strix_gfx="$_runtime_gfx" ;;
esac
# Skip rocm7.13+ generic indexes: they already ship the fixes, so the
# arch build (rocm7.13) would be a downgrade rather than a rescue.
@ -3306,12 +3346,14 @@ elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then
# gfx1102 matched BEFORE gfx1100 so the spaceless "RX 7700S" lands on
# gfx1102 (bash case has no negative lookahead like the PS tables).
case "$_gpu_disp_mkt" in
*"9070 XT"*|*9080*) _gpu_disp_gfx="gfx1201" ;; # RDNA 4
*9070*|*9060*) _gpu_disp_gfx="gfx1200" ;; # RDNA 4
*9070*|*9080*) _gpu_disp_gfx="gfx1201" ;; # RDNA 4 (Navi 48)
*9060*) _gpu_disp_gfx="gfx1200" ;; # RDNA 4 (Navi 44)
*"8065S"*|*"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) _gpu_disp_gfx="gfx1151" ;; # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+)
*"890M"*|*"880M"*|*"860M"*|*"840M"*|*"Strix Point"*|*"Krackan"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) _gpu_disp_gfx="gfx1150" ;; # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375)
*"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*|*"PRO V710"*) _gpu_disp_gfx="gfx1102" ;; # RDNA 3 (Navi 33)
*"RX 7900"*|*"RX 7800"*|*"RX 7700"*|*"PRO W7900"*|*"PRO W7800"*|*"PRO W7700"*) _gpu_disp_gfx="gfx1100" ;; # RDNA 3 desktop / workstation (Navi 31)
*"890M"*|*"880M"*|*"Strix Point"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*) _gpu_disp_gfx="gfx1150" ;; # RDNA 3.5 (Strix Point: Radeon 890M/880M, Ryzen AI 9 HX 370/375)
*"860M"*|*"840M"*|*"Krackan"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) _gpu_disp_gfx="gfx1152" ;; # RDNA 3.5 (Krackan Point: Radeon 860M/840M, Ryzen AI 7 350 / AI 5 340)
*"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*) _gpu_disp_gfx="gfx1102" ;; # RDNA 3 (Navi 33)
*"RX 7800"*|*"RX 7700"*|*"PRO W7700"*|*"PRO V710"*) _gpu_disp_gfx="gfx1101" ;; # RDNA 3 (Navi 32)
*"RX 7900"*|*"PRO W7900"*|*"PRO W7800"*) _gpu_disp_gfx="gfx1100" ;; # RDNA 3 desktop / workstation (Navi 31)
*"780M"*|*"760M"*|*"740M"*|*"Phoenix"*|*"Hawk Point"*|*"Z1 Extreme"*|*"Z2 Extreme"*) _gpu_disp_gfx="gfx1103" ;; # RDNA 3 iGPU (Phoenix / Hawk Point)
*"RX 6900"*|*"RX 6800"*|*"RX 6750"*|*"RX 6700"*|*"PRO W6800"*|*"PRO W6900"*) _gpu_disp_gfx="gfx1030" ;; # RDNA 2 (Navi 21)
*"RX 6650"*|*"RX 6600"*|*"PRO W6600"*|*"PRO W6650"*) _gpu_disp_gfx="gfx1032" ;; # RDNA 2 (Navi 23)

22
pyproject.toml
View file

@ -93,6 +93,17 @@ huggingfacenotorch = [
"trl>=0.18.2,!=0.19.0,<=0.24.0",
"sentence-transformers",
]
# torchcodec backend for Gemma audio / datasets>=4 (#7225).
# Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC).
audio-torch210 = [
"torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10'",
]
audio-torch290 = [
"torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10'",
]
audio-torch280 = [
"torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9'",
]
huggingface = [
"unsloth[huggingfacenotorch]",
"unsloth_zoo>=2026.7.6",
@ -532,16 +543,19 @@ cu126-torch2100 = [
"unsloth[huggingface]",
"bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",
"unsloth[cu126onlytorch2100]",
"unsloth[audio-torch210]",
]
cu128-torch2100 = [
"unsloth[huggingface]",
"bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",
"unsloth[cu128onlytorch2100]",
"unsloth[audio-torch210]",
]
cu130-torch2100 = [
"unsloth[huggingface]",
"bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",
"unsloth[cu130onlytorch2100]",
"unsloth[audio-torch210]",
]
kaggle = [
"unsloth[huggingface]",
@ -831,16 +845,19 @@ cu126-ampere-torch2100 = [
"unsloth[huggingface]",
"bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",
"unsloth[cu126onlytorch2100]",
"unsloth[audio-torch210]",
]
cu128-ampere-torch2100 = [
"unsloth[huggingface]",
"bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",
"unsloth[cu128onlytorch2100]",
"unsloth[audio-torch210]",
]
cu130-ampere-torch2100 = [
"unsloth[huggingface]",
"bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",
"unsloth[cu130onlytorch2100]",
"unsloth[audio-torch210]",
]
flashattentiontorch260abiFALSEcu12x = [
"flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'",
@ -1125,7 +1142,8 @@ intelgputorch210 = [
"torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=1c4b44b36a557f7381e3076fb8843366742238648441d607c8d049c6da0f8886 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')",
]
intel-gpu-torch210 = [
"unsloth[intelgputorch210]"
"unsloth[intelgputorch210]",
"unsloth[audio-torch210]",
]
intelgputorch2110 = [
"unsloth_zoo[intelgpu]",
@ -1279,6 +1297,7 @@ rocm72-torch2100 = [
"torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'",
"torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'",
"torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'",
"unsloth[audio-torch210]",
]
rocm711-torch2100 = [
"unsloth[amd]",
@ -1297,6 +1316,7 @@ rocm711-torch2100 = [
"torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'",
"torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'",
"torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'",
"unsloth[audio-torch210]",
]
[project.urls]

4
scripts/notebook_validator.py
View file

@ -95,8 +95,8 @@ COLAB_ORACLE_BASE_URL = "https://raw.githubusercontent.com/googlecolab/backend-i
# Source: pytorch/torchcodec compatibility matrix on its README.
TORCH_TORCHCODEC: dict[str, set[str]] = {
"2.10": {"0.10"},
"2.9": {"0.7", "0.8", "0.9"},
"2.8": {"0.6"},
"2.9": {"0.8", "0.9"},
"2.8": {"0.6", "0.7"},
"2.7": {"0.3", "0.4", "0.5"},
"2.6": {"0.2", "0.3"},
"2.5": {"0.1", "0.2"},

22
scripts/scan_packages_baseline.json
View file

@ -1,5 +1,5 @@
{
"_comment": "scan_packages.py allowlist. Each entry is a CRITICAL/HIGH finding manually judged benign. Matched on (package, package-relative file, check, evidence_hash); evidence_hash is over the matched code with L<NN>: markers stripped, so version bumps and line shifts do not reopen an entry but changed code does. severity and evidence are for review only. Regenerate with --write-baseline AFTER reviewing every line.",
"_comment": "scan_packages.py allowlist (reviewed). Each entry is a CRITICAL/HIGH finding manually judged benign. Matched on (package, package-relative file, check, evidence_hash); evidence_hash is over the matched code with L<NN>: markers stripped, so version bumps and line shifts do not reopen an entry but changed code does. severity and evidence are for review only. Regenerate with --write-baseline AFTER reviewing every line.",
"version": 1,
"entries": [
{
@ -303,8 +303,8 @@
"file": "openai/_base_client.py",
"check": "C2 polling/beaconing loop detected",
"severity": "CRITICAL",
"evidence": "L264: while True: sha256:95ca67e46d42354ae650abbdc5b0d97df8b0ed43187800bf40f5690c3901b94b",
"evidence_hash": "a57d8d15fed0bf04f9967dcc18a18b80bb19f4095675bccbb78ac0450d7fce14"
"evidence": "L274: while True: sha256:90a38e5c1e26893c7c273354143612640e9a9c0f079d3e2b60612d79f24e80a6",
"evidence_hash": "1022e8e8649436ec64a98a9d9141d085452c49549fd2157b0278fc369a83ac66"
},
{
"package": "openai",
@ -319,8 +319,8 @@
"file": "openai/auth/_workload.py",
"check": "Accesses cloud metadata/IMDS AND makes network calls",
"severity": "CRITICAL",
"evidence": "IMDS: L96: url = \"http://169.254.169.254/metadata/identity/oauth2/token\" | L149: url = \"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity\"\nNetwork: L77: http_client: httpx.Client | None = None, | L108: with httpx.Client() as client: | L133: http_client: httpx.Client | None = None, | L155: with httpx.Client() as client: | L248: with httpx.Client() as client:",
"evidence_hash": "1581d9f4a23393e9af23fbe5ef9f66807b22c5b5a3f1fe167254c9ebee108567"
"evidence": "IMDS: L97: url = \"http://169.254.169.254/metadata/identity/oauth2/token\" | L150: url = \"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity\"\nNetwork: L78: http_client: httpx.Client | None = None, | L109: with httpx.Client() as client: | L134: http_client: httpx.Client | None = None, | L156: with httpx.Client() as client: | L251: exchange_client = DefaultHttpx2Client(follow_redirects=False) if self._use_httpx2 else httpx.Client()",
"evidence_hash": "9717e51cb961dc14c458955d91a1e48e3753997346ecea0106bded3a8d64bfe0"
},
{
"package": "openai",
@ -343,8 +343,8 @@
"file": "openai/resources/beta/responses/responses.py",
"check": "C2 polling/beaconing loop detected",
"severity": "CRITICAL",
"evidence": "L3999: while True: sha256:df298b6eaf3416589b79f4ef283f8fb76e54d505bfda8840673f8e6419117e2e",
"evidence_hash": "10ce5cb5a7097fcff4042ddcfb4802edda60aa4b7b113c8b926a52ddb76f78c2"
"evidence": "L4000: while True: sha256:f8ab538118daba9ec06e27399dbdc90a4521c3390e6a47a6348a1f180a83effd",
"evidence_hash": "31481ea83c687acc27144d72d3832d4fb98dd1c79fb5e0ddd85080de95997b9f"
},
{
"package": "openai",
@ -359,16 +359,16 @@
"file": "openai/resources/realtime/realtime.py",
"check": "C2 polling/beaconing loop detected",
"severity": "CRITICAL",
"evidence": "L310: while True: sha256:458198ff3d3f05870bf98c9564cbfd68c739e57b9bbe4120ed81e3eb6af74a05",
"evidence_hash": "a3165d21e46b3ce553795daeae53e8f80e8e89c5cb228e68e6dcaff54bca5a89"
"evidence": "L311: while True: sha256:5b63313072aae9ca28677e03426513ccf12221e4f4e0ea6c31efbe09790633b5",
"evidence_hash": "05e1af469d651b51673763a7c4cdf759af9472fb627b7b470adc28cc237bd650"
},
{
"package": "openai",
"file": "openai/resources/responses/responses.py",
"check": "C2 polling/beaconing loop detected",
"severity": "CRITICAL",
"evidence": "L3950: while True: sha256:1ce0b5a388c747945cdfda1a71b77afdfd03ae840d7aa9fa62f02eb00aa5e29f",
"evidence_hash": "6de300ebb5e6e17cb51c89cbcdf08515a44655182f0776f0908a9d1043ebbcd7"
"evidence": "L3951: while True: sha256:d68ef896bf0743ca430cfacb9a3353da1f3b9c51c3a21b6450a07a32b55aa2ac",
"evidence_hash": "160eecdd79b521bffbe8476f782b69a0724c35d1b19376a7600807165fd54f9f"
},
{
"package": "openai",

273
studio/Unsloth_Studio_Colab.ipynb
View file

@ -1,134 +1,145 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/github/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/github/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "6b87de59"
},
"source": [
"To run this, press \"*Runtime*\" and press \"*Run all*\" on a **free** Tesla T4 Google Colab instance!\n",
"<div class=\"align-center\">\n",
"<a href=\"https://unsloth.ai/\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/unsloth%20new%20logo.png\" width=\"115\"></a>\n",
"<a href=\"https://discord.gg/unsloth\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/Discord button.png\" width=\"145\"></a>\n",
"<a href=\"https://unsloth.ai/docs/\"><img src=\"https://github.com/unslothai/unsloth/blob/main/images/documentation%20green%20button.png?raw=true\" width=\"125\"></a> Join Discord if you need help + ⭐ <i>Star us on <a href=\"https://github.com/unslothai/unsloth\">Github</a> </i> ⭐\n",
"</div>\n",
"\n",
"To install Unsloth Studio on your local device, follow [our guide](https://unsloth.ai/docs/new/unsloth-studio/install). Unsloth Studio is licensed [AGPL-3.0](https://github.com/unslothai/unsloth/blob/main/studio/LICENSE.AGPL-3.0).\n",
"\n",
"### Unsloth Studio\n",
"\n",
"Train and run open models with [**Unsloth Studio**](https://unsloth.ai/docs/new/unsloth-studio/start). NEW! Installation should now only take 2 mins!\n",
"\n",
"\n",
"We are actively working on making Unsloth Studio install on Colab T4 GPUs faster.\n",
"\n",
"[Features](https://unsloth.ai/docs/new/unsloth-studio#features) • [Quickstart](https://unsloth.ai/docs/new/unsloth-studio/start) • [Data Recipes](https://unsloth.ai/docs/new/unsloth-studio/data-recipe) • [Unsloth Chat](https://unsloth.ai/docs/new/unsloth-studio/chat) • [Export](https://unsloth.ai/docs/new/unsloth-studio/export)"
],
"id": "6b87de59"
},
{
"cell_type": "markdown",
"metadata": {
"id": "e4206349"
},
"source": [
"<p align=\"left\"><img src=\"https://github.com/unslothai/unsloth/raw/main/studio/frontend/public/studio%20github%20landscape%20colab%20display.png\" width=\"600\"></p>"
],
"id": "e4206349"
},
{
"cell_type": "markdown",
"metadata": {
"id": "27da2957"
},
"source": [
"### Setup: Clone repo and run setup"
],
"id": "27da2957"
},
{
"cell_type": "code",
"metadata": {
"id": "27e68f91"
},
"source": "!git clone --depth 1 --branch main https://github.com/unslothai/unsloth.git\n%cd /content/unsloth\n!chmod +x studio/setup.sh && ./studio/setup.sh --local",
"execution_count": null,
"outputs": [],
"id": "27e68f91"
},
{
"cell_type": "markdown",
"metadata": {
"id": "3e1771a9"
},
"source": [
"### Start Unsloth Studio"
],
"id": "3e1771a9"
},
{
"cell_type": "code",
"metadata": {
"id": "277e431e"
},
"source": [
"import sys\n",
"sys.path.insert(0, \"/content/unsloth/studio/backend\")\n",
"from colab import start\n",
"\n",
"# On Colab, start() auto-opens a Cloudflare link and prints admin login credentials.\n",
"# Use the Cloudflare link above the ready card to open Studio (in-cell iframes often stay blank).\n",
"start()\n",
"\n",
"# To skip the Cloudflare tunnel and try the in-notebook proxy iframe only:\n",
"# start(cloudflare=False)"
],
"execution_count": null,
"outputs": [],
"id": "277e431e"
},
{
"cell_type": "markdown",
"metadata": {
"id": "f2b0c6a1"
},
"source": [
"And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n",
"\n",
"Some other resources:\n",
"1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs.\n",
"2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide).\n",
"3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support.\n",
"4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model.\n",
"5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.\n",
"\n",
"<div class=\"align-center\">\n",
" <a href=\"https://unsloth.ai\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/unsloth%20new%20logo.png\" width=\"115\"></a>\n",
" <a href=\"https://discord.gg/unsloth\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/Discord.png\" width=\"145\"></a>\n",
" <a href=\"https://unsloth.ai/docs/\"><img src=\"https://github.com/unslothai/unsloth/blob/main/images/documentation%20green%20button.png?raw=true\" width=\"125\"></a>\n",
"\n",
" Join Discord if you need help + ⭐️ <i>Star us on <a href=\"https://github.com/unslothai/unsloth\">Github</a> </i> ⭐️\n",
"\n",
" <b>This notebook is licensed <a href=\"https://github.com/unslothai/unsloth/blob/main/studio/LICENSE.AGPL-3.0\">AGPL-3.0</a></b>\n",
"</div>"
],
"id": "f2b0c6a1"
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "T4",
"provenance": [],
"include_colab_link": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
{
"cell_type": "markdown",
"id": "6b87de59",
"metadata": {
"id": "6b87de59"
},
"source": [
"To run this, press \"*Runtime*\" and press \"*Run all*\" on a **free** Tesla T4 Google Colab instance!\n",
"<div class=\"align-center\">\n",
"<a href=\"https://unsloth.ai/\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/unsloth%20new%20logo.png\" width=\"115\"></a>\n",
"<a href=\"https://discord.gg/unsloth\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/Discord button.png\" width=\"145\"></a>\n",
"<a href=\"https://unsloth.ai/docs/\"><img src=\"https://github.com/unslothai/unsloth/blob/main/images/documentation%20green%20button.png?raw=true\" width=\"125\"></a> Join Discord if you need help + ⭐ <i>Star us on <a href=\"https://github.com/unslothai/unsloth\">Github</a> </i> ⭐\n",
"</div>\n",
"\n",
"To install Unsloth Studio on your local device, follow [our guide](https://unsloth.ai/docs/new/unsloth-studio/install). Unsloth Studio is licensed [AGPL-3.0](https://github.com/unslothai/unsloth/blob/main/studio/LICENSE.AGPL-3.0).\n",
"\n",
"### Unsloth Studio\n",
"\n",
"Train and run open models with [**Unsloth Studio**](https://unsloth.ai/docs/new/unsloth-studio/start). NEW! Installation should now only take 2 mins!\n",
"\n",
"\n",
"We are actively working on making Unsloth Studio install on Colab T4 GPUs faster.\n",
"\n",
"[Features](https://unsloth.ai/docs/new/unsloth-studio#features) • [Quickstart](https://unsloth.ai/docs/new/unsloth-studio/start) • [Data Recipes](https://unsloth.ai/docs/new/unsloth-studio/data-recipe) • [Unsloth Chat](https://unsloth.ai/docs/new/unsloth-studio/chat) • [Export](https://unsloth.ai/docs/new/unsloth-studio/export)"
]
},
{
"cell_type": "markdown",
"id": "e4206349",
"metadata": {
"id": "e4206349"
},
"source": [
"<p align=\"left\"><img src=\"https://github.com/unslothai/unsloth/raw/main/studio/frontend/public/studio%20github%20landscape%20colab%20display.png\" width=\"600\"></p>"
]
},
{
"cell_type": "markdown",
"id": "27da2957",
"metadata": {
"id": "27da2957"
},
"source": [
"### Setup: Clone repo and run setup"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "27e68f91",
"metadata": {
"id": "27e68f91"
},
"outputs": [],
"source": "!git clone --depth 1 --branch main https://github.com/unslothai/unsloth.git\n%cd /content/unsloth\n!chmod +x studio/setup.sh && ./studio/setup.sh --local"
},
{
"cell_type": "markdown",
"id": "3e1771a9",
"metadata": {
"id": "3e1771a9"
},
"source": [
"### Start Unsloth Studio"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "277e431e",
"metadata": {
"id": "277e431e"
},
"outputs": [],
"source": "import sys\nsys.path.insert(0, \"/content/unsloth/studio/backend\")\nfrom colab import start\n\n# Default: in-tab iframe only. start() blocks to keep the kernel alive.\nstart()\n\n# For a shareable Cloudflare link, replace start() above with:\n# start(cloudflare=True)"
},
{
"cell_type": "markdown",
"id": "f2b0c6a1",
"metadata": {
"id": "f2b0c6a1"
},
"source": [
"And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n",
"\n",
"Some other resources:\n",
"1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs.\n",
"2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide).\n",
"3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support.\n",
"4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model.\n",
"5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.\n",
"\n",
"<div class=\"align-center\">\n",
" <a href=\"https://unsloth.ai\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/unsloth%20new%20logo.png\" width=\"115\"></a>\n",
" <a href=\"https://discord.gg/unsloth\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/Discord.png\" width=\"145\"></a>\n",
" <a href=\"https://unsloth.ai/docs/\"><img src=\"https://github.com/unslothai/unsloth/blob/main/images/documentation%20green%20button.png?raw=true\" width=\"125\"></a>\n",
"\n",
" Join Discord if you need help + ⭐️ <i>Star us on <a href=\"https://github.com/unslothai/unsloth\">Github</a> </i> ⭐️\n",
"\n",
" <b>This notebook is licensed <a href=\"https://github.com/unslothai/unsloth/blob/main/studio/LICENSE.AGPL-3.0\">AGPL-3.0</a></b>\n",
"</div>"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "T4",
"provenance": [],
"include_colab_link": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
"nbformat": 4,
"nbformat_minor": 5
}

1
studio/backend/assets/configs/full_finetune.yaml
View file

@ -30,6 +30,7 @@ lora:
vision_all_linear: false
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/lora_text.yaml
View file

@ -30,6 +30,7 @@ lora:
vision_all_linear: false
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/default.yaml
View file

@ -33,6 +33,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml
View file

@ -30,6 +30,7 @@ lora:
- "query"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml
View file

@ -30,6 +30,7 @@ lora:
- "value"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml
View file

@ -33,6 +33,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml
View file

@ -29,6 +29,7 @@ lora:
- "Wqkv"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml
View file

@ -33,6 +33,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B-it.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B-it.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B-it.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B-it.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B.yaml
View file

@ -26,6 +26,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml
View file

@ -37,6 +37,7 @@ lora:
- "shared_mlp.output_linear"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml
View file

@ -37,6 +37,7 @@ lora:
- "shared_mlp.output_linear"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml
View file

@ -30,6 +30,7 @@ lora:
- "v_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: false

1
studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml
View file

@ -33,6 +33,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml
View file

@ -33,6 +33,7 @@ lora:
- "v_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml
View file

@ -38,6 +38,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml
View file

@ -37,6 +37,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml
View file

@ -35,6 +35,7 @@ lora:
- "out_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml
View file

@ -37,6 +37,7 @@ lora:
- "out_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml
View file

@ -33,6 +33,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml
View file

@ -38,6 +38,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml
View file

@ -33,6 +33,7 @@ lora:
- "v_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml
View file

@ -36,6 +36,7 @@ lora:
- "gate_up_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml
View file

@ -34,6 +34,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml
View file

@ -36,6 +36,7 @@ lora:
- "gate_up_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml
View file

@ -35,6 +35,7 @@ lora:
- "down_proj"
use_rslora: false
use_loftq: false
use_dora: false
logging:
enable_wandb: false

1
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml
View file

@ -29,6 +29,7 @@ lora:
- "all-linear"
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

1
studio/backend/assets/configs/vision_lora.yaml
View file

@ -30,6 +30,7 @@ lora:
vision_all_linear: true
use_rslora: false
use_loftq: false
use_dora: false
finetune_vision_layers: true
finetune_language_layers: true
finetune_attention_modules: true

526
studio/backend/colab.py
View file

@ -1,9 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""
Colab helpers for Unsloth Studio. Uses Colab's built-in proxy.
"""
"""Colab helpers for Unsloth Studio. Uses Colab's built-in proxy."""
from pathlib import Path
import sys
@ -22,11 +20,9 @@ logger = get_logger(__name__)
def get_colab_url(port: int = 8888) -> str:
"""
Get the Colab proxy URL for a port.
"""Get the Colab proxy URL for a port.
Retries up to 3 times, validating the result is a real HTTPS Colab URL.
Falls back to http://localhost:{port} only when all attempts fail.
Retries 3x validating a real HTTPS Colab URL; falls back to localhost on failure.
"""
import time as _time
@ -55,28 +51,243 @@ def get_colab_url(port: int = 8888) -> str:
return fallback
def show_link(port: int = 8888, *, _url: "str | None" = None):
"""Display a styled clickable link to the UI.
*_url* is an optional pre-fetched proxy URL; pass it to avoid a second eval_js round-trip.
"""
from IPython.display import display, HTML
url = _url if _url is not None else get_colab_url(port)
# Truncated display URL; try/except so an odd URL shape still renders the link.
def _short_colab_url(url: str, port: int) -> str:
"""Truncated display form of a Colab proxy URL; falls back to the full URL."""
try:
port_prefix = f"{port}-"
idx = url.index(port_prefix)
next_dash = url.index("-", idx + len(port_prefix))
short_url = url[: next_dash + 1] + "..."
return url[: next_dash + 1] + "..."
except (ValueError, IndexError):
short_url = url
return url
# Plain-text line so the URL shows even if HTML display fails.
logger.info(f"🌐 Unsloth Studio URL: {url}")
html = f"""
def _is_colab_proxy_url(url: str, port: int) -> bool:
"""True when *url* looks like a real Colab kernel proxy, not a localhost fallback."""
return bool(url and isinstance(url, str) and url.startswith("https://") and str(port) in url)
def _is_colab_runtime() -> bool:
"""True on a hosted Colab notebook kernel.
Reuses the backend's main Colab detector (``/content`` + Colab env / ``google.colab``)
instead of a single env var, which is not always present on hosted runtimes.
"""
try:
from main import _IS_COLAB
return bool(_IS_COLAB)
except Exception:
return False
def _colab_login_credentials_path() -> Path:
from auth.storage import DB_PATH
return DB_PATH.parent / ".colab_notebook_login"
def _store_colab_login_credentials(username: str, password: str) -> None:
"""Persist Colab admin credentials for notebook re-runs after interrupt."""
path = _colab_login_credentials_path()
try:
path.parent.mkdir(parents = True, exist_ok = True)
path.write_text(f"{username}\n{password}\n")
try:
import os
os.chmod(path, 0o600)
except OSError:
pass
except OSError as e:
logger.info(f"Could not persist Colab login credentials ({e}).")
def _load_colab_login_credentials() -> "tuple[str, str] | None":
"""Return stored Colab admin credentials from a previous ``start()`` run, if any."""
path = _colab_login_credentials_path()
try:
if not path.is_file():
return None
lines = path.read_text().splitlines()
if len(lines) >= 2 and lines[0] and lines[1]:
return lines[0], lines[1]
except OSError as e:
logger.info(f"Could not load Colab login credentials ({e}).")
return None
def _clear_colab_login_credentials() -> None:
"""Drop the cached Colab credentials once they no longer authenticate."""
path = _colab_login_credentials_path()
try:
path.unlink(missing_ok = True)
except OSError as e:
logger.info(f"Could not clear Colab login credentials ({e}).")
def _colab_credentials_still_valid(username: str, password: str) -> bool:
"""True when *password* still matches the stored admin hash.
Guards against redisplaying a cached first-run password after the user has
changed the admin password through the app, which would print credentials
that no longer authenticate to the current Cloudflare tunnel.
"""
try:
from auth.storage import get_user_and_secret
from auth.hashing import verify_password
except Exception as e:
logger.info(f"Could not load auth to validate cached Colab credentials ({e}).")
return False
try:
row = get_user_and_secret(username)
if not row:
return False
salt, pwd_hash = row[0], row[1]
return bool(verify_password(password, salt, pwd_hash))
except Exception as e:
logger.info(f"Could not validate cached Colab credentials ({e}).")
return False
def _colab_wants_cloudflare(cloudflare: "bool | None") -> bool:
"""Resolve whether to open a Cloudflare tunnel.
``None`` auto-enables on real Colab (the in-cell proxy embed is often blank);
pass ``False`` to opt out.
"""
if cloudflare is not None:
return cloudflare
return _is_colab_runtime()
def _finalize_colab_admin_password() -> "tuple[str, str] | None":
"""Clear the bootstrap-password gate on Colab so Cloudflare tunnels can start.
Returns ``(username, password)`` for display in the notebook. On first run the
random admin password is finalized; on later runs (e.g. after interrupt) the
stored credentials are re-displayed so the Cloudflare link stays usable.
Anyone who can read this cell already controls the runtime.
"""
if not _is_colab_runtime():
return None
try:
from auth.storage import (
DEFAULT_ADMIN_USERNAME,
ensure_default_admin,
generate_bootstrap_password,
get_bootstrap_password,
requires_password_change,
update_password,
)
except Exception as e:
logger.warning(
f"Could not load auth for Colab setup ({e}); Cloudflare link may be blocked."
)
return None
try:
ensure_default_admin()
username = DEFAULT_ADMIN_USERNAME
if not requires_password_change(username):
creds = _load_colab_login_credentials()
if creds is not None and _colab_credentials_still_valid(username, creds[1]):
return creds
# The admin password was changed through the app after the first run,
# so the cached copy is stale; drop it instead of printing dead credentials.
_clear_colab_login_credentials()
return None
password = get_bootstrap_password() or generate_bootstrap_password()
if not update_password(username, password):
logger.warning(
"Could not finalize Colab admin password; Cloudflare link may be blocked."
)
return None
_store_colab_login_credentials(username, password)
return username, password
except Exception as e:
logger.warning(
f"Could not finalize Colab admin password ({e}); Cloudflare link may be blocked."
)
return None
def _colab_login_html(username: str, password: str) -> str:
"""Notebook card with Colab admin credentials (shown once after auto-finalize)."""
return f"""
<div style="display: inline-block; padding: 20px; background: #ffffff; border: 2px solid #000000;
border-radius: 12px; margin: 10px 0; font-family: system-ui, -apple-system, sans-serif;">
<h2 style="color: #000000; margin: 0 0 12px 0; font-size: 22px; font-weight: 800;">
Unsloth Studio Login (Colab)
</h2>
<p style="color: #333333; margin: 0 0 12px 0; font-size: 14px; font-weight: bold;">
Log in as <code>{username}</code> with this password. This cell is visible only in
your notebook session.
</p>
<p style="color: #333333; margin: 0; font-size: 14px; font-family: monospace; font-weight: bold;">
Password: <code>{password}</code>
</p>
</div>
"""
def _show_colab_login_credentials(username: str, password: str) -> None:
"""Display Colab admin credentials in the notebook output."""
from IPython.display import HTML, display
logger.info(f"🔐 Unsloth Studio login — user: {username}")
display(HTML(_colab_login_html(username, password)))
def _ready_card_html(
url: str,
port: int,
*,
has_cloudflare_link: bool = False,
cloudflare_requested: bool = False,
) -> str:
"""Branded ready card for the in-notebook Studio view.
Colab ``*.prod.colab.dev`` proxy URLs are session-scoped and 404 when opened as a
top-level tab or on another device, so never ``window.open`` them. On real Colab the
Cloudflare link is the supported entry point because in-cell proxy embeds often stay blank.
"""
short_url = _short_colab_url(url, port)
if _is_colab_runtime() or _is_colab_proxy_url(url, port):
if has_cloudflare_link:
embed_note = (
"Open Studio with the Cloudflare link above. In-cell proxy previews on "
"current Colab often stay blank, so the tunnel link is the supported path."
)
elif cloudflare_requested:
embed_note = (
"Could not open a Cloudflare tunnel, so Studio may be unreachable on Colab. "
"Check the logs above and re-run this cell. Pass "
'<code style="background:#f3f3f3;padding:2px 6px;border-radius:4px;">'
"cloudflare=True</code> after fixing any tunnel errors."
)
else:
embed_note = (
"Colab proxy links cannot be opened in a new tab (they 404 outside this "
'notebook). Re-run with <code style="background:#f3f3f3;padding:2px 6px;'
'border-radius:4px;">start(cloudflare=True)</code> for a working link.'
)
return f"""
<div style="display: inline-block; padding: 20px; background: #ffffff; border: 2px solid #000000;
border-radius: 12px; margin: 10px 0; font-family: system-ui, -apple-system, sans-serif;">
<h2 style="color: #000000; margin: 0 0 12px 0; font-size: 26px; font-weight: 800;
display: flex; align-items: center; gap: 12px;">
<img src="https://github.com/unslothai/unsloth/raw/main/studio/frontend/public/unsloth-gem.png"
height="48" style="display:block;">
Unsloth Studio is Ready!
</h2>
<p style="color: #333333; margin: 0 0 8px 0; font-size: 15px; font-weight: bold;">
{embed_note}
</p>
<p style="color: #666666; margin: 16px 0 0 0; font-size: 13px; font-family: monospace; font-weight: bold;">
{short_url}
</p>
</div>
"""
return f"""
<div style="display: inline-block; padding: 20px; background: #ffffff; border: 2px solid #000000;
border-radius: 12px; margin: 10px 0; font-family: system-ui, -apple-system, sans-serif;">
<h2 style="color: #000000; margin: 0 0 12px 0; font-size: 26px; font-weight: 800;
@ -100,15 +311,52 @@ def show_link(port: int = 8888, *, _url: "str | None" = None):
</p>
</div>
"""
display(HTML(html))
def show_link(
port: int = 8888,
*,
_url: "str | None" = None,
has_cloudflare_link: bool = False,
cloudflare_requested: bool = False,
):
"""Display a styled ready card for the UI.
Colab proxy URLs are informational only (no new-tab open; they 404 outside the cell);
non-proxy URLs keep a clickable open button. *_url* is an optional pre-fetched proxy
URL to avoid a second eval_js round-trip.
"""
from IPython.display import display, HTML
url = _url if _url is not None else get_colab_url(port)
logger.info(f"🌐 Unsloth Studio URL: {url}")
display(
HTML(
_ready_card_html(
url,
port,
has_cloudflare_link = has_cloudflare_link,
cloudflare_requested = cloudflare_requested,
)
)
)
def _warn_colab_cloudflare_missing(*, use_cloudflare: bool, cloudflare_url: "str | None") -> None:
"""Log a prominent warning when Colab expected a tunnel but none was opened."""
if not use_cloudflare or cloudflare_url or not _is_colab_runtime():
return
logger.warning(
"Colab Cloudflare tunnel unavailable — Studio is unlikely to be reachable in this "
"notebook. Check the logs above for tunnel or auth errors, then re-run start()."
)
def _bootstrap_password_pending() -> bool:
"""True while the default admin still owes a bootstrap-password change.
While pending, main.py injects that password into same-origin GETs, and a public
tunnel GET (no Origin) reads as same-origin, so sharing the link would leak admin
access. Fails safe to pending if the state cannot be read.
While pending, a public tunnel GET (no Origin) reads as same-origin and gets the
injected password, so sharing the link would leak admin access. Fails safe to pending.
"""
try:
from auth.storage import requires_password_change, DEFAULT_ADMIN_USERNAME
@ -121,9 +369,8 @@ def _bootstrap_password_pending() -> bool:
def start_cloudflare_tunnel(port: int) -> "str | None":
"""Open a shareable Cloudflare quick tunnel to localhost:*port*, or None.
run_server suppresses the tunnel on Colab by design, so we start it directly.
Refused while the bootstrap password is pending; any failure collapses to None
and the Colab proxy still works.
run_server suppresses the tunnel on Colab, so we start it directly. Refused while the
bootstrap password is pending; any failure collapses to None (Colab proxy still works).
"""
if _bootstrap_password_pending():
logger.warning(
@ -152,9 +399,9 @@ def start_cloudflare_tunnel(port: int) -> "str | None":
def _publish_cloudflare_url(cloudflare_url: "str | None") -> None:
"""Publish a directly-started tunnel URL onto app.state so /api/health advertises it.
run_server only sets this when it opens the tunnel itself, which it skips on Colab,
so we set it here. Otherwise the frontend's API examples fall back to an
unreachable server_url. Best-effort.
run_server sets this only when it opens the tunnel itself (skipped on Colab), so we
set it here; otherwise the frontend's API examples fall back to an unreachable
server_url. Best-effort.
"""
if not cloudflare_url:
return
@ -183,8 +430,7 @@ def _stop_cloudflare_tunnel() -> None:
def _is_studio_healthy(port: int, timeout: float = 2.0) -> bool:
"""True only if Unsloth Studio (not some other app) answers /api/health on *port*.
The service-marker check stops the reuse path reusing or tunneling a foreign
process that merely serves /api/health.
The service-marker check stops the reuse path reusing or tunneling a foreign process.
"""
import json, urllib.request
try:
@ -194,8 +440,29 @@ def _is_studio_healthy(port: int, timeout: float = 2.0) -> bool:
return False
def _shareable_link_html(cloudflare_url: str) -> str:
"""Branded card for the shareable Cloudflare link, styled like the show_link banner."""
def _shareable_link_html(
cloudflare_url: str,
password: "str | None" = None,
username: "str | None" = None,
) -> str:
"""Branded card for the shareable Cloudflare link, styled like the show_link banner.
*password* renders under the link so the credential sits in the card with the button
it unlocks. The username is always the default admin, so it reads inline.
"""
login_block = ""
if password:
login_block = f"""
<p style="color: #000000; margin: 16px 0 0 0; font-size: 20px; font-weight: 800;">
Password
</p>
<p style="margin: 6px 0 0 0;"><code style="display: inline-block; font-size: 24px;
font-weight: 800; text-decoration: underline; background: #f3f3f3;
padding: 4px 10px; border-radius: 6px;">{password}</code></p>
<p style="color: #666666; margin: 6px 0 0 0; font-size: 12px;">
Log in as <code>{username}</code> with this password. Shown only in your
notebook session, and never included in the shared link.
</p>"""
return f"""
<div style="display: inline-block; padding: 20px; background: #ffffff; border: 2px solid #000000;
border-radius: 12px; margin: 10px 0; font-family: system-ui, -apple-system, sans-serif;">
@ -213,40 +480,55 @@ def _shareable_link_html(cloudflare_url: str) -> str:
Open Unsloth Studio
</a>
<p style="color: #333333; margin: 12px 0 0 0; font-size: 14px; font-weight: bold;">
This Cloudflare HTTPS link works from any device — share it with anyone. The Colab view below only works in this tab.
This Cloudflare HTTPS link works from any device, so you can share it with anyone.
</p>
<p style="color: #333333; margin: 16px 0 0 0; font-size: 13px; font-family: monospace; font-weight: bold;">
🔗 {cloudflare_url}
</p>
🔗 <a href="{cloudflare_url}" onclick="var w=window.open(this.href,'_blank');if(!w){{return true;}}return false;"
style="color: #000000; text-decoration: underline; cursor: pointer;">{cloudflare_url}</a>
</p>{login_block}
</div>
"""
def _show_and_embed(port: int, *, cloudflare_url: "str | None" = None):
"""Render the Unsloth header + iframe for *port*, with a shareable-link card above
when *cloudflare_url* is set. Falls back to serve_kernel_port_as_iframe."""
url = get_colab_url(port)
logger.info(f"🌐 Unsloth Studio URL: {url}")
if cloudflare_url:
logger.info(f"🔗 Shareable Cloudflare link: {cloudflare_url}")
# Height for serve_kernel_port_as_iframe (~82vh on a 1080p screen, clamped).
_COLAB_IFRAME_HEIGHT = 900
def _embed_kernel_port_iframe(port: int) -> bool:
"""Embed Studio via Colab's native kernel-port iframe helper.
Only trusted on a real Colab runtime: colabtools can import ``google.colab`` and
queue browser-side JS without appending an iframe, so callers outside Colab must use
the HTML iframe path instead.
"""
if not _is_colab_runtime():
return False
try:
from google.colab import output as colab_output
except ImportError:
return False
try:
colab_output.serve_kernel_port_as_iframe(
port,
height = _COLAB_IFRAME_HEIGHT,
width = "100%",
)
return True
except Exception as e:
logger.info(f"serve_kernel_port_as_iframe failed ({e}); trying HTML iframe.")
return False
def _embed_html_iframe(url: str, port: int) -> bool:
"""Fallback embed: raw HTML iframe when the Colab helper is unavailable."""
try:
from IPython.display import HTML, display
except ImportError:
return False
iframe_id = f"unsloth-studio-{port}"
# Truncated header URL — best-effort, falls back to full URL.
try:
port_prefix = f"{port}-"
idx = url.index(port_prefix)
next_dash = url.index("-", idx + len(port_prefix))
short_url = url[: next_dash + 1] + "..."
except (ValueError, IndexError):
short_url = url
if cloudflare_url:
display(HTML(_shareable_link_html(cloudflare_url)))
short_url = _short_colab_url(url, port)
iframe_id = f"unsloth-studio-{port}"
try:
display(
HTML(f"""
<div style="font-family:system-ui,-apple-system,sans-serif;margin:8px 0;
@ -266,41 +548,110 @@ def _show_and_embed(port: int, *, cloudflare_url: "str | None" = None):
</div>
""")
)
except Exception:
# Fallback: Colab's built-in helper.
return True
except Exception as e:
logger.info(f"HTML iframe embed failed ({e}).")
return False
def _show_and_embed(
port: int,
*,
cloudflare_url: "str | None" = None,
colab_login: "tuple[str, str] | None" = None,
cloudflare_requested: bool = False,
):
"""Render the Unsloth ready card + iframe for *port*.
Prefer Colab's ``serve_kernel_port_as_iframe`` on real Colab; raw HTML iframe is the
fallback. Cloudflare cards stay clickable.
"""
url = get_colab_url(port)
logger.info(f"🌐 Unsloth Studio URL: {url}")
if cloudflare_url:
logger.info(f"🔗 Shareable Cloudflare link: {cloudflare_url}")
_warn_colab_cloudflare_missing(
use_cloudflare = cloudflare_requested,
cloudflare_url = cloudflare_url,
)
# Fold the credentials into the link card rather than a second card below it.
credentials_shown = False
if cloudflare_url:
try:
from google.colab import output as colab_output
colab_output.serve_kernel_port_as_iframe(port, height = 900, width = "100%")
except ImportError:
pass
from IPython.display import HTML, display
username, password = colab_login if colab_login else (None, None)
display(HTML(_shareable_link_html(cloudflare_url, password, username)))
credentials_shown = bool(colab_login)
except Exception as e:
logger.info(f"Could not render Cloudflare link card ({e}).")
if colab_login and not credentials_shown:
try:
_show_colab_login_credentials(*colab_login)
except Exception as e:
logger.info(f"Could not render Colab login card ({e}).")
# With a tunnel up the embed below is skipped, so the ready card would only restate
# the link card and print a proxy URL that 404s outside this tab.
skip_ready_card = _is_colab_runtime() and bool(cloudflare_url)
if not skip_ready_card:
try:
show_link(
port,
_url = url,
has_cloudflare_link = bool(cloudflare_url),
cloudflare_requested = cloudflare_requested,
)
except Exception as e:
logger.info(f"Could not render Unsloth link card ({e}).")
# On Colab with a working tunnel, skip the in-cell proxy embed (often blank).
if _is_colab_runtime() and cloudflare_url:
return
# Real Colab: kernel helper needs only the port (works when eval_js failed).
if _is_colab_runtime():
if _embed_kernel_port_iframe(port):
return
_embed_html_iframe(url, port)
def start(port: int = 8888, *, cloudflare: bool = False):
def start(port: int = 8888, *, cloudflare: "bool | None" = None):
"""Start Unsloth Studio in Colab and display the URL.
Args:
port: Port to bind/serve on.
cloudflare: Opt in to a shareable Cloudflare HTTPS link reachable from any
device (default OFF). It exposes Unsloth's login page beyond Colab, so it
stays an explicit opt-in; the default shows only the in-tab proxy iframe.
cloudflare: Shareable Cloudflare HTTPS link. ``None`` (default) auto-enables on
real Colab because the in-cell proxy embed is often blank; pass ``False`` to
skip the tunnel or ``True`` to force it on other runtimes.
Usage:
start() # Colab-proxy iframe only (default)
start(cloudflare=True) # also open a shareable Cloudflare link
start() # Cloudflare link on Colab (auto); proxy iframe elsewhere
start(cloudflare=False) # Colab proxy iframe only (often blank on current Colab)
start(cloudflare=True) # force Cloudflare link on any runtime
"""
import time
logger.info("🦥 Starting Unsloth Studio...")
use_cloudflare = _colab_wants_cloudflare(cloudflare)
# Fast path: Unsloth already running (cell re-run). Re-launching would collide on
# the port, so just re-show the link and iframe.
# Fast path: already running (cell re-run); re-show link/iframe instead of rebinding the port.
if _is_studio_healthy(port):
logger.info(f" Unsloth is already running on port {port} — reusing existing server.")
# try/finally: tear the tunnel down even if interrupted mid-start/render.
try:
cf_url = start_cloudflare_tunnel(port) if cloudflare else None
colab_login = _finalize_colab_admin_password() if use_cloudflare else None
cf_url = start_cloudflare_tunnel(port) if use_cloudflare else None
_publish_cloudflare_url(cf_url)
_show_and_embed(port, cloudflare_url = cf_url)
_show_and_embed(
port,
cloudflare_url = cf_url,
colab_login = colab_login,
cloudflare_requested = use_cloudflare,
)
for _ in range(10000):
time.sleep(300)
print("=", end = "", flush = True)
@ -313,7 +664,6 @@ def start(port: int = 8888, *, cloudflare: bool = False):
logger.info(" Loading backend...")
from run import run_server
# Auto-detect frontend path
repo_root = Path(__file__).parent.parent
frontend_path = repo_root / "frontend" / "dist"
@ -323,8 +673,7 @@ def start(port: int = 8888, *, cloudflare: bool = False):
logger.info(" Starting server...")
try:
# cloudflare=False: this helper owns the tunnel (Colab's own
# start(cloudflare=...) drives it), so pin it off explicitly.
# cloudflare=False: this helper owns the tunnel (via start(cloudflare=...)), so pin it off.
app = run_server(
host = "0.0.0.0",
port = port,
@ -339,14 +688,12 @@ def start(port: int = 8888, *, cloudflare: bool = False):
logger.error(f"❌ Unsloth Studio failed to start: {exc}")
return
# run_server auto-increments the port if in use; read back the bound port so the
# proxy URL and iframe point at the right place.
# run_server may auto-increment the port; read back the bound port for the proxy URL/iframe.
actual_port: int = getattr(getattr(app, "state", None), "server_port", None) or port
logger.info(f" Server started on port {actual_port}!")
# Poll health endpoint before showing the link — avoids the race where ready_event
# fires but the process hasn't finished binding.
# Poll health before showing the link: avoids the race where ready_event fires pre-bind.
import urllib.request
server_ready = False
@ -365,12 +712,17 @@ def start(port: int = 8888, *, cloudflare: bool = False):
)
return
# Open the tunnel now the server is healthy, publish its URL for /api/health, and
# tear it down on interrupt (try/finally) rather than orphan the process.
# Server healthy: finalize Colab auth, open the tunnel, publish URL, tear down on interrupt.
try:
cf_url = start_cloudflare_tunnel(actual_port) if cloudflare else None
colab_login = _finalize_colab_admin_password() if use_cloudflare else None
cf_url = start_cloudflare_tunnel(actual_port) if use_cloudflare else None
_publish_cloudflare_url(cf_url)
_show_and_embed(actual_port, cloudflare_url = cf_url)
_show_and_embed(
actual_port,
cloudflare_url = cf_url,
colab_login = colab_login,
cloudflare_requested = use_cloudflare,
)
# Keep kernel alive so the daemon server thread runs.
for _ in range(10000):

15
studio/backend/core/export/export.py
View file

@ -1048,6 +1048,21 @@ class ExportBackend:
"Use the safetensors adapter instead.",
None,
)
# llama.cpp's convert_lora_to_gguf.py has no concept of DoRA's
# lora_magnitude_vector tensors: it only reads the standard
# lora_A/lora_B delta, so exporting a DoRA adapter would silently
# drop the magnitude rescaling and produce a GGUF LoRA file that
# loads fine but no longer matches the trained model.
_peft_config = getattr(self.current_model, "peft_config", {}).get("default")
if getattr(_peft_config, "use_dora", False):
return (
False,
"GGUF LoRA export is not supported for DoRA adapters: the GGUF LoRA "
"format has no way to represent DoRA's magnitude vectors, so the "
"exported file would silently lose the DoRA behavior. Use the "
"safetensors adapter instead, or merge to a full GGUF model.",
None,
)
outtype = str(gguf_outtype).lower()
if outtype not in _GGUF_LORA_OUTTYPES:
return (

25
studio/backend/core/inference/inference.py
View file

@ -8,6 +8,7 @@ from unsloth.chat_templates import get_chat_template
from transformers import TextIteratorStreamer, TextStreamer
from peft import PeftModel, PeftModelForCausalLM
import contextlib
import json
import sys
import torch
@ -1942,8 +1943,30 @@ class InferenceBackend:
+ text
+ "<|text_end|>\n<|audio_start|><|global_features_start|>\n"
)
with torch.inference_mode():
with torch.amp.autocast("cuda", dtype = model.dtype):
# Derive the autocast device from the loaded model, not from the
# global backend: a CPU-fallback DAC on an XPU/CUDA host must not
# open a GPU autocast context around CPU tensors.
device_type = (
model.device.type
if hasattr(model.device, "type")
else str(model.device).split(":", 1)[0]
)
# Clamp to autocast-supported backends so exotic devices
# (e.g. "meta" during accelerate offloaded loading) do not raise.
# MPS is autocast-supported since torch 2.3, keep it in the set.
if device_type not in ("cuda", "xpu", "mps", "cpu"):
device_type = "cpu"
# CPU and XPU autocast only accept bfloat16/float16. For a
# float32 model, skip autocast entirely to avoid raising or
# producing a warning on every generate call.
autocast_dtype_supported = model.dtype in (torch.bfloat16, torch.float16)
if device_type in ("cpu", "xpu") and not autocast_dtype_supported:
autocast_ctx = contextlib.nullcontext()
else:
autocast_ctx = torch.amp.autocast(device_type, dtype = model.dtype)
with autocast_ctx:
inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
generated = model.generate(
**inputs,

564
studio/backend/core/inference/llama_cpp.py
View file

@ -33,6 +33,7 @@ from typing import (
List,
Literal,
Mapping,
MutableMapping,
Optional,
Union,
)
@ -307,6 +308,26 @@ def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> "list[str]":
_DEFAULT_MAX_TOKENS_FLOOR = 32768
_DEFAULT_FIRST_TOKEN_TIMEOUT_S = 1200.0 # 20 min
def _finalize_reasoning_only_cumulative(
cumulative: str, reasoning_text: str, finish_reason: Optional[str], promote_reasoning_only: bool
) -> str:
"""Close a live thinking block and promote it only after a clean stop.
Local inference streams cumulative snapshots. Replacing ``<think>...`` with
bare reasoning at EOF makes the final snapshot shorter, so suffix-based
route consumers drop the intended fallback. Keep the snapshot append-only.
A length-truncated thought is not a final answer, so close it without
promotion and let the client surface the ``length`` terminal state. Raw
consumers that do not split reasoning from visible content can disable the
fallback to avoid returning the same reasoning twice.
"""
visible_fallback = (
reasoning_text if promote_reasoning_only and finish_reason != "length" else ""
)
return cumulative + "</think>" + visible_fallback
# Only large streamed tool payloads get an early provisional card; render_html
# is exempt because it needs immediate artifact feedback.
_PROVISIONAL_ARGS_MIN_CHARS = 256
@ -2023,6 +2044,10 @@ class LlamaCppBackend:
self._tensor_split: Optional[List[float]] = None
# User-picked physical GPU indices (None = automatic selection).
self._gpu_ids: Optional[List[int]] = None
# RAW requested GPU pin, before the fit narrowed it. self._gpu_ids records the
# EFFECTIVE (fit-narrowed) pin for /status; dedupe compares this raw value so a
# [0, 1] narrowed to [0] and re-sent as [0, 1] still matches (#7239).
self._requested_gpu_ids: Optional[List[int]] = None
# Layer load kept multi-GPU only to honor a downgraded tensor request, so a
# later explicit tensor-off reloads instead of deduping to it (#6659).
self._layer_preserves_tensor_intent: bool = False
@ -2494,6 +2519,46 @@ class LlamaCppBackend:
"""User-picked physical GPU indices, or None for automatic selection."""
return self._gpu_ids
@property
def requested_gpu_ids(self) -> Optional[List[int]]:
"""RAW requested GPU pin (before the fit narrowed it), or None for auto.
gpu_ids echoes the EFFECTIVE pin for /status."""
return self._requested_gpu_ids
def matches_gpu_ids(self, gpu_ids: Optional[List[int]]) -> bool:
"""Whether a requested pin is already satisfied by the active runner.
A regular GGUF load may narrow the requested placement pool to the
smallest fitting subset. Accept both the original request and the
effective status-echoed subset so either can round-trip without a
needless reload. Diffusion drives one device and keeps its existing
lowest-device normalization.
"""
if self._is_diffusion:
requested = [sorted(int(x) for x in gpu_ids)[0]] if gpu_ids else None
return requested == (self._gpu_ids or None)
requested = sorted(int(x) for x in gpu_ids) if gpu_ids else None
raw = self._requested_gpu_ids or None
effective = self._gpu_ids or None
return requested == raw or requested == effective
def _record_matching_gpu_request(self, gpu_ids: Optional[List[int]]) -> None:
"""Adopt the caller's explicit pool after a full already-loaded match.
Matching an effective subset avoids a reload, but the incoming request
is still the user's latest placement intent. Record it so status and a
later reload do not restore GPUs the user just removed.
"""
if self._is_diffusion:
self._requested_gpu_ids = [sorted(int(x) for x in gpu_ids)[0]] if gpu_ids else None
else:
self._requested_gpu_ids = sorted(int(x) for x in gpu_ids) if gpu_ids else None
if self._last_load_kwargs is not None:
self._last_load_kwargs["gpu_ids"] = (
list(self._requested_gpu_ids) if self._requested_gpu_ids else None
)
@property
def n_layers(self) -> Optional[int]:
"""Model layer count (GGUF block_count), or None if unknown."""
@ -2737,6 +2802,7 @@ class LlamaCppBackend:
"found": False,
"mtp_token": None,
"supports_mtp": False,
"mtp_probe_inconclusive": True,
"ngram_mod_flavor": None,
"supports_ngram_mod": False,
"spec_draft_n_max_flag": None,
@ -2769,6 +2835,9 @@ class LlamaCppBackend:
supports_no_cache_prompt = False
supports_metrics = False
supports_slot_save = False
saw_spec_type = False
probe_ok = False
help_text = ""
try:
probe_env = cls._llama_server_env_for_binary(bin_path)
result = subprocess.run(
@ -2780,6 +2849,7 @@ class LlamaCppBackend:
check = False,
env = probe_env,
)
probe_ok = result.returncode == 0
help_text = (result.stdout or "") + "\n" + (result.stderr or "")
# Split into per-flag blocks (each --flag line + its indented
# continuation), so the "argument has been removed" description
@ -2824,17 +2894,19 @@ class LlamaCppBackend:
return False
return "argument has been removed" not in desc
# MTP token from the --spec-type line.
spec_line = ""
for line in help_text.splitlines():
if "--spec-type" in line:
spec_line = line
break
# PR #22673 used draft-mtp; later renamed to mtp.
if "draft-mtp" in spec_line:
mtp_token = "draft-mtp"
elif re.search(r"[|,\[]mtp[|,\]]", spec_line):
mtp_token = "mtp"
# MTP token from the full --spec-type help block (decl + indented
# continuation). First-line-only probing missed builds putting the
# enum on the next line (#7302). Prefer draft-mtp (PR #22673) over mtp.
spec_help = blocks.get("--spec-type") or ""
if not spec_help:
# Fallback: join --spec-type lines, avoiding incidental "mtp" in --help.
spec_help = "\n".join(
line for line in help_text.splitlines() if "--spec-type" in line
)
mtp_token = cls._mtp_token_from_spec_help(spec_help)
# Only a resolved --spec-type block confirms missing MTP; empty/crash
# leaves saw_spec_type False so supports_mtp fails open.
saw_spec_type = bool(spec_help.strip()) and "--spec-type" in spec_help
# ngram-mod flag flavor. Post-rename builds advertise both new
# args (real) and legacy ones (stubs); pre-rename builds only
@ -2870,11 +2942,29 @@ class LlamaCppBackend:
supports_slot_save = _is_real("--slot-save-path")
except (OSError, subprocess.SubprocessError) as exc:
logger.debug(f"llama-server --help probe failed: {exc}")
saw_spec_type = False
probe_ok = False
help_text = ""
help_nonempty = bool(help_text.strip())
# Confirmed only when a successful --help lists a --spec-type block with
# mtp/draft-mtp; nonempty --help without it is a definitive pre-spec
# binary; failed/empty probes stay inconclusive (#7302).
if saw_spec_type and probe_ok:
supports_mtp = mtp_token is not None
mtp_probe_inconclusive = False
elif help_nonempty and probe_ok:
supports_mtp = False
mtp_probe_inconclusive = False
else:
supports_mtp = False
mtp_probe_inconclusive = True
info = {
"found": True,
"mtp_token": mtp_token,
"supports_mtp": mtp_token is not None,
"supports_mtp": supports_mtp,
"mtp_probe_inconclusive": mtp_probe_inconclusive,
"ngram_mod_flavor": ngram_mod_flavor,
"supports_ngram_mod": ngram_mod_flavor is not None,
"spec_draft_n_max_flag": spec_draft_n_max_flag,
@ -2890,6 +2980,21 @@ class LlamaCppBackend:
cls._capability_cache[cache_key] = info
return info
@staticmethod
def _mtp_token_from_spec_help(spec_help: str) -> Optional[str]:
"""Extract ``draft-mtp`` / ``mtp`` from a ``--spec-type`` help snippet.
Prefers ``draft-mtp`` (llama.cpp PR #22673) over the later bare ``mtp``
rename. Returns ``None`` when neither token appears as an enum value.
"""
text = spec_help or ""
if "draft-mtp" in text:
return "draft-mtp"
# Bare `mtp` enum token (`|mtp|`, `,mtp,`, ...), not a substring.
if re.search(r"(?<![A-Za-z0-9_-])mtp(?![A-Za-z0-9_-])", text):
return "mtp"
return None
# ── GPU allocation ────────────────────────────────────────────
@staticmethod
@ -3078,7 +3183,7 @@ class LlamaCppBackend:
@staticmethod
def _amd_apu_wants_unified_memory(gpu_indices = None) -> bool:
"""True only for AMD unified-memory APUs (gfx1150/gfx1151), where
"""True only for AMD unified-memory APUs (gfx1150/gfx1151/gfx1152), where
GGML_CUDA_ENABLE_UNIFIED_MEMORY lets llama.cpp use shared system RAM (it
hurts discrete GPUs). gpu_indices (PHYSICAL ids) scopes the check to the
selected GPUs, so a dGPU on a mixed host is not treated as unified-memory;
@ -3108,7 +3213,9 @@ class LlamaCppBackend:
)
arch_by_id[pid] = _arch.split(":")[0].strip().lower()
for _i in list(gpu_indices) if gpu_indices is not None else list(arch_by_id):
if arch_by_id.get(_i) in {"gfx1150", "gfx1151"}:
# gfx1152 is Krackan Point (Radeon 860M/840M), the third RDNA 3.5
# APU: same shared GPU/system-RAM pool as Strix Point/Halo.
if arch_by_id.get(_i) in {"gfx1150", "gfx1151", "gfx1152"}:
return True
except Exception:
return False
@ -3592,6 +3699,14 @@ class LlamaCppBackend:
# aborts a --split-mode tensor load, so it's dropped for the tensor attempt.
_TENSOR_PARALLEL_KV_TYPES = frozenset({"f16", "bf16", "f32"})
# V cache types that llama.cpp can run WITHOUT flash attention. Only the V
# axis has the dependency: a quantized V cache (q8_0/q4_0/q4_1/q5_0/q5_1/
# iq4_nl) aborts init with "V cache quantization requires flash_attn", while
# a quantized K cache runs fine without FA. So the flash-attn-off crash-
# recovery fallback must reset a quantized V cache to f16 before it can
# launch (and leaves K alone). These three are the only non-quantized types.
_NON_QUANTIZED_KV_TYPES = frozenset({"f16", "bf16", "f32"})
# Main-model placement settings that Manual mode owns. They must not leak
# from Studio's parent environment into llama-server and silently override
# the command assembled from the current request. Draft-model placement is
@ -4581,6 +4696,14 @@ class LlamaCppBackend:
LlamaCppBackend._gguf_skip_value(f, atype)
return None
@classmethod
def _gguf_path_is_diffusion(cls, gguf_path: str, model_identifier: str) -> bool:
"""Classify a downloaded GGUF without mutating the active backend."""
probe = object.__new__(cls)
probe._model_identifier = model_identifier
probe._read_gguf_metadata(gguf_path)
return probe._is_diffusion
def _read_gguf_metadata(self, gguf_path: str) -> None:
"""Read context_length, architecture params, and chat_template from a GGUF header.
@ -5032,11 +5155,14 @@ class LlamaCppBackend:
# the unload reset) so /status doesn't misreport TP and an identical
# re-Apply doesn't reload against stale tensor-parallel state.
self._tensor_parallel = False
# Record only the single device the runner actually uses (the lowest
# selected GPU, chosen above) -- not the whole pick. The diffusion runner
# is single-device, so echoing a multi-GPU list would misreport placement
# in /status and let a re-Apply dedup against GPUs the runner never used.
# The single-device runner records only the lowest selected GPU (chosen
# above), not the whole pick, and clears any explicit pin from a prior
# chat load; a multi-GPU list would misreport placement and mis-dedup.
self._gpu_ids = [sorted(gpu_ids)[0]] if gpu_ids else None
# The frontend prefers requested_gpu_ids when hydrating the picker.
# Diffusion uses only one device, so echo the collapsed effective pin,
# not unused members of the original request.
self._requested_gpu_ids = list(self._gpu_ids) if self._gpu_ids else None
if hf_variant:
self._hf_variant = hf_variant
elif gguf_path:
@ -5960,6 +6086,24 @@ class LlamaCppBackend:
and ("unknown" in text or "unsupported" in text or "not supported" in text)
)
@staticmethod
def _mmproj_retry_failure_message(*, projector_confirmed: bool, detail: str) -> str:
"""User-facing error when the text-only --mmproj strip retry also fails.
Confirmed projector-format mismatches keep the historical wording.
Bare signal crashes (common on some ROCm/driver paths) must not be
reported as "Vision projector incompatible" — that misled #7302.
"""
if projector_confirmed:
return (
"Vision projector incompatible with this llama.cpp "
"build, and the text-only retry also failed: " + detail
)
return (
"Vision model failed to start (llama-server crashed with "
"--mmproj), and the text-only retry also failed: " + detail
)
@staticmethod
def _output_has_nonprojector_diagnostic(output: str) -> bool:
"""True when the output already names a concrete non-projector cause (out
@ -6016,6 +6160,21 @@ class LlamaCppBackend:
cls._is_signal_crash(returncode) or cls._is_abort_exit(returncode)
)
@staticmethod
def _canonical_long_flag(name: str) -> str:
"""Return ``name`` with llama.cpp's long-option underscore normalization.
llama.cpp runs ``std::replace(arg.begin(), arg.end(), '_', '-')`` on any
argv token that starts with ``--`` before looking it up, so a legal
pass-through spelling like ``--cache_type_v`` parses as
``--cache-type-v``. Mirror that here so managed-flag matching sees the
same canonical name. Short flags (``-ctv``) never carry underscores and
keep their exact spelling; pass only the flag name (no attached value).
"""
if name.startswith("--"):
return name.replace("_", "-")
return name
@staticmethod
def _with_flash_attn_off(cmd: list[str]) -> Optional[list[str]]:
"""Return cmd with flash attention forced off, or None when its effective
@ -6048,8 +6207,76 @@ class LlamaCppBackend:
out[i + 1] = "off"
elif explicit(i) is None: # bare flag (reads as on) -> explicit off
out[i] = f"{tok}=off"
# A quantized V cache requires flash attention in llama.cpp: the init
# aborts with "V cache quantization requires flash_attn". A quantized K
# cache has no such requirement and runs fine without FA, so it is left
# untouched -- resetting it would needlessly enlarge the K cache and can
# OOM a memory-constrained config. Studio launches with FA on, so a
# quantized --cache-type-v is legal at launch but would make THIS FA-off
# retry crash on init instead of recovering. Reset a quantized V cache --
# main and draft (the draft context shares the global --flash-attn flag,
# so its V cache aborts too) -- to f16 (the llama.cpp default);
# non-quantized types -- f16/bf16/f32 -- run fine without FA and are left
# untouched. The value is rewritten in place so the list length is
# preserved for downstream slices, matching the flash-attn flip above.
_v_cache_flags = (
"--cache-type-v",
"-ctv",
"--cache-type-v-draft",
"--spec-draft-type-v",
"-ctvd",
)
_cache_reset = False
for i, tok in enumerate(out):
# llama.cpp rewrites '_' to '-' for any argv token starting with
# '--' before matching, so a legal pass-through spelling such as
# --cache_type_v parses as --cache-type-v and still enables a
# quantized V cache. Canonicalize the flag name the same way so the
# reset recognizes the underscore aliases too; short flags (-ctv)
# and the type value are left untouched.
name = LlamaCppBackend._canonical_long_flag(tok.partition("=")[0])
if name not in _v_cache_flags:
continue
if "=" in tok:
flag, _, value = tok.partition("=")
if value.strip().lower() not in LlamaCppBackend._NON_QUANTIZED_KV_TYPES:
out[i] = f"{flag}=f16"
_cache_reset = True
elif i + 1 < len(out):
if out[i + 1].strip().lower() not in LlamaCppBackend._NON_QUANTIZED_KV_TYPES:
out[i + 1] = "f16"
_cache_reset = True
if _cache_reset:
logger.info(
"V cache dtype reset to f16 because flash attention was disabled "
"by the crash-recovery fallback (quantized V cache requires flash "
"attention in llama.cpp; the K cache is left untouched)."
)
return out
@staticmethod
def _drop_env_quantized_v_cache(env: MutableMapping[str, str]) -> bool:
"""Drop an inherited quantized V-cache env var (main or draft) in place
before a flash-attn-off retry, returning True if anything was removed.
The argv rewrite in ``_with_flash_attn_off`` only reaches flags on the
command line. Studio deliberately lets an env-only cache type reach the
child untouched (an asymmetric K/V env must survive), so a quantized V
cache set purely through ``LLAMA_ARG_CACHE_TYPE_V`` (or the draft
``LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V``) would still abort the FA-off retry
with "V cache quantization requires flash_attn". Dropping it lets
llama.cpp fall back to the f16 default. Only V is dropped: a quantized K
cache runs fine without flash attention, so its env var is preserved.
"""
dropped = False
for var in ("LLAMA_ARG_CACHE_TYPE_V", "LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V"):
value = (env.get(var) or "").strip().lower()
if value and value not in LlamaCppBackend._NON_QUANTIZED_KV_TYPES:
env.pop(var, None)
dropped = True
return dropped
@staticmethod
def _strip_mmproj_args(cmd: list[str]) -> list[str]:
"""Return cmd without the '--mmproj <path>' pair (text-only retry).
@ -6161,6 +6388,8 @@ class LlamaCppBackend:
gpu_layers: int = -1,
n_cpu_moe: int = 0,
tensor_split: Optional[List[float]] = None,
# Explicit GPU placement pool (issue #7164). None/[] = auto-select;
# the fitter may pin the smallest subset of this pool that fits.
gpu_ids: Optional[List[int]] = None,
n_threads: Optional[int] = None,
n_gpu_layers: Optional[int] = None, # caller compat, unused
@ -6258,15 +6487,63 @@ class LlamaCppBackend:
self._cancel_event.clear()
# ── Phase 1: kill old process (under lock, fast) ──────────
with self._lock:
self._kill_process()
# Resolve llama-server now but defer a not-found error: a block-diffusion
# GGUF uses the diffusion runner, and its arch is only known after the header.
binary = self._find_llama_server_binary()
is_vulkan_backend = self._is_vulkan_backend(binary)
# ── Vulkan-ordinal preflight (BEFORE the Phase 1 kill) ────────
# An explicit Vulkan pin the ggml probe never enumerated cannot be honored.
# Validate it ABOVE the kill so an invalid selection leaves the live model
# untouched: CUDA ids are range-checked at the route, but Vulkan ordinals are
# not, so a stale gpu_ids=[99] used to kill the server then 400, leaving
# nothing running (#7239). _get_gpu_memory needs only the binary (safe pre-
# download) and reuses the later fit's issubset logic. Guarded on a found
# Vulkan build + a pin so a deferred not-found stays deferred for diffusion.
if is_vulkan_backend and gpu_ids and binary:
_pf_wanted = {int(x) for x in gpu_ids}
_pf_probed = {g[0] for g in self._get_gpu_memory(binary)}
if not _pf_wanted.issubset(_pf_probed):
raise ValueError(
f"Requested Vulkan GPU ordinal(s) {sorted(_pf_wanted)} not "
f"present. Available Vulkan devices: {sorted(_pf_probed)}."
)
# A remote uncached GGUF may only reveal that it needs the
# single-device diffusion runner after download. On Vulkan, an
# explicit gpu_ids request cannot be mapped from ggml ordinals to
# that runner's CUDA physical index. Download and classify the main
# file before killing the healthy server so this late rejection is
# non-destructive. The Phase 2 call below reuses this cached path.
_preflight_model_path = None
if is_vulkan_backend and gpu_ids and hf_repo:
_resolved_repo = _resolve_repo_id_casing(hf_repo)
if _resolved_repo != hf_repo:
logger.info(
"Using cached repo_id casing '%s' for requested '%s'",
_resolved_repo,
hf_repo,
)
hf_repo = _resolved_repo
with _hf_offline_if_dns_dead():
_preflight_model_path = self._download_gguf(
hf_repo = hf_repo,
hf_variant = hf_variant,
hf_token = hf_token,
)
if self._gguf_path_is_diffusion(_preflight_model_path, model_identifier):
raise ValueError(
"GPU selection (gpu_ids) is not supported for a DiffusionGemma "
"GGUF on a Vulkan llama.cpp build: the diffusion runner selects "
"its device by CUDA physical index, which has no defined mapping "
"to ggml Vulkan device ordinals. Omit gpu_ids to use the default "
"device."
)
# ── Phase 1: kill old process (under lock, fast) ──────────
with self._lock:
self._kill_process()
# ── Phase 2: download (NO lock held, so cancel can proceed) ──
# mtp_draft_path arrives set for local Gemma loads (detected
# sibling); for -hf loads it's None here and resolved just below.
@ -6288,7 +6565,7 @@ class LlamaCppBackend:
)
hf_repo = _resolved_repo
with _hf_offline_if_dns_dead():
model_path = self._download_gguf(
model_path = _preflight_model_path or self._download_gguf(
hf_repo = hf_repo,
hf_variant = hf_variant,
hf_token = hf_token,
@ -6338,6 +6615,18 @@ class LlamaCppBackend:
# Block-diffusion GGUFs (DiffusionGemma) cannot run on llama-server;
# serve them with the diffusion runner (same OpenAI-compat interface).
if self._is_diffusion:
# The diffusion runner pins its child by CUDA visibility mask, so a
# ggml Vulkan ordinal cannot be honored (wrong GPU / CPU fallback).
# Route and remote-download preflights reject before teardown; keep
# this as a final defense if classification ever disagrees.
if is_vulkan_backend and gpu_ids:
raise ValueError(
"GPU selection (gpu_ids) is not supported for a DiffusionGemma "
"GGUF on a Vulkan llama.cpp build: the diffusion runner selects "
"its device by CUDA physical index, which has no defined mapping "
"to ggml Vulkan device ordinals. Omit gpu_ids to use the default "
"device."
)
# Not a tensor/layer GGUF: clear any preserved-fallback flag from a
# prior load (this path skips the command builder that clears it).
self._layer_preserves_tensor_intent = False
@ -6558,6 +6847,12 @@ class LlamaCppBackend:
# Layer-fallback min GPUs; raised below on a tensor downgrade. Bound
# before the try so the --fit-on except path still has it (no UnboundLocal).
_layer_min_gpus = 1
# An explicit Vulkan ordinal absent from the ggml probe cannot be
# honored; flag it in the fit and reject after the try (raising inside
# would be swallowed into the --fit-on fallback). Bound before the try.
_vulkan_explicit_unmatched = False
_vulkan_requested_ids: list[int] = []
_vulkan_available_ordinals: list[int] = []
try:
gguf_size = self._get_gguf_size_bytes(model_path)
# Include GPU-loaded mmproj in the fit budget (#5825).
@ -6570,6 +6865,28 @@ class LlamaCppBackend:
# Pass binary so a Vulkan build probes ggml's Vulkan ordinals.
_gpu_mem = self._get_gpu_memory(binary)
gpus = [(idx, free) for idx, free, _t in _gpu_mem]
# Restrict the fit (and thus the layer plan + pin env) to the
# selected GPUs; fail-open if none match so a stale UI choice
# can't strand the load on CPU (issue #7164).
if gpu_ids:
# A Vulkan build indexes by ggml ordinal. An explicit ordinal
# absent from the probe can't be pinned, so reject after the try
# rather than fail-open onto a device the user didn't pick.
_wanted_ids = {int(x) for x in gpu_ids}
# Reject if ANY requested ordinal is absent, not only when none
# match: [0, 99] against {0, 1} silently drops 99. Comparing the
# full requested set (before filter narrows) still lets the fitter
# pick a valid subset later -- that is narrowing, not absence.
_probed_ordinals = {g[0] for g in gpus}
if is_vulkan_backend and not _wanted_ids.issubset(_probed_ordinals):
_vulkan_explicit_unmatched = True
_vulkan_requested_ids = sorted(_wanted_ids)
_vulkan_available_ordinals = sorted(_probed_ordinals)
# Restrict the probed pool to the selection; fail-open (keep the
# full pool) if none match so a stale UI choice can't strand the
# load on CPU (issue #7164).
_sel_gpus = [g for g in gpus if g[0] in _wanted_ids]
gpus = _sel_gpus if _sel_gpus else gpus
total_by_idx = {idx: total for idx, _f, total in _gpu_mem}
# GPU picker: restrict every mode to the chosen devices, so
# auto selection only considers them and manual mask to
@ -7396,6 +7713,17 @@ class LlamaCppBackend:
tp_tensor_split = None
effective_ctx = requested_ctx # fall back to original
# An unenumerated explicit Vulkan ordinal can't be pinned; fail loudly
# instead of fitting onto an unselected device. Clear the raw selection
# the early state-publish recorded so it never leaks into gpu_ids (#7239).
if _vulkan_explicit_unmatched:
self._gpu_ids = None
self._requested_gpu_ids = None
raise ValueError(
f"Requested Vulkan GPU ordinal(s) {_vulkan_requested_ids} not "
f"present. Available Vulkan devices: {_vulkan_available_ordinals}."
)
# GPU picker: when no narrower subset was chosen (manual, or
# a failed/file-size selection), pin the whole picked set so the
# model can't spill onto an unpicked GPU.
@ -7759,11 +8087,45 @@ class LlamaCppBackend:
", ".join(unsupported_cache_flags),
)
# Vulkan pins via --device (a cmd arg, unlike the env-based
# CUDA/ROCm pin below), emitted BEFORE user extras so llama.cpp's
# last-wins parsing lets a user --device override Unsloth's pick.
if is_vulkan_backend and gpu_indices is not None:
cmd += LlamaCppBackend._vulkan_pin_args(gpu_indices)
# Vulkan pins via --device (a cmd arg), before user extras so a user
# --device wins. Fall back to raw ids when the fit did not narrow.
_vulkan_pin_ids = gpu_indices if gpu_indices is not None else (gpu_ids or None)
# Record the pin actually applied (fit-narrowed gpu_indices, else the raw
# request) for the keep-warm loop, dedupe, and /status, so an explicit
# [0, 1] narrowed to [0] records [0] and /status never echoes an ordinal
# the child never saw. Auto selection (no gpu_ids) stays None (#7239).
if is_vulkan_backend:
# Only record an EXPLICIT Vulkan pin: an auto pick still narrows +
# pins below, but recording it would misreport an explicit pin and
# make dedupe miss the loaded server; mirrors the CUDA/ROCm branch.
self._gpu_ids = (
sorted(int(x) for x in _vulkan_pin_ids)
if (gpu_ids and _vulkan_pin_ids)
else None
)
elif gpu_ids:
# Physical pin: the fit-selected subset when the fit ran, else the raw
# user selection so an explicit choice is honoured even when the fit
# could not size the model.
_effective_pin_ids = (
[int(x) for x in gpu_indices]
if gpu_indices is not None
else [int(x) for x in gpu_ids]
)
self._gpu_ids = (
sorted(int(x) for x in _effective_pin_ids) if _effective_pin_ids else None
)
else:
self._gpu_ids = None
# Also record the RAW requested pin (before the fit narrowed it). Load
# dedupe compares this so a [0, 1] narrowed to [0] and re-sent as [0, 1]
# still matches, while /status keeps echoing the effective pin (#7239).
self._requested_gpu_ids = sorted(int(x) for x in gpu_ids) if gpu_ids else None
if is_vulkan_backend and _vulkan_pin_ids is not None:
cmd += LlamaCppBackend._vulkan_pin_args(_vulkan_pin_ids)
# User pass-through args go last so llama.cpp's last-wins parsing
# lets the user override Unsloth's auto-set flags. Already
@ -7832,10 +8194,10 @@ class LlamaCppBackend:
f"Data-center GPU detected: applied DC llama.cpp env tuning (multi_gpu={multi_gpu})"
)
# Pin to selected GPU(s). On ROCm, narrowing only
# CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full set, so
# set HIP_VISIBLE_DEVICES too. Vulkan is pinned via --device
# (above), not here.
# Pin to selected GPU(s) (issue #7164; resolved above into gpu_indices).
# On ROCm, narrowing only CUDA_VISIBLE_DEVICES leaves the AMD child
# seeing the full set, so set HIP_VISIBLE_DEVICES too. Vulkan is pinned
# via --device (above), not here.
# A deliberate zero-offload load with no GPU companions runs
# entirely on CPU, yet a visible CUDA device still costs the child
# ~0.5 GB (context + compute scratch) that the CPU-only
@ -8071,6 +8433,13 @@ class LlamaCppBackend:
_fa_rc,
)
self._kill_process()
# The argv rewrite can't reach an env-only quantized V
# cache; drop it so the FA-off child doesn't abort on it.
if self._drop_env_quantized_v_cache(env):
logger.info(
"Dropped inherited quantized V-cache env for the "
"--flash-attn off retry (requires flash attention)."
)
cmd = _fa_cmd
healthy = _spawn_and_wait(_fa_cmd, label = "-noflash")
@ -8116,6 +8485,13 @@ class LlamaCppBackend:
_probe_rc,
)
self._kill_process()
# The argv rewrite can't reach an env-only quantized V
# cache; drop it so the FA-off child doesn't abort on it.
if self._drop_env_quantized_v_cache(env):
logger.info(
"Dropped inherited quantized V-cache env for the "
"--flash-attn off retry (requires flash attention)."
)
cmd = _fa_cmd
healthy = (
_spawn_and_wait(_fa_cmd, label = "-noflash-mtp")
@ -8198,23 +8574,29 @@ class LlamaCppBackend:
self._kill_process()
# The #6415 split-axis abort is latched earlier (first spawn).
# Skip if a cancel/unload is pending (mirrors the MTP guard).
_projector_msg = self._is_projector_incompatibility(out)
_signal_mmproj_guess = self._is_signal_crash(
_crash_rc
) and not self._output_has_nonprojector_diagnostic(out)
if (
launched_with_mmproj
and not self._cancel_event.is_set()
and (
self._is_projector_incompatibility(out)
or (
self._is_signal_crash(_crash_rc)
and not self._output_has_nonprojector_diagnostic(out)
)
)
and (_projector_msg or _signal_mmproj_guess)
):
logger.warning(
"llama-server could not load this model's vision "
"projector (--mmproj). The installed llama.cpp build is "
"likely too old for it. Loading text-only for this "
"session; run 'unsloth studio update' to enable vision."
)
if _projector_msg:
logger.warning(
"llama-server could not load this model's vision "
"projector (--mmproj). The installed llama.cpp build is "
"likely too old for it. Loading text-only for this "
"session; run 'unsloth studio update' to enable vision."
)
else:
logger.warning(
"llama-server crashed while loading this model's vision "
"projector (--mmproj). Retrying text-only for this "
"session; if this persists, run 'unsloth studio update' "
"or check GPU/driver logs."
)
cmd = self._strip_mmproj_args(_last_spawn_cmd)
# This retry bypasses _spawn_and_wait, so refresh the
# launched-argv snapshot itself -- the zero-offload
@ -8242,14 +8624,16 @@ class LlamaCppBackend:
"(e.g. ROCR_VISIBLE_DEVICES=0 exposes only the first "
"GPU) before launching Unsloth Studio."
)
_retry_detail = self._classify_llama_start_failure(
"\n".join(self._stdout_lines[-50:]),
gguf_path,
self._model_identifier,
_retry_rc,
)
raise RuntimeError(
"Vision projector incompatible with this llama.cpp "
"build, and the text-only retry also failed: "
+ self._classify_llama_start_failure(
"\n".join(self._stdout_lines[-50:]),
gguf_path,
self._model_identifier,
_retry_rc,
self._mmproj_retry_failure_message(
projector_confirmed = _projector_msg,
detail = _retry_detail,
)
)
else:
@ -8479,18 +8863,29 @@ class LlamaCppBackend:
caps = self.probe_server_capabilities(binary)
mtp_token = caps.get("mtp_token") if caps else None
if not mtp_token:
logger.warning(
"Requested MTP speculative decoding but "
"llama-server lacks --spec-type mtp/draft-mtp; "
"run `unsloth studio update`. Loading without "
"speculative decoding."
)
inconclusive = bool(caps.get("mtp_probe_inconclusive")) if caps else True
if inconclusive:
logger.info(
"Requested MTP speculative decoding but llama-server MTP "
"capability probe was inconclusive; loading without "
"speculative decoding."
)
else:
logger.warning(
"Requested MTP speculative decoding but "
"llama-server lacks --spec-type mtp/draft-mtp; "
"run `unsloth studio update`. Loading without "
"speculative decoding."
)
# Override an inherited LLAMA_ARG_SPEC_TYPE=draft-mtp (CLI wins
# over env) so the child matches the binary-capability gate and
# the no-MTP budget, like the sibling no-head/non-MTP fallbacks.
flags.append("--spec-default")
self._speculative_type = "default"
self._spec_fallback_reason = "binary_no_mtp"
if inconclusive:
self._spec_fallback_reason = None
else:
self._spec_fallback_reason = "binary_no_mtp"
return False
draft_n_max = _resolved_draft_n_max()
n_max_flag = caps.get("spec_draft_n_max_flag") or "--spec-draft-n-max"
@ -8756,16 +9151,10 @@ class LlamaCppBackend:
)
):
return False
# A changed GPU pick must reload (compare order-insensitively; None/[]
# both mean automatic). The diffusion runner collapses a multi-GPU pick
# to its single lowest device, so self._gpu_ids holds just that device;
# normalize the request the same way, or a multi-GPU pick that resolves
# to the same device needlessly reloads.
if self._is_diffusion:
requested_gpu_pick = [sorted(gpu_ids)[0]] if gpu_ids else None
else:
requested_gpu_pick = sorted(gpu_ids) if gpu_ids else None
if (self._gpu_ids or None) != requested_gpu_pick:
# A changed GPU pick must reload. Regular GGUF accepts either the raw
# requested placement pool or the effective status-echoed subset;
# diffusion compares its normalized single-device pick.
if not self.matches_gpu_ids(gpu_ids):
return False
# Compare on the canonical requested mode. With --spec-type in
@ -8823,6 +9212,7 @@ class LlamaCppBackend:
current = list(self._extra_args) if self._extra_args is not None else []
if list(extra_args) != current:
return False
self._record_matching_gpu_request(gpu_ids)
return True
def _classify_gpu_offload(
@ -8954,12 +9344,15 @@ class LlamaCppBackend:
self._supports_preserve_thinking = False
self._supports_tools = False
self._cache_type_kv = None
# GPU-pin state describes the active runner only; clear it so an explicit
# pin never leaks into the next (or diffusion) runner.
self._gpu_ids = None
self._requested_gpu_ids = None
self._tensor_parallel = False
self._gpu_memory_mode = "auto"
self._gpu_layers = -1
self._n_cpu_moe = 0
self._tensor_split = None
self._gpu_ids = None
self._layer_preserves_tensor_intent = False
self._speculative_type = None
self._requested_spec_mode = None
@ -10291,6 +10684,7 @@ class LlamaCppBackend:
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
seed: Optional[int] = None,
promote_reasoning_only: bool = True,
_allow_respawn_retry: bool = True,
) -> Generator[Union[str, dict], None, None]:
"""
@ -10373,7 +10767,12 @@ class LlamaCppBackend:
# model put its whole reply in reasoning
# (e.g. Qwen3 always-think). Show it as
# the main response, not a thinking block.
cumulative = reasoning_text
cumulative = _finalize_reasoning_only_cumulative(
cumulative,
reasoning_text,
_metadata_finish_reason,
promote_reasoning_only,
)
yield cumulative
_stream_done = True
break # exit inner while
@ -10470,6 +10869,7 @@ class LlamaCppBackend:
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
seed = seed,
promote_reasoning_only = promote_reasoning_only,
_allow_respawn_retry = False,
)
return
@ -10511,6 +10911,7 @@ class LlamaCppBackend:
confirm_tool_calls: bool = False,
bypass_permissions: bool = False,
permission_mode: Optional[str] = None,
promote_reasoning_only: bool = True,
) -> Generator[dict, None, None]:
"""
Agentic loop: let the model call tools, execute them, and continue.
@ -10853,7 +11254,12 @@ class LlamaCppBackend:
),
}
else:
cumulative_display = reasoning_accum
cumulative_display = _finalize_reasoning_only_cumulative(
cumulative_display,
reasoning_accum,
_iter_finish_reason,
promote_reasoning_only,
)
if not _suppress_visible_output:
yield {
"type": "content",
@ -11317,7 +11723,12 @@ class LlamaCppBackend:
if _reasoning_started_at is not None and not _reasoning_summary_emitted:
_reasoning_summary_emitted = True
yield _reasoning_summary_event(_reasoning_started_at)
cumulative_display = reasoning_accum
cumulative_display = _finalize_reasoning_only_cumulative(
cumulative_display,
reasoning_accum,
_iter_finish_reason,
promote_reasoning_only,
)
if not _suppress_visible_output:
yield {
"type": "content",
@ -11881,7 +12292,12 @@ class LlamaCppBackend:
"text": _strip_tool_markup(cumulative, final = True),
}
else:
cumulative = reasoning_text
cumulative = _finalize_reasoning_only_cumulative(
cumulative,
reasoning_text,
_metadata_finish_reason,
promote_reasoning_only,
)
yield {"type": "content", "text": cumulative}
_stream_done = True
break # exit inner while

4
studio/backend/core/inference/orchestrator.py
View file

@ -27,7 +27,7 @@ import uuid
from io import BytesIO
from pathlib import Path
from typing import Any, Generator, Optional, Tuple, Union
from utils.hardware import prepare_gpu_selection
from utils.hardware import get_device, prepare_gpu_selection
# Re-exported from the shared helper so GGUF, training, and inference share one
# type; kept importable here for backwards compatibility.
@ -1012,6 +1012,8 @@ class InferenceOrchestrator:
)
sub_config["resolved_gpu_ids"] = resolved_gpu_ids
sub_config["gpu_selection"] = gpu_selection
# Parent-detected backend for the worker's apply_gpu_ids().
sub_config["device_backend"] = get_device().value
# Recheck the sidecar reservation BEFORE tearing the old worker down,
# for REPAIRS only: an install holds this same lifecycle gate, so it

160
studio/backend/core/inference/tools.py
View file

@ -18,6 +18,7 @@ import queue
import random
import re
import shlex
import shutil
import ssl
import subprocess
import sys
@ -48,6 +49,7 @@ from loggers import get_logger
logger = get_logger(__name__)
_EXEC_TIMEOUT = 300 # 5 minutes
_DISABLE_DNS_PINNING_ENV = "UNSLOTH_STUDIO_DISABLE_DNS_PINNING"
# Splits the UI source-map from the result; loops strip it (like __IMAGES__).
RAG_SOURCES_SENTINEL = "\n__RAG_SOURCES__:"
@ -328,6 +330,7 @@ def _find_blocked_commands(command: str) -> set[str]:
# Directory holding the sandbox ``sitecustomize.py`` shim (code-interpreter
# path remap); placed on the sandboxed child's PYTHONPATH in _build_safe_env.
_SANDBOX_SITE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox_site")
# ── "Approve for me" (permission_mode="auto") safety detection ──────────────
# Auto mode pauses only calls classified here as potentially unsafe. The sandbox
# and hard blocks (blocklist, rlimits) still apply at run time; this gate only
@ -2491,15 +2494,124 @@ def is_potentially_unsafe_tool_call(name: str, arguments: dict) -> bool:
return True
def _canon_win_path(p: str) -> str:
"""Canonical form for trust comparison: realpath (expands 8.3 aliases and
resolves junctions/symlinks) + normcase/normpath."""
return os.path.normcase(os.path.normpath(os.path.realpath(p)))
def _augment_native_program_roots(roots: list[str]) -> list[str]:
"""Add the native Program Files sibling for any x86 root by stripping the
`` (x86)`` suffix, so a 32-bit process (whose known-folder ids map only to
the x86 root) still trusts a 64-bit Git install."""
out = list(roots)
for root in roots:
base = root.rstrip("\\/")
if base.lower().endswith(" (x86)"):
native = base[: -len(" (x86)")]
if native and native not in out:
out.append(native)
return out
def _windows_program_roots() -> list[str]:
"""Program Files install roots, resolved ONLY from the Windows known-folder
API (SHGetKnownFolderPath). Fails closed (returns ``[]``) if the API is
unavailable: env vars (%ProgramFiles%, even %SystemDrive%) are caller-
overrideable and could relocate the trust boundary, so we never derive a
trusted root from them. On any real Windows host shell32 is present, so
this only returns empty in a broken/non-Windows environment where the
sandbox git-PATH feature is not needed anyway (#7317).
"""
roots: list[str] = []
try:
import ctypes
from ctypes import wintypes
# FOLDERID_ProgramFiles, _ProgramFilesX86, _ProgramFilesX64. The X64
# id (Win10 1703+) yields the native root even from a 32-bit process,
# where the first two both map to Program Files (x86).
folder_ids = (
"{905e63b6-c1bf-494e-b29c-65b732d3d21a}",
"{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}",
"{6D809377-6AF0-444b-8957-A3773F02200E}",
)
_SHGet = ctypes.windll.shell32.SHGetKnownFolderPath
_CoTaskMemFree = ctypes.windll.ole32.CoTaskMemFree
for fid in folder_ids:
guid = ctypes.create_string_buffer(16)
ctypes.windll.ole32.CLSIDFromString(wintypes.LPCWSTR(fid), ctypes.byref(guid))
ptr = ctypes.c_wchar_p()
if _SHGet(ctypes.byref(guid), 0, None, ctypes.byref(ptr)) == 0:
if ptr.value:
roots.append(ptr.value)
_CoTaskMemFree(ptr)
except Exception:
return []
return _augment_native_program_roots(roots)
def _resolve_trusted_windows_git() -> tuple[str, str]:
"""Find a git launcher in a TRUSTED Program Files dir. Returns
``(canonical_dir, ext)`` or ``("", "")``.
``shutil.which`` returns only the first PATH match, which may be an
untrusted user shim; scan the remaining PATH entries for a later trusted
Git so bare ``git`` still resolves (#7317).
"""
exts = [e for e in (os.environ.get("PATHEXT") or ".EXE;.CMD;.BAT;.COM").split(os.pathsep)]
candidates: list[str] = []
primary = shutil.which("git")
if primary:
candidates.append(primary)
for entry in (os.environ.get("PATH") or "").split(os.pathsep):
entry = entry.strip().strip('"')
if not entry or not os.path.isabs(entry):
continue
for ext in exts:
cand = os.path.join(entry, "git" + ext)
if os.path.isfile(cand):
candidates.append(cand)
for git_exe in candidates:
git_dir = os.path.dirname(git_exe)
if os.path.isabs(git_dir) and _is_trusted_windows_program_dir(git_dir):
return os.path.realpath(git_dir), os.path.splitext(git_exe)[1].upper()
return "", ""
def _is_trusted_windows_program_dir(path: str) -> bool:
"""True when ``path`` sits under a system-managed Program Files root.
Only the Program Files roots are trusted (admin-writable only), resolved
via the known-folder API so an overridden env var cannot relocate them,
never ``%SystemRoot%`` (Git does not install there and it holds
world-writable subdirs like ``Windows\\Temp``). Per-user managers
(Scoop/Choco shims under the profile) are refused. Paths are canonicalized
so 8.3 aliases and junctions still resolve to their real root (#7317).
"""
norm = _canon_win_path(path)
for root in _windows_program_roots():
root_norm = _canon_win_path(root)
if norm == root_norm or norm.startswith(root_norm + os.sep):
return True
return False
def _build_safe_env(workdir: str) -> dict[str, str]:
"""Build a minimal, credential-free environment for sandboxed subprocesses.
Whitelist-built from scratch (parent env NOT inherited): only PATH/HOME/
TMPDIR/LANG/TERM/PYTHONIOENCODING/PYTHONPATH (+VIRTUAL_ENV or Windows
SystemRoot) reach the child; all credential vars (HF_TOKEN, AWS_*, etc.)
are absent. HOME points at the sandbox workdir so SDKs can't read the
SystemRoot and a minimal PATHEXT) reach the child; all credential vars
(HF_TOKEN, AWS_*, etc.) are absent. HOME points at the sandbox workdir so SDKs can't read the
operator's cached creds. PYTHONPATH carries only the sandbox sitecustomize
shim directory.
PATH starts with the Studio interpreter / venv and OS system dirs so
``python``/``pip`` stay pinned. On Windows only, Git-for-Windows install
dirs from the host PATH are appended so bare ``git`` resolves (#7317).
User-writable host PATH entries (venv, ``node_modules/.bin``, etc.) are
never inherited — they could shadow auto-safe terminal commands.
"""
# Start from the running interpreter's dir so 'python'/'pip' resolve to the
# same environment the Unsloth server runs in.
@ -2519,6 +2631,20 @@ def _build_safe_env(workdir: str) -> dict[str, str]:
else:
path_entries.extend(["/usr/local/bin", "/usr/bin", "/bin"])
# Windows Git installs live outside System32; inherit the dir of the git
# the HOST shell resolves, but ONLY when it sits under a system install
# root (Program Files, windir). A user-writable dir (Scoop/Choco shims)
# is refused: it would let an attacker drop rg.exe/jq.exe beside git and
# have an auto-approved bare command execute it (#7317).
git_ext = ""
if sys.platform == "win32":
# Append the CANONICAL (realpath) trusted git dir, scanning past any
# untrusted user shim that sorts first on PATH; the canonical path
# cannot be retargeted via a junction after the trust check.
_trusted_git_dir, git_ext = _resolve_trusted_windows_git()
if _trusted_git_dir:
path_entries.append(_trusted_git_dir)
# Deduplicate, preserving order.
deduped = list(dict.fromkeys(p for p in path_entries if p))
@ -2538,6 +2664,15 @@ def _build_safe_env(workdir: str) -> dict[str, str]:
# Windows needs SystemRoot for Python/subprocess to work.
if sys.platform == "win32":
env["SystemRoot"] = os.environ.get("SystemRoot", r"C:\Windows")
# Restrict PATHEXT so cwd .BAT/.CMD cannot hijack bare names (#7317).
pathext = ".EXE;.COM"
if git_ext and git_ext not in (".EXE", ".COM"):
# Keep the host git launcher (e.g. a .CMD shim) resolvable.
pathext += ";" + git_ext
env["PATHEXT"] = pathext
# cmd/CreateProcess search cwd before PATH for bare names; disable so
# a workdir rg.exe/git.exe cannot shadow auto-approved commands.
env["NoDefaultCurrentDirectoryInExePath"] = "1"
return env
@ -4060,13 +4195,18 @@ def _fetch_url_raw(
budget_error = _fetch_budget_exceeded(deadline, cancel_event)
if budget_error is not None:
return budget_error, "", ""
# Pin to the validated IP (prevents DNS rebinding): rewrite URL to
# the IP, set the Host header.
cp = urlparse(current_url)
# Bracket IPv6 addresses so the netloc is valid in a URL.
ip_str = f"[{pinned_ip}]" if ":" in pinned_ip else pinned_ip
ip_netloc = f"{ip_str}:{cp.port}" if cp.port else ip_str
pinned_url = urlunparse(cp._replace(netloc = ip_netloc))
validated_netloc = f"[{current_host}]" if ":" in current_host else current_host
if cp.port:
validated_netloc = f"{validated_netloc}:{cp.port}"
if os.environ.get(_DISABLE_DNS_PINNING_ENV) == "1":
# Enterprise proxies need the hostname in CONNECT for policy and TLS interception.
request_url = urlunparse(cp._replace(netloc = validated_netloc))
else:
# Pin to the validated IP to prevent DNS rebinding.
ip_str = f"[{pinned_ip}]" if ":" in pinned_ip else pinned_ip
ip_netloc = f"{ip_str}:{cp.port}" if cp.port else ip_str
request_url = urlunparse(cp._replace(netloc = ip_netloc))
opener = urllib.request.build_opener(
_NoRedirect,
@ -4075,11 +4215,11 @@ def _fetch_url_raw(
headers = {
"User-Agent": ua,
"Host": current_host,
"Host": validated_netloc,
}
if extra_headers:
headers.update(extra_headers)
req = urllib.request.Request(pinned_url, headers = headers)
req = urllib.request.Request(request_url, headers = headers)
try:
# Cap the socket timeout at the time left on the overall deadline
# so a single slow hop cannot outlast the whole fetch budget.

2
studio/backend/core/inference/worker.py
View file

@ -794,7 +794,7 @@ def run_inference_process(
env = os.getenv("ENVIRONMENT_TYPE", "production"),
)
apply_gpu_ids(config.get("resolved_gpu_ids"))
apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend"))
model_name = config["model_name"]

22
studio/backend/core/training/trainer.py
View file

@ -891,6 +891,7 @@ class UnslothTrainer:
use_gradient_checkpointing: str = "unsloth",
use_rslora: bool = False,
use_loftq: bool = False,
use_dora: bool = False,
modules_to_save: list = None,
) -> bool:
"""
@ -993,6 +994,7 @@ class UnslothTrainer:
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
)
# Audio VLM models support VLM-style layer selection
@ -1023,6 +1025,7 @@ class UnslothTrainer:
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
task_type = None,
)
@ -1042,6 +1045,7 @@ class UnslothTrainer:
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
)
@ -1067,6 +1071,7 @@ class UnslothTrainer:
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
modules_to_save = modules_to_save,
)
@ -1087,6 +1092,7 @@ class UnslothTrainer:
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
modules_to_save = modules_to_save,
)
@ -1481,6 +1487,9 @@ class UnslothTrainer:
SNAC_MODEL_NAME = "hubertsiuzdak/snac_24khz"
SNAC_SAMPLE_RATE = 24000
# SNAC codec unvalidated on Intel XPU; keep the pre-PR CPU
# fallback for non-CUDA hosts.
device = "cuda" if torch.cuda.is_available() else "cpu"
max_length = self.max_seq_length or 2048
tokenizer = self.tokenizer
@ -1642,7 +1651,8 @@ class UnslothTrainer:
del snac_model
gc.collect()
torch.cuda.empty_cache()
clear_gpu_cache()
self._cuda_audio_used = True
if not processed_examples:
@ -1669,6 +1679,8 @@ class UnslothTrainer:
import numpy as np
import torchaudio.transforms as T
# Spark-TTS BiCodec unvalidated on Intel XPU; keep the pre-PR CPU
# fallback for non-CUDA hosts.
device = "cuda" if torch.cuda.is_available() else "cpu"
# sparktts lives in the SparkAudio/Spark-TTS GitHub repo, not the HF model
@ -1857,7 +1869,8 @@ class UnslothTrainer:
del audio_tokenizer
gc.collect()
torch.cuda.empty_cache()
clear_gpu_cache()
self._cuda_audio_used = True
if not processed_examples:
@ -1894,6 +1907,8 @@ class UnslothTrainer:
from datasets import Dataset as HFDataset
from utils.paths import ensure_dir, tmp_root
# OuteTTS DAC/Whisper preprocess unvalidated on Intel XPU; keep the
# pre-PR CPU fallback for non-CUDA hosts.
device = "cuda" if torch.cuda.is_available() else "cpu"
# Clone OuteTTS repo (same as audio_codecs._load_dac)
@ -2065,7 +2080,8 @@ class UnslothTrainer:
del prompt_processor
gc.collect()
torch.cuda.empty_cache()
clear_gpu_cache()
self._cuda_audio_used = True
if not processed_examples:

9
studio/backend/core/training/training.py
View file

@ -30,7 +30,7 @@ from typing import Optional, Tuple, Any, Callable, Union, TYPE_CHECKING
if TYPE_CHECKING:
import matplotlib.pyplot as plt
from utils.hardware import prepare_gpu_selection
from utils.hardware import get_device, prepare_gpu_selection
from utils.native_path_leases import (
native_path_secret_removed_for_child_start,
run_without_native_path_secret,
@ -196,6 +196,7 @@ def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]:
"gradient_checkpointing": values.get("gradient_checkpointing", "unsloth"),
"use_rslora": values.get("use_rslora", False),
"use_loftq": values.get("use_loftq", False),
"use_dora": values.get("use_dora", False),
"train_on_completions": values.get("train_on_completions", False),
"finetune_vision_layers": values.get("finetune_vision_layers", True),
"finetune_language_layers": values.get("finetune_language_layers", True),
@ -219,6 +220,9 @@ def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]:
config[key] = values.get(key)
if config["training_type"] == "Full Finetuning":
config["load_in_4bit"] = False
# The parent's detected backend: the worker's apply_gpu_ids() targets the
# right visibility env var from this, without probing torch pre-mask.
config["device_backend"] = get_device().value
return config
@ -452,6 +456,7 @@ class _MLXTrainerAdapter:
use_gradient_checkpointing: Union[str, bool] = "unsloth",
use_rslora: bool = False,
use_loftq: bool = False,
use_dora: bool = False,
) -> bool:
self._peft_config = {
"use_lora": bool(use_lora),
@ -462,6 +467,7 @@ class _MLXTrainerAdapter:
"gradient_checkpointing": use_gradient_checkpointing,
"use_rslora": bool(use_rslora),
"use_loftq": bool(use_loftq),
"use_dora": bool(use_dora),
"finetune_vision_layers": bool(finetune_vision_layers),
"finetune_language_layers": bool(finetune_language_layers),
"finetune_attention_modules": bool(finetune_attention_modules),
@ -569,6 +575,7 @@ class _MLXTrainerAdapter:
"gradient_checkpointing": "unsloth",
"use_rslora": False,
"use_loftq": False,
"use_dora": False,
"finetune_vision_layers": True,
"finetune_language_layers": True,
"finetune_attention_modules": True,

26
studio/backend/core/training/worker.py
View file

@ -764,8 +764,8 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
- ``gcn_arch``: canonical arch string (e.g. ``"gfx1151"``) when a known
attribute is present, else ``""``.
- ``is_unified``: ``True`` for AMD APUs with a shared GPU/system-RAM pool
(gfx1150 Strix Point, gfx1151 Strix Halo) — these need a lower
``set_per_process_memory_fraction`` cap to leave OS headroom.
(gfx1150 Strix Point, gfx1151 Strix Halo, gfx1152 Krackan Point) — these
need a lower ``set_per_process_memory_fraction`` cap to leave OS headroom.
Classification priority:
1. ``props.is_integrated`` truthy (hipDeviceProp_t.integrated -- the
@ -778,6 +778,7 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
- gfx1151 Strix Halo / Gorgon Halo: ``Radeon 8065S`` (Ryzen AI
Max+ 495), ``Radeon 8060S`` (Ryzen AI MAX+
395), ``Radeon 8050S`` (cut-down SKU)
- gfx1152 Krackan Point: ``Radeon 860M``, ``Radeon 840M``
"""
gcn_arch = ""
for _attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):
@ -797,9 +798,13 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
return gcn_arch, True
if gcn_arch:
return gcn_arch, gcn_arch in {"gfx1150", "gfx1151"}
# gfx1152 is Krackan Point, the third RDNA 3.5 APU: same shared
# GPU/system-RAM pool as Strix Point (gfx1150) and Strix Halo (gfx1151).
return gcn_arch, gcn_arch in {"gfx1150", "gfx1151", "gfx1152"}
# Arch attrs absent — fall back to device-name matching.
# Arch attrs absent — fall back to device-name matching. Only reached under
# _hw.IS_ROCM, so the NVIDIA GeForce 840M cannot collide with the Krackan
# markers here.
dev_lower = (getattr(props, "name", "") or "").lower()
is_unified = (
"890m" in dev_lower
@ -807,6 +812,8 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
or "8065s" in dev_lower
or "8060s" in dev_lower
or "8050s" in dev_lower
or "860m" in dev_lower
or "840m" in dev_lower
)
return gcn_arch, is_unified
@ -1547,6 +1554,10 @@ def _run_mlx_training(event_queue, stop_queue, config):
message = "LoftQ is not supported for MLX training yet."
_send("error", error = message)
raise NotImplementedError(message)
if config.get("use_dora"):
message = "DoRA is not supported for MLX training yet."
_send("error", error = message)
raise NotImplementedError(message)
if config.get("is_embedding"):
message = "Embedding model training is not supported for MLX training yet."
_send("error", error = message)
@ -2373,7 +2384,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
env = os.getenv("ENVIRONMENT_TYPE", "production"),
)
apply_gpu_ids(config.get("resolved_gpu_ids"))
apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend"))
model_name = config["model_name"]
@ -2824,7 +2835,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
# On ROCm, exhausting VRAM can hang the HIP driver instead of raising.
# set_per_process_memory_fraction caps the allocator so PyTorch raises
# OutOfMemoryError first (NVIDIA already has a graceful OOM path).
# Unified-memory APUs (gfx1150/gfx1151) share GPU+system RAM, so use 0.80
# Unified-memory APUs (gfx1150/gfx1151/gfx1152) share GPU+system RAM, so use 0.80
# vs 0.90 for discrete. Classify via gcnArchName, else device-name markers.
# Non-fatal: skipped if torch is not importable.
if _hw.IS_ROCM:
@ -3186,6 +3197,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
use_rslora = config.get("use_rslora", False),
use_loftq = config.get("use_loftq", False),
use_dora = config.get("use_dora", False),
)
elif use_lora:
_send_status(event_queue, "Configuring LoRA adapters...")
@ -3202,6 +3214,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
use_rslora = config.get("use_rslora", False),
use_loftq = config.get("use_loftq", False),
use_dora = config.get("use_dora", False),
)
else:
_send_status(event_queue, "Preparing model for full finetuning...")
@ -3630,6 +3643,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->
use_gradient_checkpointing = gradient_checkpointing,
random_state = config.get("random_seed", 3407),
use_rslora = config.get("use_rslora", False),
use_dora = config.get("use_dora", False),
loftq_config = {"loftq_bits": 4, "loftq_iter": 1}
if config.get("use_loftq")
else None,

6
studio/backend/main.py
View file

@ -438,7 +438,11 @@ def _run_llama_cpp_startup_probes(app: FastAPI) -> None:
import structlog as _structlog
_log = _structlog.get_logger(__name__)
if _caps.get("found") and not _caps.get("supports_mtp"):
if (
_caps.get("found")
and not _caps.get("supports_mtp")
and not _caps.get("mtp_probe_inconclusive")
):
_msg = (
"llama.cpp prebuilt lacks MTP support "
"(--spec-type mtp/draft-mtp). Run `unsloth studio update`. "

45
studio/backend/models/inference.py
View file

@ -70,11 +70,23 @@ class LoadRequest(BaseModel):
cache_type_kv: Optional[str] = Field(
None,
description = "KV cache data type for both K and V (e.g. 'f16', 'bf16', 'q8_0', 'q4_1', 'q5_1')",
description = (
"KV cache data type for both K and V "
"(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32')"
),
)
gpu_ids: Optional[List[int]] = Field(
None,
description = "Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries. For GGUF models the picked devices are pinned via CUDA/HIP_VISIBLE_DEVICES.",
description = (
"GPU placement pool, for example [0, 1]. Omit or pass [] to use "
"automatic selection. CUDA/ROCm and Intel XPU values are physical "
"GPU indices; Vulkan values are ggml device ordinals. Explicit "
"physical IDs are unsupported when the parent visibility mask uses "
"non-numeric or subdevice entries, including CUDA_VISIBLE_DEVICES "
"with UUID/MIG entries and ZE_AFFINITY_MASK with subdevice tokens "
"(for example '0.0,0.1') or FLAT-hierarchy tile handles. For GGUF "
"models the fitter may pin the smallest subset of this pool that fits."
),
)
speculative_type: Optional[str] = Field(
None,
@ -433,7 +445,10 @@ class LoadResponse(BaseModel):
)
cache_type_kv: Optional[str] = Field(
None,
description = "KV cache data type for K and V (e.g. 'f16', 'bf16', 'q8_0')",
description = (
"KV cache data type for K and V "
"(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32')"
),
)
chat_template: Optional[str] = Field(
None,
@ -485,7 +500,14 @@ class LoadResponse(BaseModel):
)
gpu_ids: Optional[List[int]] = Field(
None,
description = "Physical GPU indices the model is pinned to, or None for automatic selection.",
description = "Effective GPU indices the model is using after fit-time narrowing, or None for automatic selection.",
)
requested_gpu_ids: Optional[List[int]] = Field(
None,
description = (
"GPU placement pool requested by the user before fit-time narrowing, "
"or None for automatic selection."
),
)
@ -586,7 +608,11 @@ class InferenceStatusResponse(BaseModel):
)
cache_type_kv: Optional[str] = Field(
None,
description = "KV cache quantization dtype (e.g. 'q8_0'), or None for default",
description = (
"KV cache quantization dtype "
"(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32'), "
"or None for default"
),
)
chat_template: Optional[str] = Field(
None, description = "Model's default chat template (Jinja2 source), if any"
@ -649,7 +675,14 @@ class InferenceStatusResponse(BaseModel):
)
gpu_ids: Optional[List[int]] = Field(
None,
description = "Physical GPU indices the model is pinned to, or None for automatic selection.",
description = "Effective GPU indices the model is using after fit-time narrowing, or None for automatic selection.",
)
requested_gpu_ids: Optional[List[int]] = Field(
None,
description = (
"GPU placement pool requested by the user before fit-time narrowing, "
"or None for automatic selection."
),
)
llama_cpp_supports_mtp: bool = Field(
True,

Some files were not shown because too many files have changed in this diff Show more