From 08a985d28abcb4d7186b1000ff15c287bc4cc7d0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 06:07:43 +0000 Subject: [PATCH 001/367] Fix fused LoRA dtype mismatch on fp32 activations into fp16/bf16 weights The fused LoRA ops are decorated with @custom_fwd, which disables autocast. When an fp32 activation (for example the fp32 output of fast_rms_layernorm on an fp32 hidden/residual stream) reaches a fused LoRA path whose base weight is fp16/bf16, nothing downcasts the activation, so torch.matmul raises: expected mat1 and mat2 to have the same dtype: float != c10::Half This is not torch-version specific; torch.matmul never auto-promotes a mixed fp16/fp32 matmul. Reconcile the dtypes at the matmul_lora choke point: after dequantizing W (both the plain and the fast_dequantize branches, not the fp8 branch), downcast X to the base weight compute dtype the way autocast would for a plain Linear, and keep the LoRA A/B cast dtype consistent with it. Keep the backward pass dtype-consistent: LoRA_MLP, LoRA_QKV and LoRA_W now record the original input dtype in ctx.input_dtype, pin the saved activation to the compute dtype, cast the incoming gradients to that dtype, and cast the returned dX back to the original input dtype to honor the autograd grad-dtype contract. Every added branch is a no-op when the dtypes already match, so normal fp16/bf16/fp32 training is unchanged. --- unsloth/kernels/fast_lora.py | 48 +++++++++++++++++++++++++++++++++--- unsloth/kernels/utils.py | 11 +++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index f5d85e8088..265e728862 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -95,6 +95,15 @@ class LoRA_MLP(torch.autograd.Function): h = _forward_function(e, g) i = matmul_lora(h, downW, downW_quant, downA, downB, downS) + # custom_fwd disables autocast, so X may arrive in a different dtype than + # the fused-op compute dtype (e.g. fp32 hidden states from + # fast_rms_layernorm meeting fp16/bf16 base weights). matmul_lora computes + # in the base weight dtype (== e.dtype); keep the saved activation in that + # same dtype so the backward pass stays dtype-consistent. The incoming + # dtype is remembered in ctx.input_dtype and restored on the returned dX. + ctx.input_dtype = dtype + X = X.to(e.dtype) + ctx.custom_saved_tensors = ( gateW, gateW_quant, @@ -134,6 +143,9 @@ class LoRA_MLP(torch.autograd.Function): e = e.view(-1, e.shape[-1]) g = g.view(-1, g.shape[-1]) dtype = X.dtype + # X (and thus dtype) is the compute dtype pinned in forward; align dY too. + if dY.dtype != dtype: + dY = dY.to(dtype) gateA, gateB, upA, upB, downA, downB = ( gateA.to(dtype), @@ -206,8 +218,11 @@ class LoRA_MLP(torch.autograd.Function): # gateW, gateW_quant, gateA, gateB, gateS, # upW, upW_quant, upA, upB, upS, # downW, downW_quant, downA, downB, downS, + dX = dX.view(batch, seq_len, hd) + if dX.dtype != ctx.input_dtype: + dX = dX.to(ctx.input_dtype) return ( - dX.view(batch, seq_len, hd), + dX, None, None, d_gateA.t(), @@ -404,6 +419,12 @@ class LoRA_QKV(torch.autograd.Function): K = K.view(orig_shape[0], orig_shape[1], -1) V = V.view(orig_shape[0], orig_shape[1], -1) + # custom_fwd disables autocast; matmul_lora computed in the base weight + # (compute) dtype == Q.dtype. Pin the saved activation to it so backward + # is dtype-consistent, and remember the incoming dtype for the dX grad. + ctx.input_dtype = dtype + X = X.to(Q.dtype) + ctx.custom_saved_tensors = ( QW, QW_quant, @@ -447,6 +468,13 @@ class LoRA_QKV(torch.autograd.Function): dV = dV.view(-1, dV.shape[-1]) X = X.view(-1, X.shape[-1]) dtype = X.dtype + # X (and thus dtype) is the compute dtype pinned in forward; align grads. + if dQ.dtype != dtype: + dQ = dQ.to(dtype) + if dK.dtype != dtype: + dK = dK.to(dtype) + if dV.dtype != dtype: + dV = dV.to(dtype) QA, QB, KA, KB, VA, VB = ( QA.to(dtype), @@ -519,8 +547,11 @@ class LoRA_QKV(torch.autograd.Function): # QW, QW_quant, QA, QB, QS, # KW, KW_quant, KA, KB, KS, # VW, VW_quant, VA, VB, VS, + dX = dX.view(batch, seq_len, hd) + if dX.dtype != ctx.input_dtype: + dX = dX.to(ctx.input_dtype) return ( - dX.view(batch, seq_len, hd), + dX, None, None, d_QA.t(), @@ -604,6 +635,11 @@ class LoRA_W(torch.autograd.Function): def forward(ctx, X: torch.Tensor, W, W_quant, A, B, S): dtype = X.dtype XW = matmul_lora(X, W, W_quant, A, B, S) + # custom_fwd disables autocast; pin the saved activation to the compute + # dtype (== XW.dtype) so backward is dtype-consistent, and remember the + # incoming dtype for the returned dX grad. + ctx.input_dtype = dtype + X = X.to(XW.dtype) ctx.custom_saved_tensors = ( W, W_quant, @@ -622,6 +658,9 @@ class LoRA_W(torch.autograd.Function): dY = dY.reshape(-1, dY.shape[-1]) # Must be reshape X = X.reshape(-1, X.shape[-1]) # Must be reshape dtype = X.dtype + # X (and thus dtype) is the compute dtype pinned in forward; align dY too. + if dY.dtype != dtype: + dY = dY.to(dtype) A, B = A.to(dtype), B.to(dtype) @@ -647,7 +686,10 @@ class LoRA_W(torch.autograd.Function): dX.addmm_(dY @ B.t(), A.t(), alpha = S) # W, W_quant, A, B, S - return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None + dX = dX.view(batch, seq_len, hd) + if dX.dtype != ctx.input_dtype: + dX = dX.to(ctx.input_dtype) + return dX, None, None, d_A.t(), d_B.t(), None def apply_lora_o(self, X): diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 43ed198a4a..89b610f5d0 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -1091,11 +1091,22 @@ def matmul_lora( W = W.dequantize() else: W = W.contiguous() + # custom_fwd disables autocast, so reconcile the activation dtype to the + # weight (compute) dtype the way autocast would for a plain Linear. This + # covers fp32 hidden states (e.g. fp32 fast_rms_layernorm output) meeting + # fp16/bf16 base weights. + if X.dtype != W.dtype: + X = X.to(W.dtype) + dtype = W.dtype out = torch_matmul(X, W.t(), out = out) elif W.dtype == torch.float8_e4m3fn: out = fp8_linear(X, W, W_quant) else: W = fast_dequantize(W, W_quant, use_global_buffer = True) + # See note above: align the activation dtype to the base weight dtype. + if X.dtype != W.dtype: + X = X.to(W.dtype) + dtype = W.dtype out = torch_matmul(X, W.t(), out = out) if W_quant is not None: del W From 1b825213ea2ffe4774404f27795ad287634f6681 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:16:05 -0700 Subject: [PATCH 002/367] Stabilize floating monitor drag (#6984) * Stabilize floating monitor drag * Restore floating monitor exit animation * Harden Windows Studio smoke checks * Keep API menu badge removed * Apply no-build-tools env overrides in-script The runner does not apply step-level env keys containing parentheses, so ProgramFiles(x86) kept its real value and Find-VsBuildTools still detected VS through vswhere. Set the overrides inside each pwsh step instead; child processes inherit them. The resolver step moves to pwsh because bash cannot export a variable named ProgramFiles(x86). * Reset chat UI session without a second browser context macOS runs Chromium with --single-process, where closing the last context tears down the whole browser, so the shutdown re-login died with TargetClosedError on new_page. Clear cookies and swap pages inside the same context instead, opening the replacement page before closing the old one. * Keep the no-build-tools Path filtered across session refreshes install.ps1's Refresh-SessionPath and setup.ps1's Refresh-Environment rebuild the session Path from the Machine and User registry scopes, so the process-level filter could be undone mid-install and re-expose CMake. Filter those scopes in the Prepare step with normalized dir matching and restore them in cleanup. * Drop stale localStorage auth tokens before re-login Auth tokens live in localStorage, not cookies, and the login guest guard redirects on their mere presence. Remove them during the session reset so the /login navigation is deterministic instead of relying on the tolerated redirect bounce. --- .../studio-windows-inference-smoke.yml | 161 +++++++++---- .../frontend/src/components/app-sidebar.tsx | 17 +- .../src/components/floating-monitor.tsx | 223 ++++++++++-------- studio/frontend/src/features/chat/index.ts | 1 + .../frontend/src/features/settings/index.ts | 1 + studio/frontend/src/i18n/locales/en.ts | 1 - studio/frontend/src/i18n/locales/ja.ts | 1 - studio/frontend/src/i18n/locales/pt-br.ts | 1 - studio/frontend/src/i18n/locales/zh-CN.ts | 1 - tests/studio/playwright_chat_ui.py | 36 ++- 10 files changed, 277 insertions(+), 166 deletions(-) diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index 0bc216d65a..dbb0f9ea6f 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -1334,42 +1334,75 @@ jobs: try { Add-MpPreference -ExclusionPath $p -ErrorAction Stop } catch { } } - - name: Hide Visual Studio + CMake (simulate a host with no build tools) + - name: Prepare no-build-tools simulation shell: pwsh run: | $ErrorActionPreference = 'Stop' - # A Program Files dir can hold a transient handle (Defender / MSBuild node) - # so Rename-Item intermittently fails with "Access is denied"; retry to ride it out. - function Rename-WithRetry($Path, $NewName) { - for ($i = 1; $i -le 6; $i++) { - try { Rename-Item -LiteralPath $Path -NewName $NewName -ErrorAction Stop; return } - catch { if ($i -eq 6) { throw }; Start-Sleep -Seconds 3 } + $root = Join-Path $env:GITHUB_WORKSPACE 'no-build-tools' + $pf = Join-Path $root 'ProgramFiles' + $pfx86 = Join-Path $root 'ProgramFilesx86' + New-Item -ItemType Directory -Force -Path $pf, $pfx86 | Out-Null + + $blocked = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($tool in @('cmake', 'cl.exe')) { + foreach ($cmd in (Get-Command $tool -All -ErrorAction SilentlyContinue)) { + if ($cmd.Source) { + $dir = Split-Path -Parent $cmd.Source + if ($dir) { + [void] $blocked.Add( + [Environment]::ExpandEnvironmentVariables($dir).Trim().Trim('"').TrimEnd('\')) + } + } } } - # Rename the Visual Studio install roots (incl. the Installer that holds - # vswhere.exe) so Find-VsBuildTools' vswhere + filesystem scan both miss. - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - if (Test-Path -LiteralPath $d) { - Rename-WithRetry $d ((Split-Path $d -Leaf) + '.vsoff') - Write-Host "Hid VS: $d" - } + # Normalized comparison so registry spellings (trailing slash, + # unexpanded %VAR%) still match. + function Test-Blocked([string]$p) { + $n = [Environment]::ExpandEnvironmentVariables($p).Trim().Trim('"').TrimEnd('\') + return $blocked.Contains($n) } - # Surgically rename each cmake executable on PATH (not its parent dir -- - # cmake can share a dir with other shims) so Get-Command cmake fails. - $hidden = @() - foreach ($c in (Get-Command cmake -All -ErrorAction SilentlyContinue)) { - if ($c.Source -and (Test-Path -LiteralPath $c.Source)) { - Rename-WithRetry $c.Source ((Split-Path $c.Source -Leaf) + '.off') - $hidden += $c.Source - Write-Host "Hid cmake: $($c.Source)" - } + + $pathParts = $env:Path -split [IO.Path]::PathSeparator | + Where-Object { $_ -and -not (Test-Blocked $_) } + $noBuildToolsPath = $pathParts -join [IO.Path]::PathSeparator + + # install.ps1's Refresh-SessionPath and setup.ps1's Refresh-Environment + # rebuild the session Path from these scopes mid-install, so filter + # them too. Originals are saved for the cleanup step. + foreach ($scope in @('Machine', 'User')) { + $orig = [Environment]::GetEnvironmentVariable('Path', $scope) + if (-not $orig) { continue } + Set-Content -LiteralPath (Join-Path $root "orig-path-$scope.txt") -Value $orig -NoNewline + $kept = ($orig -split ';' | Where-Object { $_ -and -not (Test-Blocked $_) }) -join ';' + [Environment]::SetEnvironmentVariable('Path', $kept, $scope) + Write-Host "Filtered $scope Path scope." + } + + "NO_BUILD_TOOLS_PROGRAMFILES=$pf" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PROGRAMFILES_X86=$pfx86" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PATH<&1 | Tee-Object -FilePath logs/install.log @@ -1480,19 +1517,19 @@ jobs: [ -n "$CONTENT" ] && [ "$CONTENT" != "null" ] || { echo "::error::empty completion"; exit 1; } echo "Inference OK without Visual Studio: $CONTENT" - - name: Restore Visual Studio + CMake + - name: Clean no-build-tools simulation if: always() shell: pwsh run: | - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - $off = "$d.vsoff" - if (Test-Path -LiteralPath $off) { Rename-Item -LiteralPath $off -NewName (Split-Path $d -Leaf); Write-Host "Restored $d" } - } - if ($env:HIDDEN_CMAKE) { - foreach ($src in ($env:HIDDEN_CMAKE -split '\|')) { - if ($src -and (Test-Path -LiteralPath "$src.off")) { Rename-Item -LiteralPath "$src.off" -NewName (Split-Path $src -Leaf) } + $root = Join-Path $env:GITHUB_WORKSPACE 'no-build-tools' + foreach ($scope in @('Machine', 'User')) { + $saved = Join-Path $root "orig-path-$scope.txt" + if (Test-Path -LiteralPath $saved) { + [Environment]::SetEnvironmentVariable('Path', (Get-Content -LiteralPath $saved -Raw), $scope) + Write-Host "Restored $scope Path scope." } } + Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue - name: Stop Studio if: always() @@ -1540,21 +1577,34 @@ jobs: with: python-version: '3.12' - - name: Hide Visual Studio + - name: Prepare no-build-tools simulation shell: pwsh run: | $ErrorActionPreference = 'Stop' - # Retry the rename: a Program Files dir can hold a transient handle that - # makes Rename-Item intermittently fail with "Access is denied". - function Rename-WithRetry($Path, $NewName) { - for ($i = 1; $i -le 6; $i++) { - try { Rename-Item -LiteralPath $Path -NewName $NewName -ErrorAction Stop; return } - catch { if ($i -eq 6) { throw }; Start-Sleep -Seconds 3 } + $root = Join-Path $env:GITHUB_WORKSPACE 'no-build-tools' + $pf = Join-Path $root 'ProgramFiles' + $pfx86 = Join-Path $root 'ProgramFilesx86' + New-Item -ItemType Directory -Force -Path $pf, $pfx86 | Out-Null + + $blocked = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($tool in @('cmake', 'cl.exe')) { + foreach ($cmd in (Get-Command $tool -All -ErrorAction SilentlyContinue)) { + if ($cmd.Source) { + $dir = Split-Path -Parent $cmd.Source + if ($dir) { [void] $blocked.Add($dir) } + } } } - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - if (Test-Path -LiteralPath $d) { Rename-WithRetry $d ((Split-Path $d -Leaf) + '.vsoff'); Write-Host "Hid VS: $d" } - } + + $pathParts = $env:Path -split [IO.Path]::PathSeparator | + Where-Object { $_ -and -not $blocked.Contains($_) } + $noBuildToolsPath = $pathParts -join [IO.Path]::PathSeparator + + "NO_BUILD_TOOLS_PROGRAMFILES=$pf" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PROGRAMFILES_X86=$pfx86" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PATH< /tmp/resolve.json || { - echo "::error::resolver exited non-zero"; cat /tmp/resolve.json || true; exit 1; } - cat /tmp/resolve.json - echo "Prebuilt resolver ran with no Visual Studio present." + if ($LASTEXITCODE -ne 0) { Write-Host "::error::pip install huggingface_hub failed"; exit 1 } + python studio/install_llama_prebuilt.py --resolve-prebuilt latest --output-format json > resolve.json + if ($LASTEXITCODE -ne 0) { + Write-Host "::error::resolver exited non-zero" + if (Test-Path resolve.json) { Get-Content resolve.json } + exit 1 + } + Get-Content resolve.json + Write-Host "Prebuilt resolver ran with no Visual Studio present." - - name: Restore Visual Studio + - name: Clean no-build-tools simulation if: always() shell: pwsh run: | - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - $off = "$d.vsoff" - if (Test-Path -LiteralPath $off) { Rename-Item -LiteralPath $off -NewName (Split-Path $d -Leaf); Write-Host "Restored $d" } - } + Remove-Item -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'no-build-tools') -Recurse -Force -ErrorAction SilentlyContinue # ── folded from studio-setup-ps1-vs2026.yml: setup.ps1 unit tests + real-VS detection + vcredist ── pester: diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index 823e420869..1a74b38524 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -81,7 +81,6 @@ import { TestTube01Icon, ZapIcon, } from "@hugeicons/core-free-icons"; -import { listStoredChatThreads } from "@/features/chat/utils/chat-history-storage"; import { Tooltip, TooltipContent, @@ -97,6 +96,7 @@ import { createChatProject, deleteChatProject, deleteChatItem, + listStoredChatThreads, moveChatItemToProject, renameChatItem, renameChatProject, @@ -582,7 +582,14 @@ export function AppSidebar() { useEffect(() => { if (!pendingRename) return; const match = allChatItems.find((i) => i.id === pendingRename.id); - if (match && match.title === pendingRename.title) setPendingRename(null); + if (!match || match.title !== pendingRename.title) return; + queueMicrotask(() => { + setPendingRename((current) => + current?.id === pendingRename.id && current.title === pendingRename.title + ? null + : current, + ); + }); }, [allChatItems, pendingRename]); const [creatingProject, setCreatingProject] = useState(false); const [projectNameDraft, setProjectNameDraft] = useState(""); @@ -680,12 +687,6 @@ export function AppSidebar() { useState(null); const [deleteProjectFiles, setDeleteProjectFiles] = useState(false); - useEffect(() => { - if (confirmingDelete?.kind !== "project") { - setDeleteProjectFiles(false); - } - }, [confirmingDelete]); - async function commitDelete() { const target = confirmingDelete; if (!target) return; diff --git a/studio/frontend/src/components/floating-monitor.tsx b/studio/frontend/src/components/floating-monitor.tsx index bce4bf2831..0a51875de9 100644 --- a/studio/frontend/src/components/floating-monitor.tsx +++ b/studio/frontend/src/components/floating-monitor.tsx @@ -3,27 +3,35 @@ import { Button } from "@/components/ui/button"; import { Progress } from "@/components/ui/progress"; -import { useMonitorOverlayStore } from "@/features/settings/stores/monitor-overlay-store"; +import { useMonitorOverlayStore } from "@/features/settings"; import { useSystemInfo } from "@/hooks/use-system"; import { useT } from "@/i18n"; import { cn } from "@/lib/utils"; import { CpuIcon, GripVerticalIcon, XIcon } from "lucide-react"; -import { motion } from "motion/react"; -import { useRef } from "react"; +import { AnimatePresence, motion, useDragControls } from "motion/react"; +import { type PointerEvent, useMemo, useState } from "react"; function clampPercent(value: number): number { return Math.max(0, Math.min(100, value)); } function usageIndicatorClass(percent: number): string { - if (percent >= 90) return "bg-destructive"; - if (percent >= 70) return "bg-amber-500"; + if (percent >= 90) { + return "bg-destructive"; + } + if (percent >= 70) { + return "bg-amber-500"; + } return "bg-primary"; } function usageTextClass(percent: number): string { - if (percent >= 90) return "text-destructive"; - if (percent >= 70) return "text-amber-600 dark:text-amber-400"; + if (percent >= 90) { + return "text-destructive"; + } + if (percent >= 70) { + return "text-amber-600 dark:text-amber-400"; + } return "text-primary"; } @@ -39,9 +47,18 @@ export function FloatingMonitor() { const { isOpen, setIsOpen } = useMonitorOverlayStore(); const systemInfo = useSystemInfo({ enabled: isOpen, pollMs: 5000 }); - const constraintsRef = useRef(null); + const [constraintsElement, setConstraintsElement] = + useState(null); + const constraintsRef = useMemo( + () => ({ current: constraintsElement }), + [constraintsElement], + ); + const dragControls = useDragControls(); - if (!isOpen) return null; + function startDrag(event: PointerEvent) { + event.preventDefault(); + dragControls.start(event); + } const ramTotal = systemInfo.memory?.total_gb ?? 0; const ramAvailable = systemInfo.memory?.available_gb ?? 0; @@ -64,99 +81,109 @@ export function FloatingMonitor() { const hasGpu = (systemInfo.gpu?.available ?? false) && devices.length > 0; return ( -
- -
-
- - - {t("settings.resources.liveMonitor.title")} - -
-
-
- -
- - -
-
- - + {isOpen && ( +
-
-
- {t("settings.resources.liveMonitor.ram")} - - {Math.round(ramPercent)}% - -
-
- {formatGiB(ramUsed)} / {formatGiB(ramTotal)} -
- -
- - {hasGpu && ( -
-
- - {t("settings.resources.liveMonitor.vram")}{" "} - {devices.length > 1 - ? `(${devices.length} GPUs)` - : `(${devices[0].name ?? "GPU"})`} + +
+
+ + + {t("settings.resources.liveMonitor.title")} - +
+
- {Math.round(vramPercent)}% - + +
+ +
-
- {formatGiB(vramUsed)} / {formatGiB(vramTotal)} -
-
- )} - - -
+ + +
+
+ {t("settings.resources.liveMonitor.ram")} + + {Math.round(ramPercent)}% + +
+
+ {formatGiB(ramUsed)} / {formatGiB(ramTotal)} +
+ +
+ + {hasGpu && ( +
+
+ + {t("settings.resources.liveMonitor.vram")}{" "} + {devices.length > 1 + ? `(${devices.length} GPUs)` + : `(${devices[0].name ?? "GPU"})`} + + + {Math.round(vramPercent)}% + +
+
+ {formatGiB(vramUsed)} / {formatGiB(vramTotal)} +
+ +
+ )} + + +
+ )} + ); } diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts index 7cd9611c71..d070ed15de 100644 --- a/studio/frontend/src/features/chat/index.ts +++ b/studio/frontend/src/features/chat/index.ts @@ -36,6 +36,7 @@ export { ChatSearchDialog } from "./components/chat-search-dialog"; export { setTrainingCompareHandoff } from "./lib/training-compare-handoff"; export type { ProjectRecord } from "./types"; export { clearAllChats, countAllChats } from "./utils/clear-all-chats"; +export { listStoredChatThreads } from "./utils/chat-history-storage"; export { ArtifactCard } from "./artifacts/artifact-card"; export { useChatArtifactsStore, diff --git a/studio/frontend/src/features/settings/index.ts b/studio/frontend/src/features/settings/index.ts index 364ca1611f..3fefd8c63a 100644 --- a/studio/frontend/src/features/settings/index.ts +++ b/studio/frontend/src/features/settings/index.ts @@ -7,6 +7,7 @@ export { savePersonalization, } from "./api/personalization"; export { setTheme, useTheme } from "./stores/theme-store"; +export { useMonitorOverlayStore } from "./stores/monitor-overlay-store"; export type { Personalization, PersonalizationAppearance, diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index a9b5d839b3..e0cb8030ae 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -409,7 +409,6 @@ export const en = { description: "Access Unsloth via the OpenAI-compatible API.", readDocs: "Read the API docs", noAccess: "No API access yet.", - newBadge: "New", accessTokens: "Access tokens", loadError: "Couldn't load API access.", createError: "Couldn't create access token.", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index f2020f3cfb..08b8d4f4d3 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -296,7 +296,6 @@ export const ja = { description: "OpenAI互換 API を介して Unsloth にアクセスします。", readDocs: "API ドキュメントを読む", noAccess: "まだ API アクセス権がありません。", - newBadge: "新規", accessTokens: "アクセストークン", loadError: "API アクセス権を読み込めませんでした。", createError: "アクセストークンを作成できませんでした。", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index c261e4ed0c..494a98ec32 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -363,7 +363,6 @@ export const ptBR = { "Acesse o Unsloth por meio da API compatível com OpenAI.", readDocs: "Leia a documentação da API", noAccess: "Nenhum acesso à API ainda.", - newBadge: "Novo", accessTokens: "Tokens de acesso", loadError: "Não foi possível carregar o acesso à API.", createError: "Não foi possível criar o token de acesso.", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index f6dc265fc7..dda8e017ab 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -267,7 +267,6 @@ export const zhCN = { description: "通过兼容 OpenAI 的 API 以编程方式访问 Unsloth。", readDocs: "阅读 API 文档", noAccess: "还没有 API 访问权限。", - newBadge: "新", accessTokens: "访问 token", loadError: "无法加载 API 访问权限。", createError: "无法创建访问 token。", diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index 71297f9043..0698d8e0d2 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -1264,11 +1264,37 @@ with sync_playwright() as p: # placeholder, and /api/health goes unreachable shortly after. # ───────────────────────────────────────────────────── step("Shutdown via account menu") - # Re-login with NEW2 for a valid /api/shutdown token (CLI rotation - # invalidated the old one). The stale token can make the SPA auth guard - # abort this goto with ERR_ABORTED, or redirect to the same /login URL - # ("interrupted by another navigation"); resolve on domcontentloaded and - # tolerate either -- the pw-field wait below confirms we are on /login. + # Start fresh after the CLI rotation invalidates this browser session. + # Stay in the SAME context: macOS Chromium runs --single-process, where + # closing the last context kills the browser and a second context cannot + # be created. Open the new page before closing the old one; the context + # init script covers the new page. + try: + ctx.clear_cookies() + except Exception as exc: + info(f"WARN clearing stale session cookies failed: {exc!r}") + # Auth tokens live in localStorage, and /login's guest guard redirects on + # their mere presence, so drop them before navigating. + try: + page.evaluate( + "['unsloth_auth_token', 'unsloth_auth_refresh_token']" + ".forEach((key) => localStorage.removeItem(key))" + ) + except Exception as exc: + info(f"WARN clearing stale auth tokens failed: {exc!r}") + _fresh_page = ctx.new_page() + _fresh_page.set_default_timeout(60_000) + _fresh_page.on("pageerror", lambda e: page_errors.append(str(e))) + _fresh_page.on("console", _on_console) + try: + page.close() + except Exception: + pass + page = _fresh_page + + # Re-login with NEW2 for a valid /api/shutdown token. Route changes can + # still abort or interrupt this navigation, so the field wait below is the + # final confirmation that we reached /login. _tolerated_nav = ("ERR_ABORTED", "interrupted by another navigation") try: page.goto(f"{BASE}/login", wait_until = "domcontentloaded", timeout = 60_000) From f0705ef6e5148cd600e5f0d5d1197705748f9f02 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 9 Jul 2026 07:47:23 +0000 Subject: [PATCH 003/367] Tighten fused-LoRA dtype-fix comments --- unsloth/kernels/fast_lora.py | 10 ++++------ unsloth/kernels/utils.py | 5 ++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 265e728862..027dd7b8e9 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -95,12 +95,10 @@ class LoRA_MLP(torch.autograd.Function): h = _forward_function(e, g) i = matmul_lora(h, downW, downW_quant, downA, downB, downS) - # custom_fwd disables autocast, so X may arrive in a different dtype than - # the fused-op compute dtype (e.g. fp32 hidden states from - # fast_rms_layernorm meeting fp16/bf16 base weights). matmul_lora computes - # in the base weight dtype (== e.dtype); keep the saved activation in that - # same dtype so the backward pass stays dtype-consistent. The incoming - # dtype is remembered in ctx.input_dtype and restored on the returned dX. + # custom_fwd disables autocast, so X may mismatch the compute dtype (e.g. + # fp32 fast_rms_layernorm output meeting fp16/bf16 base weights). Pin the + # saved activation to the compute dtype (== e.dtype) so backward stays + # consistent; remember the incoming dtype to restore it on the dX grad. ctx.input_dtype = dtype X = X.to(e.dtype) diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 89b610f5d0..5bc5ab9a74 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -1092,9 +1092,8 @@ def matmul_lora( else: W = W.contiguous() # custom_fwd disables autocast, so reconcile the activation dtype to the - # weight (compute) dtype the way autocast would for a plain Linear. This - # covers fp32 hidden states (e.g. fp32 fast_rms_layernorm output) meeting - # fp16/bf16 base weights. + # weight (compute) dtype as autocast would (e.g. fp32 fast_rms_layernorm + # output meeting fp16/bf16 base weights). if X.dtype != W.dtype: X = X.to(W.dtype) dtype = W.dtype From 8205d4c0819088a3c864fdd505ce1c1e6d72852d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 01:46:14 -0700 Subject: [PATCH 004/367] Retry the Studio UI shutdown re-login on transient goto timeout (#7027) * Retry the Studio UI shutdown re-login on transient goto timeout The Chat UI Playwright smoke intermittently failed at the pre-shutdown re-login: page.goto('/login') can hit a 60s TimeoutError on a slow runner even while the server is healthy, and the surrounding except only tolerated ERR_ABORTED / interrupted-navigation, so a plain timeout hard-failed the job. Wrap the re-login goto/wait/fill/submit in the same 3-attempt retry the change-password step already uses (recover_or_replace_page between tries, per-attempt fail screenshots, wait_for_health pre-gate). The composer wait stays outside the loop so a retry never re-navigates after login has set tokens (which would redirect to /chat via the guest guard); it remains the authoritative confirmation, so a genuinely broken login still fails. * Catch transient login-request failures and preserve error listeners on recovery Wait on the /api/auth/login POST inside the retry (via click_and_wait_for_response) so a transient 4xx/5xx is retried in-loop instead of surfacing only at the out-of-loop composer wait, matching the change-password step. When recover_or_replace_page swaps in a fresh page, re-attach the pageerror/console listeners so error tracking survives the replacement. --- tests/studio/playwright_chat_ui.py | 96 ++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 10 deletions(-) diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index 0698d8e0d2..6a88b98c19 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -1296,16 +1296,92 @@ with sync_playwright() as p: # still abort or interrupt this navigation, so the field wait below is the # final confirmation that we reached /login. _tolerated_nav = ("ERR_ABORTED", "interrupted by another navigation") - try: - page.goto(f"{BASE}/login", wait_until = "domcontentloaded", timeout = 60_000) - except Exception as exc: - if not any(t in str(exc) for t in _tolerated_nav): - raise - info(f"goto /login interrupted ({exc!r}); password-field wait will confirm /login") - pw_field = page.locator("#password") - pw_field.wait_for(state = "visible", timeout = 60_000) - pw_field.fill(NEW2) - page.locator('button[type="submit"]').click() + # A slow CI runner can make this re-login navigation time out even with the + # server healthy, so retry the whole goto/wait/fill/submit sequence (mirrors + # the change-password retry above). wait_for_health is a diagnostic pre-gate. + wait_for_health(BASE, timeout = 30.0, info = info) + relogin_err: Exception | None = None + for _relogin_attempt in range(3): + try: + try: + page.goto(f"{BASE}/login", wait_until = "domcontentloaded", timeout = 60_000) + except Exception as exc: + if not any(t in str(exc) for t in _tolerated_nav): + raise + info(f"goto /login interrupted ({exc!r}); password-field wait will confirm /login") + pw_field = page.locator("#password") + pw_field.wait_for(state = "visible", timeout = 60_000) + pw_field.fill(NEW2) + # Wait on the login POST so a transient 4xx/5xx is caught and retried + # here, not swallowed until the out-of-loop composer wait. + status, _ = click_and_wait_for_response( + page, + url_substr = "/api/auth/login", + method = "POST", + do_click = lambda: page.locator('button[type="submit"]').click(), + timeout_ms = 30_000, + info = lambda m: print(f"[ui] {m}", flush = True), + ) + if status is not None and status >= 400: + raise AssertionError( + f"login POST returned {status}; see console_errors={console_errors[:1]!r}" + ) + relogin_err = None + break + except Exception as e: + relogin_err = e + try: + cur_url = page.url + except Exception: + cur_url = "" + print( + f"[ui] re-login attempt {_relogin_attempt + 1} failed: " + f"{type(e).__name__}: {str(e)[:200]}; page.url={cur_url}; " + f"page_errors={len(page_errors)} console_errors={len(console_errors)}", + flush = True, + ) + if console_errors: + print( + f"[ui] first console.error: {console_errors[0][:200]!r}", + flush = True, + ) + if page_errors: + print(f"[ui] first pageerror: {page_errors[0][:200]!r}", flush = True) + try: + shoot(f"18-relogin-attempt-{_relogin_attempt + 1}-fail") + except Exception: + pass + if _relogin_attempt < 2: + # ERR_NO_BUFFER_SPACE needs the OS to recover socket + # buffers; back off 5s then 15s before retrying. + if "ERR_NO_BUFFER_SPACE" in str(e): + backoff_s = 5 if _relogin_attempt == 0 else 15 + print( + f"[ui] ENOBUFS detected; sleeping {backoff_s}s " + f"before retry to let OS recover socket buffers...", + flush = True, + ) + time.sleep(backoff_s) + # Replace the page if it died; otherwise next iteration's + # page.goto() handles the reload. + old_page = page + page = recover_or_replace_page( + page, + ctx, + default_timeout_ms = 60_000, + info = lambda m: print(f"[ui] recovery: {m}", flush = True), + ) + # A freshly created replacement page loses the pageerror/console + # listeners; re-attach so error tracking survives recovery. + if page is not old_page: + page.on("pageerror", lambda e: page_errors.append(str(e))) + page.on("console", _on_console) + if relogin_err is not None: + raise relogin_err + # Composer mount confirms the rotated session is authenticated. Kept OUTSIDE the + # retry: the loop breaks right after submit, so we never re-goto /login once login + # has set tokens -- that would hit the guest guard, redirect to /chat, and make a + # merely-slow composer look like a broken login. composer = page.locator('textarea[aria-label="Message input"]') composer.wait_for(state = "visible", timeout = 60_000) shoot("18-relogin-with-NEW2") From 5e43c623b98affc23efbf9dbe71061de7c1706a2 Mon Sep 17 00:00:00 2001 From: Etherl <61019402+Etherll@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:46:22 +0300 Subject: [PATCH 005/367] Fix FastSentenceTransformer Qwen embedding preprocessing (#6939) * Fix FastSentenceTransformer Qwen embedding preprocessing * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Document Transformer.load embedding modality fix for #6881 * Harden #6881 fix and add forwards/backwards-compatible regression tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fall back to Transformer constructor on legacy sentence-transformers without Hub-capable load * Mirror legacy sentence-transformers fallback in embedding-parity tripwire test * Tighten #6881 comments and docstrings * Skip embedding-parity test on CPU-only runners since FastSentenceTransformer requires CUDA * Honor the transformer module's saved subfolder when loading modules.json records a path for the Transformer module (root for decoder embedders like Qwen3-Embedding, 0_Transformer for the classic layout). Pooling/Normalize already load from their saved path; thread the same path into Transformer.load as subfolder so config and tokenizer resolve like stock ST. stays a no-op, so single-module models are unchanged. * Make embedding-parity test bf16-aware fp16 overflows to NaN on bf16-native embedders such as EmbeddingGemma (Gemma3), producing a false parity failure. Prefer bf16 when the GPU supports it so the tripwire can guard the full documented embedding matrix (Qwen3-Embedding, EmbeddingGemma, BGE-M3, all-MiniLM, GTE-ModernBERT), not just fp16-safe models. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: danielhanchen --- ...t_sentence_transformer_embedding_parity.py | 122 ++++++++++++++++++ ...st_sentence_transformers_pinned_symbols.py | 38 ++++++ unsloth/models/sentence_transformer.py | 65 +++++++++- 3 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 tests/python/test_fast_sentence_transformer_embedding_parity.py diff --git a/tests/python/test_fast_sentence_transformer_embedding_parity.py b/tests/python/test_fast_sentence_transformer_embedding_parity.py new file mode 100644 index 0000000000..252d4486a5 --- /dev/null +++ b/tests/python/test_fast_sentence_transformer_embedding_parity.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. +"""Regression guard for issue #6881: FastSentenceTransformer must preprocess text +like a stock SentenceTransformer for decoder embedding models. ST 5.x infers a +"message" modality for chat-template models (e.g. Qwen/Qwen3-Embedding), so building +via `Transformer(model_name, ...)` chat-wraps inputs and degrades embeddings; +`_create_transformer_module` uses `Transformer.load(...)` instead. + +Layers: test_transformer_load_signature_supports_unsloth_kwargs (fast, runs when ST +is importable) and test_fast_sentence_transformer_matches_stock_st (end-to-end parity, +opt-in via UNSLOTH_EMBEDDING_PARITY_MODEL so default CI is unaffected). +""" + +from __future__ import annotations + +import inspect +import os + +import pytest + + +def test_transformer_load_signature_supports_unsloth_kwargs(): + """Forwards-compat tripwire: a Hub-capable Transformer.load must accept the kwargs + the #6881 fix passes. Legacy ST 3.x/4.x expose load(input_path); the code falls back + to Transformer(...) there, so mirror that gate and skip.""" + models = pytest.importorskip("sentence_transformers.models") + load = getattr(models.Transformer, "load", None) + assert callable(load), ( + "sentence_transformers Transformer.load is missing; the #6881 fix in " + "unsloth.models.sentence_transformer._create_transformer_module depends on it." + ) + params = inspect.signature(load).parameters + accepts_var_kw = any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()) + # Mirror _create_transformer_module's hub_capable gate. + hub_capable = accepts_var_kw or any(k in params for k in ("token", "cache_folder", "revision")) + if not hub_capable: + pytest.skip( + "legacy Transformer.load(input_path); production path falls back to Transformer(...)" + ) + unsupported = [ + k + for k in ("token", "cache_folder", "revision", "trust_remote_code") + if not (accepts_var_kw or k in params) + ] + assert not unsupported, ( + f"installed sentence_transformers Transformer.load no longer accepts {unsupported} " + f"and has no **kwargs; update _create_transformer_module (#6881) before it silently " + f"falls back to Transformer(...)." + ) + + +def _probe_texts(): + return [ + "roasted chickpeas in 20 kg bags", + "The capital of France is Paris.", + "A fast brown fox jumps over the lazy dog.", + "recette de tarte aux pommes traditionnelle", + ] + + +def test_fast_sentence_transformer_matches_stock_st(): + """End-to-end: FastSentenceTransformer embeddings and tokenization must match a + stock SentenceTransformer load of the same checkpoint. Opt-in (needs a model) and + GPU-only (FastSentenceTransformer requires CUDA), so it skips on CPU-only runners.""" + model_id = os.environ.get("UNSLOTH_EMBEDDING_PARITY_MODEL") + if not model_id: + pytest.skip( + "set UNSLOTH_EMBEDDING_PARITY_MODEL to a chat-template embedding model " + "(HF id or local path) to run the #6881 parity test" + ) + + torch = pytest.importorskip("torch") + if not torch.cuda.is_available(): + pytest.skip("FastSentenceTransformer requires CUDA; skipping on CPU-only runner") + np = pytest.importorskip("numpy") + pytest.importorskip("sentence_transformers") + from sentence_transformers import SentenceTransformer + + device = "cuda" + # Prefer bf16 when the GPU supports it: fp16 overflows to NaN on bf16-native + # embedders such as EmbeddingGemma (Gemma3), which would mask real parity. + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + texts = _probe_texts() + max_seq_length = 256 + + # Control FIRST, before importing unsloth, so its global import patches never + # touch the stock reference (mirrors the issue's "restart runtime" repro). + ctrl = SentenceTransformer(model_id, device = device, model_kwargs = {"torch_dtype": dtype}) + ctrl.max_seq_length = max_seq_length + ctrl_ids = ctrl.tokenize([texts[0]])["input_ids"][0].tolist() + ctrl_emb = np.asarray( + ctrl.encode(texts, normalize_embeddings = True, batch_size = 8), dtype = np.float32 + ) + + import unsloth # noqa: F401 + from unsloth import FastSentenceTransformer + + fast = FastSentenceTransformer.from_pretrained( + model_id, + max_seq_length = max_seq_length, + dtype = dtype, + load_in_4bit = False, + load_in_16bit = True, + ) + fast_ids = fast.tokenize([texts[0]])["input_ids"][0].tolist() + fast_emb = np.asarray( + fast.encode(texts, normalize_embeddings = True, batch_size = 8), dtype = np.float32 + ) + + # Identical tokenization = no chat-template wrapping slipped in (the #6881 defect). + assert fast_ids == ctrl_ids, ( + f"tokenization diverged (chat-template wrapping regressed?):\n" + f" stock: {ctrl_ids}\n fast: {fast_ids}" + ) + + cos = (ctrl_emb * fast_emb).sum(1) / ( + np.linalg.norm(ctrl_emb, axis = 1) * np.linalg.norm(fast_emb, axis = 1) + ) + assert float(cos.min()) > 0.99, ( + f"embedding parity regressed: min cosine {float(cos.min()):.5f} <= 0.99 " + f"(per-text {[round(float(c), 5) for c in cos]})" + ) diff --git a/tests/version_compat/test_sentence_transformers_pinned_symbols.py b/tests/version_compat/test_sentence_transformers_pinned_symbols.py index c0c35b9d5d..d7f9a54811 100644 --- a/tests/version_compat/test_sentence_transformers_pinned_symbols.py +++ b/tests/version_compat/test_sentence_transformers_pinned_symbols.py @@ -19,6 +19,8 @@ ST_TAGS = [ "v5.2.3", "v5.3.0", "v5.4.1", + "v5.5.1", + "v5.6.0", "master", ] @@ -120,6 +122,42 @@ def test_st_transformer_base_class_either_path(tag: str): ) +# Transformer.load classmethod: unsloth builds saved-ST modules through it (#6881). +@pytest.mark.parametrize("tag", ST_TAGS) +def test_st_transformer_load_accepts_unsloth_kwargs(tag: str): + """unsloth builds saved ST models via Transformer.load(...) so the saved + modality_config is honored (#6881). If .load stops accepting the hub kwargs it + passes (and has no **kwargs), update the fix before it silently regresses. Not + locating .load is a SKIP (may be inherited); the live test guards the install.""" + candidates = [ + "sentence_transformers/models/Transformer.py", + "sentence_transformers/models/transformer.py", + "sentence_transformers/base/modules/transformer.py", + "sentence_transformers/base/modules/module.py", + ] + for p in candidates: + src = fetch_text("UKPLab/sentence-transformers", tag, p) + if src is None or not has_def(src, "load", "func"): + continue + m = re.search(r"def\s+load\s*\((.*?)\)\s*(?:->[^:]*)?:", src, re.S) + if m is None: + continue + sig = m.group(1) + accepts_var_kw = "**" in sig + missing = [ + kw + for kw in ("token", "cache_folder", "revision", "trust_remote_code") + if not (accepts_var_kw or re.search(rf"\b{re.escape(kw)}\b", sig)) + ] + assert not missing, ( + f"{tag}: Transformer.load in {p} no longer accepts {missing} and has no " + f"**kwargs; update unsloth.models.sentence_transformer._create_transformer_module " + f"(#6881) before it silently falls back to Transformer(...)." + ) + return + pytest.skip(f"{tag}: Transformer.load not locatable in {candidates} (may be inherited)") + + # sentence_transformers.util: import_from_string + load_dir_path helpers unsloth calls. @pytest.mark.parametrize("tag", ST_TAGS) def test_st_util_helpers(tag: str): diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py index c1172faa94..4a1a555bcf 100644 --- a/unsloth/models/sentence_transformer.py +++ b/unsloth/models/sentence_transformer.py @@ -990,7 +990,17 @@ class FastSentenceTransformer(FastModel): return None @staticmethod - def _create_transformer_module(model_name, model, tokenizer, max_seq_length, trust_remote_code): + def _create_transformer_module( + model_name, + model, + tokenizer, + max_seq_length, + trust_remote_code, + token = None, + cache_dir = None, + revision = None, + module_subfolder = "", + ): """Helper to create and configure a Transformer module.""" from sentence_transformers.models import Transformer @@ -1077,7 +1087,45 @@ class FastSentenceTransformer(FastModel): elif "tokenizer_args" in transformer_init_params: transformer_kwargs["tokenizer_args"] = trust_remote_code_kwargs.copy() - transformer_module = Transformer(model_name, **transformer_kwargs) + # Build via Transformer.load so the saved modality_config is honored: plain + # Transformer(...) makes ST 5.x infer a "message" modality for chat-template + # models (e.g. Qwen3-Embedding), chat-wrapping inputs and degrading embeddings + # (#6881). Only use .load when it resolves a Hub id (accepts the kwargs or + # **kwargs); legacy ST 3.x/4.x load(input_path) is local-only with no modality + # bug, so fall back to the constructor. + transformer_module = None + transformer_load = getattr(Transformer, "load", None) + has_modules_json = ( + FastSentenceTransformer._module_path( + model_name, token, cache_dir = cache_dir, revision = revision + ) + is not None + ) + if callable(transformer_load) and has_modules_json: + load_params = inspect.signature(transformer_load).parameters + accepts_var_kw = any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in load_params.values() + ) + hub_capable = accepts_var_kw or any( + key in load_params for key in ("token", "cache_folder", "revision") + ) + if hub_capable: + load_kwargs = { + "token": token, + "cache_folder": cache_dir, + "revision": revision, + "trust_remote_code": trust_remote_code, + **transformer_kwargs, + } + # Resolve config/tokenizer from the module's saved subfolder + # (modules.json "path"), like stock ST; "" (root) is a no-op. + if module_subfolder: + load_kwargs["subfolder"] = module_subfolder + if not accepts_var_kw: + load_kwargs = {k: v for k, v in load_kwargs.items() if k in load_params} + transformer_module = Transformer.load(model_name, **load_kwargs) + if transformer_module is None: + transformer_module = Transformer(model_name, **transformer_kwargs) finally: # Restore original Auto* loading immediately AutoModel.from_pretrained = original_model_from_pretrained @@ -1191,6 +1239,10 @@ class FastSentenceTransformer(FastModel): tokenizer, max_seq_length, trust_remote_code, + token, + cache_dir, + revision, + module_subfolder = module_config.get("path") or "", ) modules[name] = transformer_module else: @@ -1226,7 +1278,14 @@ class FastSentenceTransformer(FastModel): ) transformer_module = FastSentenceTransformer._create_transformer_module( - model_name, model, tokenizer, max_seq_length, trust_remote_code + model_name, + model, + tokenizer, + max_seq_length, + trust_remote_code, + token, + cache_dir, + revision, ) modules["0"] = transformer_module From 6d674e5cc9aef396ce8aae45306b2b42beb76244 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:08:39 -0700 Subject: [PATCH 006/367] unsloth start: warn before running an agent's remote installer (#7024) When a coding agent is missing, `unsloth start ` offers to run the vendor's own installer (curl | bash, irm | iex, or npm) after an interactive confirm. Those installers execute with the user's privileges and there is no signature or hash check on the fetched content, so a blind "yes" is a supply-chain risk if the delivery path is compromised. Keep the auto-install convenience but make consent informed: before the prompt, name the exact remote source the installer fetches (or the command it runs for a package installer) and state that nothing verifies a signature or hash. Behavior is otherwise unchanged: non-interactive stdin still never executes anything, and the confirm still defaults to no. --- unsloth_cli/commands/start.py | 19 ++++++++++++++++++- unsloth_cli/tests/test_start.py | 26 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py index 764f5c7963..a8665b9be3 100644 --- a/unsloth_cli/commands/start.py +++ b/unsloth_cli/commands/start.py @@ -989,6 +989,12 @@ def _refresh_windows_path() -> None: os.environ["PATH"] = os.pathsep.join(entries) +def _install_source(install_hint: str) -> Optional[str]: + """The first http(s) URL an install hint fetches, or None (e.g. an npm install).""" + match = re.search(r"https?://[^\s'\")]+", install_hint) + return match.group(0) if match else None + + def _install_agent(name: str, install_hint: str) -> Optional[str]: # Missing agent under --launch: offer to run its documented install command, then # re-resolve it on PATH. Consent-based (we never auto-run a remote install script @@ -997,7 +1003,18 @@ def _install_agent(name: str, install_hint: str) -> Optional[str]: if not sys.stdin.isatty(): return None typer.echo(f"`{name}` is not installed.") - if not typer.confirm(f"Install it now with `{install_hint}`?", default = False): + # Make the supply-chain risk explicit before the prompt: these are the vendors' + # own installers (curl | bash, irm | iex, npm), run with the user's privileges, + # and nothing checks a signature or hash on the fetched content. Naming the source + # turns a blind "yes" into informed consent. + source = _install_source(install_hint) + warning = ( + f"This will download and RUN a script from {source} with your privileges" + if source + else f"This will RUN `{install_hint}` with your privileges" + ) + typer.secho(f"{warning}; there is no signature or hash check.", fg = "yellow", err = True) + if not typer.confirm(f"Install `{name}` now with `{install_hint}`?", default = False): return None # Run each hint through the shell it is written for: PowerShell (irm | iex, or npm) # on Windows, /bin/sh (curl | bash, or npm) everywhere else. diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py index 18cb40f18d..87a295532e 100644 --- a/unsloth_cli/tests/test_start.py +++ b/unsloth_cli/tests/test_start.py @@ -128,6 +128,32 @@ def test_install_agent_uses_powershell_on_windows(monkeypatch): assert ran == [["powershell", "-NoProfile", "-Command", install_hint]] +def test_install_agent_warns_and_names_remote_source(monkeypatch, capsys): + # Before the confirm, a remote installer must name the URL it fetches so the + # user consents to a specific source rather than blindly accepting. + monkeypatch.setattr(start.os, "name", "nt") + monkeypatch.setattr(start.sys, "stdin", SimpleNamespace(isatty = lambda: True)) + monkeypatch.setattr(start.typer, "confirm", lambda *a, **k: False) # decline: nothing runs + hint = "& ([scriptblock]::Create((irm https://hermes-agent.nousresearch.com/install.ps1))) -SkipSetup" + assert start._install_agent("hermes", hint) is None + err = capsys.readouterr().err + assert "https://hermes-agent.nousresearch.com/install.ps1" in err + assert "download and RUN" in err + assert "signature or hash" in err + + +def test_install_agent_warns_for_package_installer(monkeypatch, capsys): + # An npm-style installer has no URL to fetch, but still runs with the user's + # privileges, so the warning names the command instead. + monkeypatch.setattr(start.os, "name", "posix") + monkeypatch.setattr(start.sys, "stdin", SimpleNamespace(isatty = lambda: True)) + monkeypatch.setattr(start.typer, "confirm", lambda *a, **k: False) + assert start._install_agent("codex", "npm install -g @openai/codex") is None + err = capsys.readouterr().err + assert "npm install -g @openai/codex" in err + assert "with your privileges" in err + + def test_hermes_install_hint_is_windows_native_on_windows(monkeypatch): monkeypatch.setattr(start.os, "name", "nt") From 0d4bd50768ca1d55009b51dfa097d7c71e819f4b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:26:24 -0700 Subject: [PATCH 007/367] Restore process-global torch.compile config on torch 2.12 so gradient checkpointing backward honors it (#7019) * Mirror dynamo/inductor config sets into defaults so torch 2.12 worker threads honor them torch 2.12 stores config user overrides in ContextVars, so direct assignments like torch._dynamo.config.recompile_limit = 1024 no longer reach the autograd engine worker threads. Gradient checkpointing recomputes fullgraph-compiled gpt-oss kernels inside backward on those threads, which then read the default recompile limit of 8 and raise FailOnRecompileLimitHit at step 0 of GRPO/SFT. Mirror direct config assignments into the process-global entry defaults on torch >= 2.12, restoring the torch <= 2.11 cross-thread semantics while leaving the context-scoped config.patch API untouched. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep config.patch thread-local when mirroring dynamo/inductor sets config.patch(...) also assigns through ConfigModule.__setattr__, so the default-mirror was leaking its scoped, thread-local writes into the process-global entry default. Track patch enter/exit with a per-thread depth counter (wrapping ConfigModule.patch) and skip mirroring while inside a patch, so only genuine direct assignments restore the torch 2.11 cross-thread semantics and config.patch stays context-local. * Also keep config.load_config thread-local when mirroring config sets load_config restores a saved dynamo/inductor config by calling setattr per key, which the default-mirror would otherwise leak process-wide just like config.patch did. Wrap load_config with the same per-thread depth counter (renamed to _scoped_depth) so both scoped writers skip the mirror and stay context-local, while genuine direct assignments still restore the torch 2.11 cross-thread default. * Drop the pre-existing override replay from the config thread fix The replay was redundant: this runs from _gpu_init before unsloth sets any dynamo/inductor config, so the __setattr__ wrapper already mirrors every later assignment (recompile_limit included). It could also read a value that belonged to a config.patch context still active at import time and write that thread-local override into the global default. Removing it keeps the cross-thread fix and drops the now-unused _inductor.config import. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/_gpu_init.py | 6 ++ unsloth/import_fixes.py | 132 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py index e39b44488a..e6178e60f3 100644 --- a/unsloth/_gpu_init.py +++ b/unsloth/_gpu_init.py @@ -173,6 +173,7 @@ from .import_fixes import ( fix_vllm_guided_decoding_params, fix_vllm_pdl_blackwell, fix_triton_compiled_kernel_missing_attrs, + fix_dynamo_config_thread_visibility, patch_trunc_normal_precision_issue, ignore_logger_messages, patch_ipykernel_hf_xet, @@ -203,6 +204,10 @@ fix_vllm_guided_decoding_params() fix_trl_vllm_ascend() fix_vllm_pdl_blackwell() fix_triton_compiled_kernel_missing_attrs() +# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss temporary +# patches raise the dynamo recompile limits, so those settings reach the +# autograd worker threads on torch >= 2.12. +fix_dynamo_config_thread_visibility() patch_trunc_normal_precision_issue() ignore_logger_messages() patch_ipykernel_hf_xet() @@ -233,6 +238,7 @@ del fix_vllm_guided_decoding_params del fix_trl_vllm_ascend del fix_vllm_pdl_blackwell del fix_triton_compiled_kernel_missing_attrs +del fix_dynamo_config_thread_visibility del patch_trunc_normal_precision_issue del ignore_logger_messages del patch_ipykernel_hf_xet diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e5a5d01c2f..c5300ed4d0 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -1064,6 +1064,135 @@ def fix_triton_compiled_kernel_missing_attrs(): ) +def fix_dynamo_config_thread_visibility(): + """torch 2.12 made torch._dynamo/_inductor config overrides thread-local + (ContextVars), so `config.recompile_limit = 1024` set on the main thread is + invisible to the autograd worker threads that run backward. Gradient + checkpointing recompiles fullgraph gpt-oss kernels there against the default + limit of 8, raising FailOnRecompileLimitHit at step 0. Mirror direct config + assignments into the process-global entry default (torch <= 2.11 semantics). + config.patch(...) and config.load_config(...) also assign via __setattr__ but + are thread-local by design, so skip mirroring while inside one (tracked per + thread). No-op below torch 2.12 and on any torch without this internal layout. + """ + try: + import torch + + if Version(torch.__version__) < Version("2.12.0"): + return + import torch._dynamo.config as _dynamo_config + from torch.utils._config_module import ConfigModule + from contextvars import ContextVar + except Exception: + return + + try: + probe = getattr(_dynamo_config, "_config", {}).get("recompile_limit", None) + if probe is None or not isinstance(getattr(probe, "user_override", None), ContextVar): + # Overrides are not context-local on this torch; nothing to fix. + return + original_setattr = ConfigModule.__setattr__ + if getattr(original_setattr, "__unsloth_patched__", False): + return + except Exception: + return + + mirrored_modules = ("torch._dynamo.config", "torch._inductor.config") + + # config.patch(...) and config.load_config(...) also assign via __setattr__, but + # their writes are thread-local by design; a per-thread depth counter marks them + # so they are not mirrored into the process-global default. + import threading + + _scoped_depth = threading.local() + + def _in_scoped_write(): + return getattr(_scoped_depth, "n", 0) > 0 + + def _bump(delta): + _scoped_depth.n = getattr(_scoped_depth, "n", 0) + delta + + original_patch = ConfigModule.patch + if not getattr(original_patch, "__unsloth_patched__", False): + + @functools.wraps(original_patch) + def _patched_patch(self, *args, **kwargs): + ctx = original_patch(self, *args, **kwargs) + try: + cls = type(ctx) # patch() builds a fresh ConfigPatch class each call + if not getattr(cls, "__unsloth_patch_wrapped__", False): + _enter0, _exit0 = cls.__enter__, cls.__exit__ + + def _enter(s, _e = _enter0): + _bump(1) + try: + return _e(s) + finally: + _bump(-1) + + def _exit( + s, + *a, + _x = _exit0, + ): + _bump(1) + try: + return _x(s, *a) + finally: + _bump(-1) + + cls.__enter__, cls.__exit__ = _enter, _exit + cls.__unsloth_patch_wrapped__ = True + except Exception: + pass + return ctx + + _patched_patch.__unsloth_patched__ = True + ConfigModule.patch = _patched_patch + + # load_config restores a saved config by calling setattr per key (thread-local). + original_load_config = getattr(ConfigModule, "load_config", None) + if callable(original_load_config) and not getattr( + original_load_config, "__unsloth_patched__", False + ): + + @functools.wraps(original_load_config) + def _patched_load_config(self, *args, **kwargs): + _bump(1) + try: + return original_load_config(self, *args, **kwargs) + finally: + _bump(-1) + + _patched_load_config.__unsloth_patched__ = True + ConfigModule.load_config = _patched_load_config + + @functools.wraps(original_setattr) + def _patched_setattr(self, name, value): + original_setattr(self, name, value) + if _in_scoped_write(): + return # transient patch / load_config write: keep it thread-local + # Aliases (cache_size_limit -> recompile_limit) re-enter with the real name. + if self.__dict__.get("__name__", None) in mirrored_modules: + try: + entry = self.__dict__["_config"].get(name, None) + if entry is not None and entry.alias is None: + entry.default = value + except Exception: + pass + + _patched_setattr.__unsloth_patched__ = True + ConfigModule.__setattr__ = _patched_setattr + + # No replay of existing overrides: unsloth installs this before it sets any + # dynamo/inductor config, so the wrapper mirrors every later assignment. Replaying + # would also bake a still-active config.patch override into the global default. + logger.info( + "Unsloth: Patched torch config modules so dynamo/inductor settings " + "(e.g. recompile_limit) apply across threads on torch >= 2.12." + ) + + def patch_trunc_normal_precision_issue(): """ Patch torch.nn.init.trunc_normal_ for low precision tensors to run init in fp32. @@ -1323,8 +1452,7 @@ def fix_vllm_pdl_blackwell(): if patched: logger.info( - f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - " - f"patched: {', '.join(patched)}" + f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - patched: {', '.join(patched)}" ) else: # Just set the env var - vLLM might be an older version without supports_pdl From b509d47dd7427a1ba9ff1c80d1ca64fb9889bddf Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:26:36 -0700 Subject: [PATCH 008/367] Silence torch._check_is_size FutureWarning and shim it if torch removes it (#7023) * Silence torch._check_is_size FutureWarning and shim it if torch removes it bitsandbytes 4-bit dequant calls torch._check_is_size, which torch deprecated with a FutureWarning ("Use _check(i >= 0) instead") that prints on every bnb-4bit load. Silence that warning in suppress_cuda_printf, and add fix_torch_check_is_size so a future torch that removes _check_is_size gets it shimmed to _check(i >= 0) (honoring the max bound) and bitsandbytes keeps working. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Tighten fix_torch_check_is_size docstring Lead with what the shim does and drop the redundant line; two lines instead of three, same intent. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/_gpu_init.py | 3 +++ unsloth/import_fixes.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py index e6178e60f3..984057e9f7 100644 --- a/unsloth/_gpu_init.py +++ b/unsloth/_gpu_init.py @@ -26,6 +26,7 @@ already_imported = [mod for mod in critical_modules if mod in sys.modules] # Fix some issues before importing other packages from .import_fixes import ( fix_message_factory_issue, + fix_torch_check_is_size, check_fbgemm_gpu_version, disable_broken_causal_conv1d, disable_broken_vllm, @@ -72,6 +73,7 @@ fix_bitsandbytes_rocm_arch_detection() disable_broken_causal_conv1d() disable_broken_vllm() fix_message_factory_issue() +fix_torch_check_is_size() check_fbgemm_gpu_version() torchvision_compatibility_check() fix_diffusers_warnings() @@ -81,6 +83,7 @@ del fix_bitsandbytes_rocm_arch_detection del disable_broken_causal_conv1d del disable_broken_vllm del fix_message_factory_issue +del fix_torch_check_is_size del check_fbgemm_gpu_version del torchvision_compatibility_check del fix_diffusers_warnings diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index c5300ed4d0..09de248c7b 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -172,6 +172,10 @@ if not UNSLOTH_ENABLE_LOGGING: # Deprecation warnings from torchao warnings.filterwarnings("ignore", message = "`int4_weight_only` is deprecated") warnings.filterwarnings("ignore", message = "`int8_weight_only` is deprecated") + # torch._check_is_size FutureWarning (called by bitsandbytes 4-bit dequant) + warnings.filterwarnings( + "ignore", message = r"_check_is_size will be removed", category = FutureWarning + ) # TorchAO deprecated import paths (https://github.com/pytorch/ao/issues/2752) warnings.filterwarnings( @@ -253,6 +257,30 @@ if not UNSLOTH_ENABLE_LOGGING: ) +def fix_torch_check_is_size(): + """Shim torch._check_is_size if a future torch removes it (bitsandbytes 4-bit + dequant calls it). The FutureWarning is silenced in suppress_cuda_printf.""" + try: + import torch + + if hasattr(torch, "_check_is_size"): + return + + def _check_is_size( + i, + message = None, + *, + max = None, + ): + torch._check(i >= 0, message) + if max is not None: + torch._check(i <= max, message) + + torch._check_is_size = _check_is_size + except Exception: + return + + # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' # MUST do this at the start primarily due to tensorflow causing issues def fix_message_factory_issue(): From c1e06e9ddfb53a9d40e1fb182030aded94f4b4bb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:47:59 -0700 Subject: [PATCH 009/367] unsloth start: add --persist to keep and reopen agent sessions (#7014) * unsloth start: add --resume to persist and reopen agent sessions `unsloth start ` launches a coding agent whose home is a throwaway temp dir wiped on exit, so codex/openclaw/hermes/pi (which relocate their whole home there) cannot resume a conversation after you quit. opencode and claude keep their session data in a fixed user dir, so they already resume. Add an opt-in --resume/--no-resume flag: it routes the launch to the stable Unsloth agents dir (the same one --no-launch already uses) so the session survives the exit, never touching the user's own ~/.. A bare --resume also reopens the last conversation via the agent's native flag (codex `resume --last`, opencode/claude/pi `--continue`). The default is unchanged: a plain launch still uses a temp dir and persists nothing. Add a dispatch-only `resume` job to the Local Agent Guides CI that drives the real launch path and asserts the split: codex/pi are wiped without --resume and persist with it, while opencode/claude persist either way. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * unsloth start: rename --resume to --persist The session flag collided with agents' own resume flags. `unsloth start claude --resume ` used to forward `--resume ` straight to Claude (which keeps its history in ~/.claude regardless), so a boolean --resume on unsloth start would have swallowed the session id and turned it into a stray prompt. Name the persistence flag --persist instead, so every agent's native resume flag (claude --resume , codex resume, opencode --continue, ...) still passes through untouched. Behavior is otherwise identical: --persist keeps a launched agent's session under the Unsloth agents dir, and a bare --persist reopens the last conversation. Add a regression test that `--resume ` passes through verbatim, and in the CI resume experiment skip the redundant second pass for opencode/claude (they persist either way, and a second CPU turn only risks a timeout). * unsloth start: correct --persist help and drop the buggy auto-resume Reword the --persist help to be accurate: claude and opencode keep sessions in the user's own stores and resume regardless, so --persist only stabilizes the otherwise-ephemeral relocated home of codex/openclaw/hermes/pi. Drop the bare-launch auto-append of native resume tokens: it errored on a first launch with no prior session, and was inconsistent between launch and no-launch. --persist now only keeps the session dir; resume via the agent's own command (e.g. `unsloth start codex --persist resume`), which now finds it. In the CI resume experiment, fail the pass when the launched turn exits non-zero, so a write-then-error is not misread as PERSISTED. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/scripts/agent-guides-drive.sh | 148 +++++++++++++++++ .github/workflows/local-agent-guides-ci.yml | 170 ++++++++++++++++++++ unsloth_cli/commands/start.py | 53 ++++-- unsloth_cli/tests/test_start.py | 142 ++++++++++++++++ 4 files changed, 502 insertions(+), 11 deletions(-) diff --git a/.github/scripts/agent-guides-drive.sh b/.github/scripts/agent-guides-drive.sh index defdb498c7..f4189a159e 100755 --- a/.github/scripts/agent-guides-drive.sh +++ b/.github/scripts/agent-guides-drive.sh @@ -527,6 +527,154 @@ case "$MODE" in echo "[claude] attribution A/B OK (suppressed HIT, header=1 MISS)" ;; + # ── resume: does a launched agent's session survive exit and resume? ──── + # Unlike the other modes, this drives the real LAUNCH path (`unsloth start + # ...`, the interactive default), not the --no-launch recipe. That + # path relocates each agent's home to a throwaway temp dir wiped on exit, so + # a session cannot be resumed -- unless --persist routes it to the stable + # Unsloth agents dir instead. We run one headless turn per pass and check + # whether the turn left a session in a persistent store (deterministic, no + # reliance on the model recalling anything), for a baseline pass and a + # --persist pass, and assert the expected split for this agent. + resume) + CODEWORD="PLATYPUS7" + T1="Remember this codeword for later: ${CODEWORD}. Reply with just the word OK." + T2="What codeword did I ask you to remember? Reply with just that word." + WORK="$WORKDIR_BASE/${AGENT}-resume" + + # STABLE_HOME: the stable dir that --no-launch (and --persist) relocate to. + # Read it from a --no-launch probe (which also writes the agent's config + # there). codex/pi relocate their whole home/HOME here; opencode/claude keep + # their session data in a fixed user dir, so STABLE_HOME stays empty for them. + parse_connect + case "$AGENT" in + codex) STABLE_HOME="$(raw_env CODEX_HOME)" ;; + pi) STABLE_HOME="$(raw_env HOME)" ;; + *) STABLE_HOME="" ;; + esac + + # The persistent stores a session would land in if it were NOT wiped. We + # count files here before/after each turn; a positive delta means the + # session persisted (is resumable), zero means it went to a wiped temp dir. + resume_tracked_dirs() { + case "$AGENT" in + codex) printf '%s\n' "$HOME/.codex" ;; + opencode) printf '%s\n' "$HOME/.local/share/opencode" "$HOME/.config/opencode" ;; + claude) printf '%s\n' "$HOME/.claude" ;; + pi) printf '%s\n' "$HOME/.pi" ;; + *) : ;; + esac + [ -n "$STABLE_HOME" ] && printf '%s\n' "$STABLE_HOME" + } + count_session_files() { + local total=0 d n + while IFS= read -r d; do + [ -n "$d" ] && [ -d "$d" ] || continue + n="$(find "$d" -type f 2>/dev/null | wc -l)"; total=$((total + n)) + done < <(resume_tracked_dirs) + echo "$total" + } + + # The headless first-turn subcommand per agent (mirrors file-edit's map), + # forwarded verbatim through the launch path as passthrough args. + set_t1_cmd() { + case "$AGENT" in + claude) T1_CMD=("${CLAUDE_CONNECT_FLAGS[@]}" -p "$T1") ;; + codex) T1_CMD=(exec "$T1") ;; + opencode) T1_CMD=(run "$T1") ;; + pi) T1_CMD=(-p "$T1") ;; + *) guide_fail "resume mode does not cover agent '$AGENT'" ;; + esac + } + + # Run one headless turn through the launch path. $1=outfile, $2="" or + # "--persist", rest = the agent subcommand. --yolo auto-approves so no tool + # prompt can hang; --api-key attaches to the already-served CI model. + launch_turn() { + local out="$1" rflag="$2"; shift 2 + local flag=(); [ -n "$rflag" ] && flag=("$rflag") + run_timed "$out" unsloth start "$AGENT" "${flag[@]}" --yolo \ + --api-key "$UNSLOTH_API_KEY" "$@" + local rc=$? + redact "$out" + return "$rc" + } + + # One pass: fresh work dir, one planting turn, set RESULT to PERSISTED/WIPED + # from the session-store delta. Runs in the main shell (not a command + # substitution) so a hang's guide_fail actually fails the job and the + # progress lines reach the CI log. $1 = "" (baseline) or "--persist". + RESULT="" + run_pass() { + local rflag="$1" label="baseline" + [ -n "$rflag" ] && label="resume" + rm -rf "$WORK"; mkdir -p "$WORK" + set_t1_cmd + local out="$LOGS_DIR/${AGENT}-resume-${label}.txt" + local before after rc + before="$(count_session_files)" + pushd "$WORK" >/dev/null || guide_fail "could not enter work dir $WORK" + launch_turn "$out" "$rflag" "${T1_CMD[@]}"; rc=$? + popd >/dev/null || true + after="$(count_session_files)" + echo "[$AGENT] ${label}: session files ${before} -> ${after} (rc=${rc})" + # The turn must succeed for the delta to mean anything: an agent that writes a + # session file then errors would otherwise be misread as PERSISTED. Mirror the + # file-edit mode and fail the pass on a non-zero launch (the flagship codex recall + # below stays WARN-only, driven by its own launch_turn calls). + [ "$rc" -eq 0 ] || { echo "[$AGENT] ${label} transcript (tail):"; tail -30 "$out" 2>/dev/null || true; \ + guide_fail "resume ${label} turn for ${AGENT} exited non-zero (rc=${rc})"; } + if [ "$after" -gt "$before" ]; then RESULT="PERSISTED"; else RESULT="WIPED"; fi + } + + run_pass ""; BASELINE="$RESULT" + # Only the temp-dir agents (codex/pi) need the --persist pass to prove the fix. + # opencode/claude persist either way, so the baseline already proves it and a + # second full CPU turn only risks a timeout; skip it for them. + case "$AGENT" in + codex|pi) run_pass "--persist"; RESUME="$RESULT" ;; + *) RESUME="n/a (persists either way)" ;; + esac + + # Expected: codex/pi relocate their whole home to the temp dir, so a plain + # launch is WIPED and only --persist PERSISTS. opencode/claude keep their + # session data in a fixed user dir, so the baseline already PERSISTS. + case "$AGENT" in + codex|pi) EXPECT_BASELINE="WIPED" ;; + opencode|claude) EXPECT_BASELINE="PERSISTED" ;; + esac + + echo "──────────────────────────────────────────────" + echo "[$AGENT] RESUME EXPERIMENT" + echo " baseline (unsloth start ${AGENT}): ${BASELINE} (expected ${EXPECT_BASELINE})" + echo " with --persist (unsloth start ${AGENT} --persist): ${RESUME}" + echo "──────────────────────────────────────────────" + + [ "$BASELINE" = "$EXPECT_BASELINE" ] \ + || guide_fail "baseline resume behavior for ${AGENT} was ${BASELINE}, expected ${EXPECT_BASELINE}" + case "$AGENT" in + codex|pi) + [ "$RESUME" = "PERSISTED" ] \ + || guide_fail "--persist did not persist ${AGENT}'s session (got ${RESUME}); the session dir is still not stable" ;; + esac + + # Flagship behavioral proof (codex only, WARN-only): after a --persist plant, + # resume the session and check the model actually recalls the codeword. A + # miss is not a failure (the CI model is small); the mechanism gate above is + # the real assertion. + if [ "$AGENT" = "codex" ]; then + rm -rf "$WORK"; mkdir -p "$WORK" + ( cd "$WORK" && launch_turn "$LOGS_DIR/codex-resume-plant.txt" "--persist" exec "$T1" ) || true + ( cd "$WORK" && launch_turn "$LOGS_DIR/codex-resume-recall.txt" "--persist" exec resume --last "$T2" ) || true + if grep -q "$CODEWORD" "$LOGS_DIR/codex-resume-recall.txt" 2>/dev/null; then + echo "[codex] behavioral recall HIT: resumed session remembered ${CODEWORD}" + else + echo "::warning::[codex] behavioral recall MISS (small CI model); mechanism gate still passed" + fi + fi + echo "[$AGENT] resume OK" + ;; + *) echo "agent-guides-drive.sh: unknown mode '$MODE'" >&2 exit 2 diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml index 47f75dc1ba..25796bd5cf 100644 --- a/.github/workflows/local-agent-guides-ci.yml +++ b/.github/workflows/local-agent-guides-ci.yml @@ -471,6 +471,176 @@ jobs: redacted-configs/ retention-days: 7 + # ═════════════════════════════════════════════════════════════════════ + # Job: resume + # Does a conversation started with `unsloth start ` survive exit + # and resume? This drives the REAL launch path (not the --no-launch + # recipe the other jobs use). A plain launch relocates the agent home to + # a temp dir wiped on exit, so codex/pi cannot resume; --persist routes the + # session to the stable Unsloth agents dir so it persists. opencode/claude + # keep their session data in a fixed user dir, so they persist either way. + # Dispatch-only: it is an end-to-end experiment, not a PR gate. + # ═════════════════════════════════════════════════════════════════════ + resume: + name: resume (${{ matrix.agent }}) + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # codex/pi relocate their whole home (resume broken without --persist); + # opencode/claude keep session data in a fixed dir (resume already works). + # One agent from each class proves the split end to end; openclaw/hermes + # share codex's relocation mechanism and are covered by the unit tests. + agent: [codex, opencode, claude, pi] + env: + GGUF_REPO: unsloth/gemma-4-E4B-it-GGUF + GGUF_FILE: gemma-4-E4B-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18904' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Linux deps for llama.cpp prebuilt + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev libssl-dev jq + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + cache: 'pip' + + - name: Restore GGUF model file + id: cache-gguf + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Download GGUF if cache miss + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + python -m pip install --upgrade huggingface_hub + mkdir -p gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache + + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Install Studio (--local, --no-torch) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + mkdir -p logs + set -o pipefail + bash install.sh --local --no-torch 2>&1 | tee logs/install.log + + - name: Serve unsloth run --disable-tools (gemma-4-E4B) + run: | + unsloth studio reset-password + bash .github/scripts/serve-unsloth-run.sh \ + --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ + --port "$STUDIO_PORT" --log-dir logs \ + --extra "--seed $UNSLOTH_SEED --temp 0" \ + --health-timeout 900 + + - name: Preflight the agent's API dialect (class-a isolation) + env: + AGENT: ${{ matrix.agent }} + run: | + set -uo pipefail + B="$UNSLOTH_BASE_URL"; K="$UNSLOTH_API_KEY" + preflight_fail() { + echo "::error::[server/API regression] agent=$AGENT: $* (preflight failed BEFORE install/connect). Endpoint contract lives in studio/backend/routes/**."; + exit 1 + } + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/models" \ + -H "Authorization: Bearer $K") || true + [ "$code" = "200" ] || preflight_fail "/v1/models returned HTTP $code" + case "$AGENT" in + claude) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/messages" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/messages returned HTTP $code" + ;; + codex) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/responses" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"input\":\"Hi\",\"max_output_tokens\":16,\"stream\":true}") || true + [ "$code" = "200" ] || preflight_fail "/v1/responses returned HTTP $code" + ;; + *) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/chat/completions" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/chat/completions returned HTTP $code" + ;; + esac + echo "preflight OK for $AGENT" + + - name: Install agent CLI (class-b isolation) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-install.sh "$AGENT" + + - name: Resume experiment (launch path) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-drive.sh resume "$AGENT" + + - name: Collect server logs (debug) + if: always() + run: | + mkdir -p logs/studio-logs + cp -r "$HOME/.unsloth/studio/logs/." logs/studio-logs/ 2>/dev/null || true + if [ -n "${UNSLOTH_API_KEY:-}" ]; then + grep -rlF "$UNSLOTH_API_KEY" logs redacted-configs agent-workdir 2>/dev/null | while IFS= read -r f; do + sed -i "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + done + fi + + - name: Stop Studio + if: always() + run: | + if [ -n "${UNSLOTH_SERVER_PID:-}" ] && [ "${UNSLOTH_SERVER_PID}" != "0" ]; then + kill "${UNSLOTH_SERVER_PID}" 2>/dev/null || true + fi + sleep 2 + ss -tln 2>/dev/null | grep ":${STUDIO_PORT}" || true + + - name: Upload logs + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: resume-${{ matrix.agent }}-log + path: | + logs/ + agent-workdir/ + redacted-configs/ + retention-days: 7 + # ═════════════════════════════════════════════════════════════════════ # Job 3: prompt-cache # (a) curl 2-turn /v1/chat/completions: assert turn-2 cached_tokens > 0 diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py index a8665b9be3..48c0aca34b 100644 --- a/unsloth_cli/commands/start.py +++ b/unsloth_cli/commands/start.py @@ -133,6 +133,21 @@ _YOLO_OPTION = typer.Option( "flag/config. Any of the three spellings works for any agent." ), ) +_PERSIST_OPTION = typer.Option( + False, + "--persist/--no-persist", + help = ( + "Keep this agent's Unsloth-managed session dir so you can resume it later. " + "codex/openclaw/hermes/pi have their whole home relocated into an Unsloth dir " + "that is a throwaway temp dir (wiped on exit) by default; with --persist it " + "lives under the Unsloth agents dir and survives, so their own resume can reopen " + "it. claude and opencode keep sessions in your own stores (~/.claude, " + "~/.local/share/opencode), so they already resume regardless. To reopen a " + "session, pass the agent's own resume command through, e.g. " + "`unsloth start codex --persist resume` or `claude --resume `; those flow to " + "the agent unchanged." + ), +) # Per-agent CLI flag for "run tools without prompting". opencode and openclaw have no # such flag (config only) and are handled in their config writers, so they are absent. @@ -1133,15 +1148,20 @@ def _agents_config_root() -> Path: @contextlib.contextmanager -def _session_config(agent: str, launch: bool): +def _session_config( + agent: str, + launch: bool, + persist: bool = False, +): """Yield a private directory for an agent's session config (never the user's own). - launch: an ephemeral temp dir removed after the agent process exits, so nothing - persists. no-launch: a stable Unsloth-owned dir (the printed recipe is run later - on this machine), reused across runs. Either way the user's real ~/. - config is left untouched. + launch (default): an ephemeral temp dir removed after the agent process exits, so + nothing persists. no-launch: a stable Unsloth-owned dir (the printed recipe is run + later on this machine), reused across runs. persist (from --persist): use that same + stable dir even for a launch, so the agent's session survives the exit and can be + resumed next time. Either way the user's real ~/. config is left untouched. """ - if launch: + if launch and not persist: path = Path(tempfile.mkdtemp(prefix = f"unsloth-{agent}-")) try: yield path @@ -1453,6 +1473,7 @@ def claude( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point Claude Code at the running Studio server and start it.""" base, key, entry = _connect( @@ -1497,6 +1518,9 @@ def claude( # --yolo (or its aliases) maps to Claude's own --dangerously-skip-permissions. # IS_SANDBOX is left unset on purpose: Claude refuses bypass mode as root unless a # sandbox is detected, and we don't want to falsely claim one on the user's host. + # claude keeps its history in ~/.claude/projects, which --settings/env never + # relocate, so a session already survives exit; resume it with `claude --continue` + # or `--resume ` passed through. command = [ "claude", "--model", @@ -1533,6 +1557,7 @@ def codex( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point OpenAI Codex at the running Studio server and start it.""" base, key, entry = _connect( @@ -1558,7 +1583,7 @@ def codex( *_yolo_command_flags("codex", yolo), *ctx.args, ] - with _session_config("codex", launch) as home: + with _session_config("codex", launch, persist = persist) as home: write_codex_config(base, entry, home) env = {_CODEX_ENV_KEY: key, "CODEX_HOME": str(home)} _run(base, entry, env, command, launch = launch, install_hint = "npm install -g @openai/codex") @@ -1576,6 +1601,7 @@ def openclaw( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point OpenClaw at the running Studio server and start it.""" base, key, entry = _connect( @@ -1601,7 +1627,7 @@ def openclaw( if os.name == "nt" else "curl -fsSL https://openclaw.ai/install.sh | bash" ) - with _session_config("openclaw", launch) as cfg: + with _session_config("openclaw", launch, persist = persist) as cfg: config_path = cfg / "openclaw.json" # key lives in the config, not the env; --yolo writes the exec policy here too. write_openclaw_config(base, key, entry, config_path, yolo = yolo) @@ -1622,6 +1648,7 @@ def opencode( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point OpenCode at the running Studio server and start it.""" base, key, entry = _connect( @@ -1645,7 +1672,9 @@ def opencode( command = ["opencode", "--model", opencode_model] else: command = ["opencode"] - with _session_config("opencode", launch) as cfg: + # opencode keeps sessions in ~/.local/share/opencode (never relocated), so resume + # already survives exit; reopen the last one by passing `opencode --continue` through. + with _session_config("opencode", launch, persist = persist) as cfg: config_path = cfg / "opencode.json" # OPENCODE_CONFIG is an overlay (loaded between the user's global and project # configs), so this adds the Unsloth provider/model for the session without @@ -1697,6 +1726,7 @@ def hermes( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point Hermes (Nous Research) at the running Studio server and start it.""" base, key, entry = _connect( @@ -1708,7 +1738,7 @@ def hermes( ) command = ["hermes", *_yolo_command_flags("hermes", yolo), *ctx.args] install_hint = _hermes_install_hint() - with _session_config("hermes", launch) as home: + with _session_config("hermes", launch, persist = persist) as home: # HERMES_HOME relocates hermes' whole home dir (config.yaml, sessions, state) # like CODEX_HOME, so the user's ~/.hermes is left untouched for the session. write_hermes_config(base, entry, home / "config.yaml") @@ -1728,6 +1758,7 @@ def pi( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point Pi (coding agent) at the running Studio server and start it.""" base, key, entry = _connect( @@ -1752,7 +1783,7 @@ def pi( # --ignore-scripts matches Pi's documented install recipe (its README notes Pi needs # no install scripts), so accepting the prompt skips dependency lifecycle scripts. install_hint = "npm install -g --ignore-scripts @earendil-works/pi-coding-agent" - with _session_config("pi", launch) as home: + with _session_config("pi", launch, persist = persist) as home: # Pi resolves its config dir from PI_CODING_AGENT_DIR first (getAgentDir() prefers # it over $HOME/.pi/agent), so pin it at the session dir: an inherited # PI_CODING_AGENT_DIR in the user's shell would otherwise send Pi to their real diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py index 87a295532e..065972b275 100644 --- a/unsloth_cli/tests/test_start.py +++ b/unsloth_cli/tests/test_start.py @@ -2548,3 +2548,145 @@ def test_session_config_no_launch_preserves_existing_state(fake_studio, tmp_path with start._session_config("codex", launch = False) as home2: assert home2 == home assert (home2 / "sessions" / "live.sqlite").read_text() == "state" + + +# ── --persist: persist the agent session so it can be resumed ──────────────── +def test_session_config_persist_uses_stable_dir_and_survives(monkeypatch, tmp_path): + # --persist routes a launch to the stable Unsloth agents dir (the one --no-launch + # already uses) instead of a throwaway temp dir, and never wipes it on exit. + monkeypatch.setattr(start, "_agents_config_root", lambda: tmp_path / "agents") + with start._session_config("codex", launch = True, persist = True) as home: + assert home == tmp_path / "agents" / "codex" + (home / "marker").write_text("kept") + assert home.exists() + assert (home / "marker").read_text() == "kept" + + +def test_session_config_default_launch_is_ephemeral(): + # Default launch (no --persist) still uses a throwaway temp dir wiped on exit. + with start._session_config("codex", launch = True) as home: + assert home.exists() + assert "unsloth-codex-" in home.name + assert not home.exists() + + +# The temp-dir agents: --persist points each one's home/state env at the stable dir; +# without it, at an ephemeral temp path. opencode is handled separately (only its +# config overlay is relocated; its session data was never in the temp dir). +_RESUME_ENV_VAR = { + "codex": "CODEX_HOME", + "openclaw": "OPENCLAW_STATE_DIR", + "hermes": "HERMES_HOME", + "pi": "HOME", +} + + +def _capture_launch(monkeypatch, argv): + captured = {} + + def run( + command, + env = None, + **kwargs, + ): + captured["command"] = command + captured["env"] = env + return SimpleNamespace(returncode = 0) + + monkeypatch.setattr(start.subprocess, "run", run) + result = CliRunner().invoke(start.start_app, argv) + assert result.exit_code == 0, result.output + return captured + + +@pytest.mark.parametrize("agent", sorted(_RESUME_ENV_VAR)) +def test_resume_persists_agent_home_to_stable_dir(agent, fake_studio, tmp_path, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: f"/usr/local/bin/{agent}") + captured = _capture_launch(monkeypatch, [agent, "--persist"]) + stable = tmp_path / "agents" / agent + assert captured["env"][_RESUME_ENV_VAR[agent]] == str(stable) + # The stable dir survives the agent exit, so the session can be resumed. + assert stable.exists() + + +@pytest.mark.parametrize("agent", sorted(_RESUME_ENV_VAR)) +def test_default_launch_home_is_ephemeral(agent, fake_studio, tmp_path, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: f"/usr/local/bin/{agent}") + captured = _capture_launch(monkeypatch, [agent]) + home = captured["env"][_RESUME_ENV_VAR[agent]] + assert f"unsloth-{agent}-" in home + assert str(tmp_path / "agents") not in home + + +def test_resume_opencode_config_in_stable_dir(fake_studio, tmp_path, monkeypatch): + # opencode's session data lives in ~/.local/share/opencode (never relocated), so + # resume already survives exit; --persist also stabilizes its config overlay dir. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/opencode") + captured = _capture_launch(monkeypatch, ["opencode", "--persist"]) + stable = tmp_path / "agents" / "opencode" + assert captured["env"]["OPENCODE_CONFIG"] == str(stable / "opencode.json") + assert stable.exists() + + +def test_persist_bare_codex_launch_has_no_resume_token(fake_studio, monkeypatch): + # A bare `--persist` only persists the session dir; it must NOT auto-append a native + # resume token, or the very first launch (no session yet) would send codex down its + # no-session error path. The user resumes explicitly: `unsloth start codex --persist resume`. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/codex") + captured = _capture_launch(monkeypatch, ["codex", "--persist"]) + assert "resume" not in captured["command"] + # command[0] is the resolved executable path; assert the argv after it. + assert captured["command"][1:] == ["--oss", "--profile", start._CODEX_PROFILE] + + +def test_persist_bare_opencode_launch_has_no_resume_token(fake_studio, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/opencode") + captured = _capture_launch(monkeypatch, ["opencode", "--persist"]) + assert "--continue" not in captured["command"] + assert captured["command"][1:] == ["--model", f"{start._OPENCODE_PROVIDER}/{MODEL['id']}"] + + +def test_persist_bare_claude_launch_has_no_resume_token(fake_studio, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/claude") + monkeypatch.setattr(start, "_claude_flags", lambda: []) + captured = _capture_launch(monkeypatch, ["claude", "--persist"]) + assert "--continue" not in captured["command"] + assert captured["command"][1:] == ["--model", MODEL["id"]] + + +def test_resume_with_passthrough_does_not_auto_append(fake_studio, monkeypatch): + # When the caller drives their own subcommand, --persist only persists the dir; it + # must not inject a resume token that would collide with the user's command. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/codex") + captured = _capture_launch(monkeypatch, ["codex", "--persist", "exec", "hello"]) + assert "resume" not in captured["command"] + assert captured["command"][-2:] == ["exec", "hello"] + + +def test_default_launch_has_no_resume_token(fake_studio, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/codex") + captured = _capture_launch(monkeypatch, ["codex"]) + assert "resume" not in captured["command"] + + +def test_resume_persist_only_agents_have_no_resume_token(fake_studio, monkeypatch): + # openclaw/hermes persist their session dir but have no non-interactive resume + # selector, so --persist must not append a token; their own picker resumes. + for agent in ("openclaw", "hermes"): + monkeypatch.setattr(start.shutil, "which", lambda _, a = agent: f"/usr/local/bin/{a}") + captured = _capture_launch(monkeypatch, [agent, "--persist"]) + assert "resume" not in captured["command"] + assert "--continue" not in captured["command"] + + +def test_native_resume_flag_passes_through_unchanged(fake_studio, monkeypatch): + # The persistence flag is --persist, NOT --resume, so an agent's own + # `--resume ` (e.g. `unsloth start claude --resume `) still flows + # through to the agent verbatim and is not swallowed as a Studio option. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/claude") + monkeypatch.setattr(start, "_claude_flags", lambda: []) + captured = _capture_launch(monkeypatch, ["claude", "--resume", "some-session-guid"]) + assert captured["command"][-2:] == ["--resume", "some-session-guid"] + # Studio never auto-appends its own resume token when the user drives resume. + assert captured["command"].count("--resume") == 1 + assert "--continue" not in captured["command"] From eb775d320778bb496378344c061bd538e4d39ad9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 03:20:02 -0700 Subject: [PATCH 010/367] Studio /v1/messages: accept thinking and unknown content blocks (#7017) * Studio /v1/messages: accept thinking and unknown content blocks The Anthropic-compatible /v1/messages endpoint modeled a message's content as Union[str, list[{text|image|tool_use|tool_result}]], so any other block type made Pydantic reject the whole request with `messages.N.content.str: Input should be a valid string`. Resuming a Claude session commonly replays assistant turns that carry `thinking` (extended thinking) blocks, and sometimes a null content for a tool-only turn, both of which tripped this and returned a 400. Accept them: - Add a permissive AnthropicUnknownBlock fallback (any block whose type is not one of the four known ones), so thinking/redacted_thinking/provider-specific/ future blocks validate. A validator keeps known types on their typed models, so a malformed known block (e.g. a tool_use without id) still fails cleanly. - Coerce a null message (and tool_result) content to "" so the converter's `for block in content` stays safe. The converter already drops block types it does not translate, so a thinking block is not forwarded to the model. * Studio /v1/messages: keep user content validation strict Make the thinking/null leniency role-aware so it never silently drops real user input. Assistant turns (replayed history) still accept unknown/thinking blocks and coerce a null tool-only turn to empty. User turns keep the strict boundary: a null user content is rejected, and a content block the converter cannot translate is rejected instead of being dropped into an empty prompt. Also remove an empty file committed by accident. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio /v1/messages: coalesce resumed user turns and tighten content checks - The /v1/messages count and generation paths now coalesce the adjacent user turns that dropping an empty or null assistant turn can leave behind, so a strict GGUF chat template no longer 400s on non-alternating roles. - A user content block with a non-string type (list / dict) is rejected as a clean 400 instead of raising TypeError and escaping as a 500. - The assistant null-to-empty coercion only applies to an explicit null; an assistant turn that omits content entirely still fails required-field validation instead of being silently coerced to an empty string. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio /v1/messages: tighten comments --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/models/inference.py | 63 ++++++ studio/backend/routes/inference.py | 14 +- .../backend/tests/test_anthropic_messages.py | 184 ++++++++++++++++++ 3 files changed, 257 insertions(+), 4 deletions(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 0f27b695fe..53b0f14b09 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -1533,12 +1533,41 @@ class AnthropicToolResultBlock(BaseModel): tool_use_id: str content: Union[str, list] = "" + @field_validator("content", mode = "before") + @classmethod + def _coerce_null_content(cls, v): + # Some clients send null content for an empty tool result; the str|list + # union would 400 on it, so treat null as "". + return "" if v is None else v + + +# Block types the converter translates explicitly. Anything else (thinking / +# redacted_thinking, a provider block a resumed session replays, or a future type) +# is accepted as an unknown block and dropped by the converter, rather than 400-ing +# the whole request on strict validation. +_KNOWN_ANTHROPIC_BLOCK_TYPES = frozenset({"text", "image", "tool_use", "tool_result"}) + + +class AnthropicUnknownBlock(BaseModel): + type: str + model_config = {"extra": "allow"} + + @field_validator("type") + @classmethod + def _only_unknown_types(cls, v): + # Known types parse as their typed models above (so a malformed known block + # still fails cleanly); this fallback only catches the rest. + if v in _KNOWN_ANTHROPIC_BLOCK_TYPES: + raise ValueError("known block type handled by its typed model") + return v + AnthropicContentBlock = Union[ AnthropicTextBlock, AnthropicImageBlock, AnthropicToolUseBlock, AnthropicToolResultBlock, + AnthropicUnknownBlock, ] @@ -1583,6 +1612,40 @@ class AnthropicMessage(BaseModel): role: Literal["user", "assistant"] content: Union[str, list[AnthropicContentBlock]] + @model_validator(mode = "before") + @classmethod + def _normalize_content(cls, data): + # Role-aware leniency that never silently drops real user input: + # - assistant: a resumed tool-only turn's null content -> "" (str|list would + # 400 on null; "" keeps the converter's `for block in content` safe). + # Unknown blocks (thinking / future types) validate via + # AnthropicUnknownBlock and are dropped by the converter. + # - user: keep strict. Null user content stays None so str|list rejects it + # (400) rather than forwarding an empty prompt; and reject block types the + # converter cannot translate, since it silently skips unknown user blocks + # -- a user turn made only of them would validate yet send no content + # (silent data loss). + if not isinstance(data, dict): + return data + content = data.get("content") + if data.get("role") == "assistant": + # Coerce only an explicit null (resumed tool-only turn). A missing + # content key stays malformed so the required-field check still 400s. + if "content" in data and content is None: + return {**data, "content": ""} + return data + if isinstance(content, list): + for block in content: + btype = ( + block.get("type") if isinstance(block, dict) else getattr(block, "type", None) + ) + # Guard the value: a non-string type is unsupported too, and a + # membership test on an unhashable value would raise TypeError + # (escaping as a 500 instead of a clean 400). + if not isinstance(btype, str) or btype not in _KNOWN_ANTHROPIC_BLOCK_TYPES: + raise ValueError(f"unsupported content block type {btype!r} in a user message") + return data + class AnthropicTool(BaseModel): # Client tools have input_schema; server tools may only have type/name. diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d9901a5b2e..4bb9ce655e 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -10088,8 +10088,11 @@ async def anthropic_count_tokens( # Apply the same sanitization /messages does before generation, so the count # matches the prompt the real request would build (otherwise empty-assistant # sentinels / synthetic tool history inflate the count or hit the fallback). - openai_messages = _strip_provider_synthetic_tool_history( - _drop_empty_assistant_sentinels(openai_messages) + # Coalesce adjacent user turns left behind by dropping an empty / null assistant + # turn, so a strict GGUF chat template does not 400 on non-alternating roles + # (mirrors the GGUF chat path); a no-op for already-alternating histories. + openai_messages = _coalesce_consecutive_user_turns( + _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(openai_messages)) ) openai_tools = anthropic_tools_to_openai(payload.tools or []) or None @@ -10217,8 +10220,11 @@ async def anthropic_messages( # builders apply the same strip; without it an Anthropic /v1/messages caller # replaying a prior provider-side tool_use forwards fake builtin tool # history to a backend with no matching function declarations. - openai_messages = _strip_provider_synthetic_tool_history( - _drop_empty_assistant_sentinels(openai_messages) + # Coalesce adjacent user turns left behind by dropping an empty / null assistant + # turn, so a strict GGUF chat template does not 400 on non-alternating roles + # (mirrors the GGUF chat path); a no-op for already-alternating histories. + openai_messages = _coalesce_consecutive_user_turns( + _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(openai_messages)) ) # Enforce vision guard + re-encode embedded images to PNG so the Anthropic diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py index 0c6550a3bb..54454f6563 100644 --- a/studio/backend/tests/test_anthropic_messages.py +++ b/studio/backend/tests/test_anthropic_messages.py @@ -1770,3 +1770,187 @@ class TestAnthropicMessagesToolRouting: _drive(anthropic_messages(payload, request = None, current_subject = "t")) assert backend.calls[0][0] == "plain" + + +def test_resumed_session_thinking_and_null_content_do_not_400(): + # A resumed session replays assistant turns with `thinking` (and sometimes null) + # content. Those must be accepted (thinking dropped by the converter), not 400ed. + from pydantic import ValidationError + + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "secret reasoning", "signature": "s"}, + {"type": "text", "text": "the answer"}, + {"type": "tool_use", "id": "t1", "name": "f", "input": {}}, + ], + }, + {"role": "assistant", "content": None}, # tool-only turn serialized as null + ], + ) + # Known blocks still parse as their typed models; only the unknown one is loose. + assert type(req.messages[1].content[0]).__name__ == "AnthropicUnknownBlock" + assert type(req.messages[1].content[1]).__name__ == "AnthropicTextBlock" + assert req.messages[2].content == "" # null coerced + + openai = anthropic_messages_to_openai([m.model_dump() for m in req.messages]) + assistant = next(m for m in openai if m["role"] == "assistant" and m.get("content")) + assert assistant["content"] == "the answer" + assert "secret reasoning" not in json.dumps(openai) # thinking never forwarded + + # A malformed KNOWN block still fails cleanly instead of being swallowed. + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "assistant", "content": [{"type": "tool_use", "name": "f"}]}], + ) + + +def test_user_null_content_rejected(): + # The null->"" leniency is assistant-only; a null user content must be rejected + # at the boundary, not coerced into an empty prompt and forwarded to the model. + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "user", "content": None}], + ) + + +def test_user_unknown_block_rejected_not_silently_dropped(): + # The converter skips user blocks it cannot translate, so a user turn whose only + # block is unknown would validate yet forward no content. Reject at the boundary + # to avoid that silent data loss (the assistant fallback is unaffected). + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": [{"type": "document", "source": {}}]}, + ], + ) + + +def test_user_translatable_blocks_still_accepted(): + # text / image / tool_result are translatable, so a real user message built from + # them must still pass; the unknown-block guard only trips on other types. + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "AA"}, + }, + {"type": "tool_result", "tool_use_id": "t1", "content": "ok"}, + ], + } + ], + ) + assert [type(b).__name__ for b in req.messages[0].content] == [ + "AnthropicTextBlock", + "AnthropicImageBlock", + "AnthropicToolResultBlock", + ] + + openai = anthropic_messages_to_openai([m.model_dump() for m in req.messages]) + assert any(m["role"] == "tool" and m["tool_call_id"] == "t1" for m in openai) + + +def test_user_malformed_known_block_still_rejected(): + # The guard only allow-lists a user block's *type*; the union still validates its + # shape, so a known-but-malformed block (tool_result without tool_use_id) fails. + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": [{"type": "tool_result", "content": "x"}]}, + ], + ) + + +def test_user_content_block_non_string_type_rejected_cleanly(): + # A user block whose `type` is a non-string (unhashable list / dict, or a stray + # int) must fail as a clean validation error, not raise TypeError from the + # frozenset membership test and escape as a 500. + from pydantic import ValidationError + for bad_type in ([], {}, 5): + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "user", "content": [{"type": bad_type}]}], + ) + + +def test_assistant_missing_content_key_still_rejected(): + # The null -> "" leniency is only for an EXPLICIT null. An assistant message that + # omits content entirely stays malformed and must fail required-field validation. + from pydantic import ValidationError + + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "assistant"}], + ) + # An explicit null is still accepted and coerced (regression guard). + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": None}, + ], + ) + assert req.messages[1].content == "" + + +def test_resumed_null_assistant_between_users_coalesced_on_messages_route(monkeypatch): + # user -> assistant(null) -> user is now accepted: the null assistant turn coerces + # to "" and is dropped. The route must then coalesce the two remaining user turns + # so a strict GGUF chat template does not 400 on non-alternating roles. + backend = _mock_backend(monkeypatch, context_length = 2048) + + class _Req: + state = SimpleNamespace() + url = SimpleNamespace(path = "/v1/messages") + method = "POST" + + async def is_disconnected(self): + return False + + payload = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "first question"}, + {"role": "assistant", "content": None}, + {"role": "user", "content": "please continue"}, + ], + ) + + response = _drive(anthropic_messages(payload, request = _Req(), current_subject = "t")) + assert response.status_code == 200 + + [(_path, kwargs)] = backend.calls + user_turns = [m for m in kwargs["messages"] if m.get("role") == "user"] + assert len(user_turns) == 1 # the two user turns were merged, not left adjacent + merged = user_turns[0]["content"] + if isinstance(merged, list): + merged = " ".join(p.get("text", "") for p in merged if isinstance(p, dict)) + assert "first question" in merged and "please continue" in merged From 350233512092fc6847b42050b7768f5f5e9a4578 Mon Sep 17 00:00:00 2001 From: oobabooga Date: Thu, 9 Jul 2026 07:39:48 -0300 Subject: [PATCH 011/367] Studio: add Vulkan llama.cpp support (#5819) * Studio: add Vulkan llama.cpp support * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Address gemini's feedback * Studio: move the Vulkan VRAM probe into a standalone script * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Improve Vulkan probe error reporting * Resolve llama-server symlink so Vulkan build is detected * Drop unreachable Vulkan fallback in GPU free-memory dispatcher * Skip the Intel GPU probe when NVIDIA or ROCm is present * Reserve host RAM headroom for Vulkan integrated GPUs * Add a `UNSLOTH_FORCE_VULKAN` environment variable * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Honor GGML_VK_VISIBLE_DEVICES, reserve discrete Vulkan VRAM headroom, and clear Intel GPU on --cpu-fallback * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Route Intel and forced-Vulkan hosts to the upstream Vulkan prebuilt, add arm64 Vulkan, keep Vulkan out of RAG auto-detect * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clear the fork release pin when routing a Vulkan host to the upstream repo * Gate auto-Vulkan routing on no physical NVIDIA so hidden CUDA devices aren't used * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Pin Vulkan launches with --device Vulkan instead of the raw GGML_VK_VISIBLE_DEVICES index space * Let user --device override the Vulkan pin, and gate direct Vulkan asset picks on no physical NVIDIA * Update RAG auto-backend test mocks for the _resolve_auto binary and Vulkan probes * Keep the add_dll_directory handle alive through the Vulkan probe DLL loads * Revert RAG auto Vulkan guard, guard multi-backend Vulkan detection, and preserve forced Vulkan across updates * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Use getattr for RTLD_GLOBAL in the Vulkan probe CDLL mode * Skip CUDA/ROCm APU and datacenter GPU tuning on Vulkan builds On a Vulkan llama.cpp build gpu_indices are ggml compact ordinals, not CUDA/ROCm physical ids, so _amd_apu_wants_unified_memory and _apply_datacenter_env were reading the wrong device. On a mixed AMD APU plus discrete GPU host that could raise a spurious system-RAM shortfall and block a valid discrete-GPU load. Gate all three call sites on not is_vulkan_backend; the Vulkan path already reserves iGPU host headroom and the backend ignores GGML_CUDA_* anyway. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Tighten Vulkan-guard comment in load_model * Reduce comments in Vulkan support to be more succinct * Resolve shell-wrapper llama-server entrypoint to the real lib dir create_exec_entrypoint falls back to a #!/bin/sh wrapper at the install root when it cannot symlink into build/bin. _find_llama_server_binary returns that root entrypoint, but Path.resolve() does not follow a shell wrapper, so _llama_lib_dir returned the install root and _is_vulkan_backend missed libggml-vulkan.so -- silently skipping the Vulkan probe and --device pin on an otherwise valid Vulkan install. Follow the wrapper's exec target to build/bin. Regression test: test_shell_wrapper_entrypoint_resolves_to_real_lib_dir. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Co-authored-by: danielhanchen --- .../backend/core/inference/_vulkan_probe.py | 110 ++++++++ studio/backend/core/inference/llama_cpp.py | 246 ++++++++++++++++-- .../tests/test_install_resolve_prebuilt.py | 170 ++++++++++++ studio/backend/tests/test_llama_cpp_update.py | 42 +++ .../tests/test_llama_cpp_vulkan_probe.py | 193 ++++++++++++++ studio/backend/utils/llama_cpp_update.py | 6 + studio/install_llama_prebuilt.py | 243 ++++++++++++++++- 7 files changed, 984 insertions(+), 26 deletions(-) create mode 100644 studio/backend/core/inference/_vulkan_probe.py create mode 100644 studio/backend/tests/test_llama_cpp_vulkan_probe.py diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.py new file mode 100644 index 0000000000..706346daad --- /dev/null +++ b/studio/backend/core/inference/_vulkan_probe.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Standalone free-VRAM probe for the bundled ggml Vulkan backend. + +Run in a short-lived subprocess (``python _vulkan_probe.py ``) so the +Vulkan instance never lives in the long-running backend process. Loads the +bundled ggml Vulkan backend from ```` and prints one +``\\t\\t\\t`` line per device to stdout. +Indices are ggml's own Vulkan device ordinals, which need not match nvidia-smi +order. ``is_igpu`` (from ggml's device type) is ``1`` for an integrated GPU +sharing system RAM. ``total_bytes`` is the device-local heap; the reader uses +it to reserve absolute headroom on a discrete card (parity with the CUDA/ROCm +fit) and ignores it for an iGPU, whose "VRAM" is shared system RAM. + +Uses only the standard library so it stays runnable as a bare script. +""" + +import ctypes +import os +import sys + +# ggml_backend_dev_type enum (ggml-backend.h): CPU=0, GPU=1, IGPU=2, ... +_GGML_BACKEND_DEVICE_TYPE_IGPU = 2 + + +def _igpu_flags(base, lib, count: int) -> list[bool]: + """Per-device integrated-GPU flags via ggml's backend registry. + + The Vulkan reg enumerates devices in the same order as + ``ggml_backend_vk_get_device_memory`` (each context uses ``ctx->device = + i``), so reg index == device ordinal. Returns all-False on any failure so + the reader never over-caps a discrete card. + """ + flags = [False] * count + try: + lib.ggml_backend_vk_reg.restype = ctypes.c_void_p + lib.ggml_backend_vk_reg.argtypes = [] + base.ggml_backend_reg_dev_count.restype = ctypes.c_size_t + base.ggml_backend_reg_dev_count.argtypes = [ctypes.c_void_p] + base.ggml_backend_reg_dev_get.restype = ctypes.c_void_p + base.ggml_backend_reg_dev_get.argtypes = [ctypes.c_void_p, ctypes.c_size_t] + base.ggml_backend_dev_type.restype = ctypes.c_int + base.ggml_backend_dev_type.argtypes = [ctypes.c_void_p] + + reg = lib.ggml_backend_vk_reg() + if not reg: + return flags + dev_count = base.ggml_backend_reg_dev_count(reg) + for i in range(min(count, dev_count)): + dev = base.ggml_backend_reg_dev_get(reg, i) + if dev: + flags[i] = base.ggml_backend_dev_type(dev) == _GGML_BACKEND_DEVICE_TYPE_IGPU + except Exception: + # Best-effort: any failure degrades to "discrete" so the memory + # readings still get through instead of crashing the probe. + pass + return flags + + +def main() -> int: + if len(sys.argv) < 2: + return 0 + bindir = sys.argv[1] + + # Hold add_dll_directory's handle for the rest of main() (the documented + # idiom) so bindir stays on the search path while the sibling ggml DLLs + # resolve below. + _dll_dir = None + if sys.platform == "win32": + base_name, vk_name = "ggml-base.dll", "ggml-vulkan.dll" + try: + _dll_dir = os.add_dll_directory(bindir) + except Exception: + pass + else: + base_name, vk_name = "libggml-base.so", "libggml-vulkan.so" + + # RTLD_GLOBAL exposes ggml-base's symbols to ggml-vulkan on POSIX. getattr + # falls back to 0 where the flag doesn't exist (Windows CDLL ignores mode). + _rtld_global = getattr(ctypes, "RTLD_GLOBAL", 0) + try: + base = ctypes.CDLL(os.path.join(bindir, base_name), mode = _rtld_global) + lib = ctypes.CDLL(os.path.join(bindir, vk_name), mode = _rtld_global) + except OSError as e: + print(f"ggml-vulkan load failed: {e}", file = sys.stderr) + return 1 + + lib.ggml_backend_vk_get_device_count.restype = ctypes.c_int + lib.ggml_backend_vk_get_device_count.argtypes = [] + lib.ggml_backend_vk_get_device_memory.restype = None + lib.ggml_backend_vk_get_device_memory.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t), + ] + + count = lib.ggml_backend_vk_get_device_count() + igpu = _igpu_flags(base, lib, count) + rows = [] + for i in range(count): + free, total = ctypes.c_size_t(0), ctypes.c_size_t(0) + lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total)) + rows.append("%d\t%d\t%d\t%d" % (i, free.value, int(igpu[i]), total.value)) + sys.stdout.write("\n".join(rows)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index f61402aa5c..3ba9eff857 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1436,6 +1436,50 @@ def _backfill_usage_from_timings(usage, timings): return out +def _vulkan_lib_filename() -> str: + return "ggml-vulkan.dll" if sys.platform == "win32" else "libggml-vulkan.so" + + +# Host RAM to leave free on an integrated GPU, matching llama.cpp's own --fit +# margin (default 1024 MiB per device). ggml reports an iGPU's "VRAM" as shared +# system RAM, so hold back the same margin rather than inventing a larger one. +_IGPU_HOST_RESERVE_MIB = 1024 + + +def _apply_igpu_host_reserve_mib(free_mib: int, is_igpu: bool) -> int: + """Reserve host headroom on an integrated (shared-memory) Vulkan GPU. + + An iGPU's reported free "VRAM" is really free system RAM, so sizing + context/offload against all of it would push the host into swap or the OOM + killer. Leave the same margin llama.cpp's --fit uses. ``is_igpu`` comes from + ggml's device type, so a discrete card is never touched; only ever reduces. + """ + if not is_igpu: + return free_mib + return max(0, free_mib - _IGPU_HOST_RESERVE_MIB) + + +def _llama_lib_dir(binary: str) -> Path: + # The installer exposes llama-server as a top-level entrypoint into build/bin/, + # where the ggml backend libs live, so callers looking for sibling libs (Vulkan + # detection, LD_LIBRARY_PATH, probe bindir) need the real dir. It is normally a + # symlink (resolve() reaches build/bin), but create_exec_entrypoint falls back to + # a shell wrapper (exec "$(dirname "$0")/build/bin/llama-server" "$@") when it + # cannot symlink, and resolve() stops at the wrapper file. Follow the wrapper's + # exec target too, so a wrapper-based install still finds build/bin. + resolved = Path(binary).resolve() + try: + with open(resolved, "rb") as _f: + _head = _f.read(256) + if _head.startswith(b"#!"): + _m = re.search(r'exec "\$\(dirname "\$0"\)/([^"]+)"', _head.decode("utf-8", "ignore")) + if _m: + return (resolved.parent / _m.group(1)).resolve().parent + except OSError: + pass + return resolved.parent + + def _is_external_link(path: Path) -> bool: """True when ``path`` is a --with-llama-cpp-dir local link: a POSIX symlink or a Windows directory junction / reparse point. Such a link resolves into @@ -2278,6 +2322,30 @@ class LlamaCppBackend: return total + @staticmethod + def _is_vulkan_backend(binary: Optional[str] = None) -> bool: + """True if the installed llama.cpp build is Vulkan-only. + + The official prebuilts are single-backend, so the Vulkan ggml lib next + to llama-server identifies a Vulkan build. Keeps the free-memory probe + and GPU pin in ggml's Vulkan device-index space. For a custom + multi-backend build with a CUDA or HIP ggml lib alongside Vulkan, defer + to that backend (torch-usable, better-understood probe/pin). + """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if not binary: + return False + lib_dir = _llama_lib_dir(binary) + if not (lib_dir / _vulkan_lib_filename()).is_file(): + return False + for _backend in ("cuda", "hip"): + sibling = ( + f"ggml-{_backend}.dll" if sys.platform == "win32" else f"libggml-{_backend}.so" + ) + if (lib_dir / sibling).is_file(): + return False + return True + @staticmethod def _resolve_visible_physical_ids() -> Optional[list[int]]: """Physical GPU ids behind the active visibility mask (HIP/ROCR/CUDA on @@ -2440,11 +2508,42 @@ class LlamaCppBackend: return True @staticmethod - def _get_gpu_free_memory() -> list[tuple[int, int]]: + def _visible_devices_mask(env_name: str) -> Optional[set[int]]: + """Physical indices a ``*_VISIBLE_DEVICES`` mask permits, or None if unset. + + ``if x.strip()`` filters trailing-comma masks ("0,1,"); an empty mask + ("") yields an empty set (all devices hidden), distinct from an unset + var (None, no mask). Used by the nvidia-smi probe. + """ + raw = os.environ.get(env_name) + if raw is None: + return None + try: + return set(int(x.strip()) for x in raw.split(",") if x.strip()) + except ValueError: + return None + + @staticmethod + def _vulkan_pin_args(gpu_indices: Optional[Iterable[int]]) -> list[str]: + """``--device Vulkan,...`` to pin a Vulkan launch to selected GPUs. + + The indices are ggml's compact Vulkan ordinals (as _get_gpu_free_memory + reports and the registry names ``Vulkan``). Pin by that name, NOT via + GGML_VK_VISIBLE_DEVICES: ggml parses that env var in the raw + vkEnumeratePhysicalDevices space (before dropping CPU/llvmpipe devices + and deduplicating ICDs), so a compact ordinal there could select a + different physical device or the CPU rasterizer. + """ + if not gpu_indices: + return [] + return ["--device", ",".join(f"Vulkan{i}" for i in gpu_indices)] + + @staticmethod + def _get_gpu_free_memory(binary: Optional[str] = None) -> list[tuple[int, int]]: """Query free memory per GPU. Returns ``(gpu_index, free_mib)`` sorted by index; empty if no supported GPU is reachable. Thin wrapper over ``_get_gpu_memory`` for callers that only need free VRAM.""" - return [(idx, free) for idx, free, _total in LlamaCppBackend._get_gpu_memory()] + return [(idx, free) for idx, free, _total in LlamaCppBackend._get_gpu_memory(binary)] @staticmethod def _apple_metal_memory_budget_bytes() -> int: @@ -2475,7 +2574,7 @@ class LlamaCppBackend: return int(rec_bytes * _APPLE_UNIFIED_MEMORY_FRACTION) @staticmethod - def _get_gpu_memory() -> list[tuple[int, int, int]]: + def _get_gpu_memory(binary: Optional[str] = None) -> list[tuple[int, int, int]]: """Query free AND total memory per GPU. Order: @@ -2487,9 +2586,18 @@ class LlamaCppBackend: probe returned [] on AMD) and NVIDIA hosts missing ``nvidia-smi`` from PATH. + On a Vulkan build the ggml Vulkan probe is authoritative, so the indices + are ggml's compact Vulkan ordinals (the space the pin selects via + ``--device Vulkan``). It reports ``total`` for discrete cards and 0 + for an iGPU (shared RAM) so the fit falls back to free*frac there. + Otherwise nvidia-smi / torch cover NVIDIA + AMD ROCm. + Returns (gpu_index, free_mib, total_mib) sorted by index; empty if no - supported GPU is reachable. ``total`` lets the fit reserve absolute headroom. + supported GPU is reachable. """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if LlamaCppBackend._is_vulkan_backend(binary): + return LlamaCppBackend._get_gpu_free_memory_vulkan(binary) # ── NVIDIA via nvidia-smi ──────────────────────────────────── try: result = subprocess.run( @@ -2505,16 +2613,7 @@ class LlamaCppBackend: **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: - allowed: Optional[set[int]] = None - cvd = os.environ.get("CUDA_VISIBLE_DEVICES") - if cvd is not None: - try: - # `if x.strip()` filters trailing-comma masks ("0,1,"). - # Empty mask (CVD="") yields an empty set -> all GPUs - # filtered out, per codebase convention. - allowed = set(int(x.strip()) for x in cvd.split(",") if x.strip()) - except ValueError: - pass + allowed = LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES") gpus: list[tuple[int, int, int]] = [] for line in result.stdout.strip().splitlines(): parts = [p.strip() for p in line.split(",")] @@ -2579,6 +2678,91 @@ class LlamaCppBackend: logger.debug(f"torch GPU probe failed: {e}") return [] + @staticmethod + def _get_gpu_free_memory_vulkan(binary: Optional[str] = None) -> list[tuple[int, int, int]]: + """Query free (and total) VRAM per device via the bundled ggml Vulkan backend. + + Loads ``libggml-vulkan`` in a short-lived subprocess (no Vulkan instance + in this process) and returns (device_index, free_mib, total_mib) sorted + by index. The index is ggml's compact Vulkan ordinal -- the one the + registry names ``Vulkan`` and load_model pins with ``--device``, + NOT the raw ``GGML_VK_VISIBLE_DEVICES`` space. A user-set + ``GGML_VK_VISIBLE_DEVICES`` is honored by ggml (passed through), so the + list already reflects it. iGPUs leave a host-RAM margin (see + ``_apply_igpu_host_reserve_mib``) and report total 0; discrete cards pass + their real total through. [] when no Vulkan build or device is reachable. + """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if not binary: + return [] + binary_dir = _llama_lib_dir(binary) + if not (binary_dir / _vulkan_lib_filename()).is_file(): + return [] + + env = child_env_without_native_path_secret() + # Pass any inherited GGML_VK_VISIBLE_DEVICES through to ggml unchanged so + # the probe enumerates the same device list the launch will, named + # Vulkan0..N in the compact order reported here and pinned by that name + # via --device -- probe, mask, and pin stay in one index space. Do NOT + # filter the mask in Python: ggml parses the env var in raw + # vkEnumeratePhysicalDevices space while this probe reports the compact + # post-filter ordinal, so a Python filter would compare mismatched spaces. + if sys.platform != "win32": + # Let the loader resolve sibling ggml libs next to the binary. + existing_ld = env.get("LD_LIBRARY_PATH", "") + env["LD_LIBRARY_PATH"] = ( + f"{binary_dir}:{existing_ld}" if existing_ld else str(binary_dir) + ) + probe_script = Path(__file__).with_name("_vulkan_probe.py") + try: + result = subprocess.run( + [sys.executable, str(probe_script), str(binary_dir)], + capture_output = True, + text = True, + timeout = 15, + env = env, + **_windows_hidden_subprocess_kwargs(), + ) + if result.returncode != 0: + logger.debug( + f"vulkan GPU probe exited {result.returncode}: {result.stderr.strip()}" + ) + return [] + except Exception as e: + logger.debug(f"vulkan GPU probe failed: {e}") + return [] + + gpus: list[tuple[int, int, int]] = [] + for line in result.stdout.strip().splitlines(): + parts = line.split("\t") + if len(parts) != 4: + continue + try: + idx = int(parts[0]) + free_mib = int(parts[1]) // (1024 * 1024) + is_igpu = parts[2] == "1" + # iGPU "total" is shared RAM, not a VRAM budget -> keep 0 so the + # fit stays on free*frac (the host reserve below is its + # headroom); a discrete card passes its real total through. + total_mib = 0 if is_igpu else int(parts[3]) // (1024 * 1024) + except ValueError: + continue + capped = _apply_igpu_host_reserve_mib(free_mib, is_igpu) + if capped < free_mib: + logger.info( + f"Vulkan device VK{idx} is an integrated GPU sharing system " + f"RAM; reserving {free_mib - capped}MiB host headroom " + f"({free_mib}->{capped}MiB usable)" + ) + gpus.append((idx, capped, total_mib)) + gpus.sort(key = lambda g: g[0]) + if gpus: + logger.info( + "Vulkan GPU memory detected: " + + ", ".join(f"VK{idx}={free}MiB" for idx, free, _total in gpus) + ) + return gpus + @staticmethod def _available_system_memory_mib() -> Optional[int]: """Available system RAM in MiB (psutil, then /proc/meminfo), or None if @@ -2807,7 +2991,8 @@ class LlamaCppBackend: def _llama_server_env_for_binary(binary: str) -> dict[str, str]: """Build a subprocess env that lets llama-server resolve native libs.""" env = child_env_without_native_path_secret() - binary_dir = str(Path(binary).parent) + # _llama_lib_dir resolves the llama-server symlink to the real build/bin. + binary_dir = str(_llama_lib_dir(binary)) if sys.platform == "win32": # Ordering: see _build_windows_path_dirs. #5106. @@ -5210,6 +5395,7 @@ class LlamaCppBackend: # Resolve llama-server now but defer a not-found error: a block-diffusion # GGUF uses the diffusion runner, and its arch is only known after the header. binary = self._find_llama_server_binary() + is_vulkan_backend = self._is_vulkan_backend(binary) # ── Phase 2: download (NO lock held, so cancel can proceed) ── # mtp_draft_path arrives set for local Gemma loads (detected @@ -5449,7 +5635,8 @@ class LlamaCppBackend: model_size = gguf_size + mmproj_size # 2-tuple gpus for existing logic + a total map for the absolute # per-GPU headroom (correct when the GPU is already partly used). - _gpu_mem = self._get_gpu_memory() + # Pass binary so a Vulkan build probes ggml's Vulkan ordinals. + _gpu_mem = self._get_gpu_memory(binary) gpus = [(idx, free) for idx, free, _t in _gpu_mem] total_by_idx = {idx: total for idx, _f, total in _gpu_mem} @@ -6222,7 +6409,12 @@ class LlamaCppBackend: # cap, not the ROCm-reported VRAM, is the real ceiling); refuse an # oversize load the OS would otherwise kill mid-flight. Base model # only: an optional MTP drafter is dropped by the MTP-drop fallback. - if model_size is not None and self._amd_apu_wants_unified_memory(gpu_indices): + # CUDA/ROCm ids only; a Vulkan build's gpu_indices are ggml ordinals. + if ( + model_size is not None + and not is_vulkan_backend + and self._amd_apu_wants_unified_memory(gpu_indices) + ): _ram_msg = self._apu_ram_shortfall_message( model_size, self._available_system_memory_mib() ) @@ -6485,6 +6677,12 @@ class LlamaCppBackend: ", ".join(unsupported_cache_flags), ) + # Vulkan pins via --device (a cmd arg, unlike the env-based + # CUDA/ROCm pin below), emitted BEFORE user extras so llama.cpp's + # last-wins parsing lets a user --device override Studio's pick. + if is_vulkan_backend and gpu_indices is not None: + cmd += LlamaCppBackend._vulkan_pin_args(gpu_indices) + # User pass-through args go last so llama.cpp's last-wins parsing # lets the user override Studio's auto-set flags. Already # validated by the route via validate_extra_args(). @@ -6536,23 +6734,25 @@ class LlamaCppBackend: env.setdefault("OMP_NUM_THREADS", "2") # AMD unified-memory APUs (gfx1150/gfx1151): let llama.cpp use - # shared system RAM. setdefault so a user value wins. - if self._amd_apu_wants_unified_memory(gpu_indices): + # shared system RAM. setdefault so a user value wins. Not on Vulkan + # (nor DC below): gpu_indices are ggml ordinals, not CUDA/ROCm ids. + if not is_vulkan_backend and self._amd_apu_wants_unified_memory(gpu_indices): env.setdefault("GGML_CUDA_ENABLE_UNIFIED_MEMORY", "1") logger.info("AMD unified-memory APU: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1") # DC NVIDIA GPUs: FP32 accum (+ P2P / launch queues for multi-GPU). # See _apply_datacenter_env; opt out with UNSLOTH_DISABLE_DC_TUNING=1. - if self._apply_datacenter_env(env, gpu_indices): + if not is_vulkan_backend and self._apply_datacenter_env(env, gpu_indices): multi_gpu = self._effective_gpu_count(gpu_indices) > 1 logger.info( f"Data-center GPU detected: applied DC llama.cpp env tuning (multi_gpu={multi_gpu})" ) # Pin to selected GPU(s). On ROCm, narrowing only - # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full - # set, so set HIP_VISIBLE_DEVICES too. - if gpu_indices is not None: + # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full set, so + # set HIP_VISIBLE_DEVICES too. Vulkan is pinned via --device + # (above), not here. + if gpu_indices is not None and not is_vulkan_backend: pinned = ",".join(str(i) for i in gpu_indices) env["CUDA_VISIBLE_DEVICES"] = pinned try: diff --git a/studio/backend/tests/test_install_resolve_prebuilt.py b/studio/backend/tests/test_install_resolve_prebuilt.py index b825172a63..090d2932ea 100644 --- a/studio/backend/tests/test_install_resolve_prebuilt.py +++ b/studio/backend/tests/test_install_resolve_prebuilt.py @@ -55,6 +55,27 @@ def _host(**kw): return ilp.HostInfo(**base) +def test_force_cpu_clears_all_gpu_attributes_including_intel(): + # --cpu-fallback is the "select the CPU prebuilt even when a GPU is present" + # escape hatch. It must drop EVERY GPU attribute, including has_intel_gpu, or + # the planner still prepends the Vulkan asset on an Intel-GPU host. + host = _host( + is_linux = True, + is_x86_64 = True, + has_usable_nvidia = True, + has_physical_nvidia = True, + has_rocm = True, + rocm_gfx_target = "gfx1100", + has_intel_gpu = True, + ) + forced = ilp._apply_host_overrides(host, force_cpu = True) + assert forced.has_usable_nvidia is False + assert forced.has_physical_nvidia is False + assert forced.has_rocm is False + assert forced.rocm_gfx_target is None + assert forced.has_intel_gpu is False + + def test_macos_upstream_pin_only_for_explicit_pre26_upstream(): pre26 = _host( system = "Darwin", @@ -313,3 +334,152 @@ def test_sm103_host_drops_cuda128_windows_build(): ) kept_b200 = ilp._drop_blackwell_incapable_windows_cuda(b200, [cuda128, cuda129]) assert [a.name for a in kept_b200] == [cuda128.name, cuda129.name] + + +def _upstream_release(tag, asset_names): + return { + "tag_name": tag, + "assets": [ + {"name": n, "browser_download_url": f"https://example/{n}"} for n in asset_names + ], + } + + +def test_direct_upstream_arm64_intel_prefers_vulkan(): + # Auto-detected Intel GPU on Linux arm64 -> Vulkan prebuilt first, CPU + # second (mirrors the x86_64 branch; ggml-org ships the arm64 Vulkan asset). + host = _host(is_linux = True, is_arm64 = True, machine = "aarch64", has_intel_gpu = True) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", "llama-b9925-bin-ubuntu-arm64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + kinds = [a.install_kind for a in plan.attempts] + assert kinds[0] == "linux-vulkan", kinds + assert "linux-arm64" in kinds + assert plan.attempts[0].name == "llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz" + + +def test_direct_upstream_intel_with_hidden_nvidia_is_cpu_only(): + # A host with a physical NVIDIA hidden via CUDA_VISIBLE_DEVICES (physical + # True, usable False) + an Intel iGPU must NOT get the Vulkan archive even + # when planning directly against upstream: Vulkan ignores CUDA_VISIBLE_DEVICES + # and could grab the reserved card. It falls through to the CPU asset. + host = _host( + is_linux = True, + is_x86_64 = True, + has_intel_gpu = True, + has_physical_nvidia = True, + has_usable_nvidia = False, + ) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-x64.tar.gz", "llama-b9925-bin-ubuntu-x64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] + + +def test_direct_upstream_arm64_without_intel_is_cpu_only(): + host = _host(is_linux = True, is_arm64 = True, machine = "aarch64") + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", "llama-b9925-bin-ubuntu-arm64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + assert [a.install_kind for a in plan.attempts] == ["linux-arm64"] + + +def test_direct_upstream_x86_intel_prefers_vulkan(): + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-x64.tar.gz", "llama-b9925-bin-ubuntu-x64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + kinds = [a.install_kind for a in plan.attempts] + assert kinds[0] == "linux-vulkan", kinds + assert "linux-cpu" in kinds + + +def test_linux_vulkan_health_glob_matches_bare_cpu_lib(): + # The widened glob must cover both arch-suffixed (x64) and bare (arm64) CPU + # libs so a valid Vulkan install is not re-flagged unhealthy every check. + choice = ilp.AssetChoice( + repo = UPSTREAM, + tag = "b9925", + name = "llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", + url = "https://example/x", + source_label = "upstream", + install_kind = "linux-vulkan", + ) + groups = ilp.runtime_payload_health_groups(choice) + assert ["libggml-cpu*.so*"] in groups + assert ["libggml-cpu-*.so*"] not in groups + + +def test_route_to_vulkan_prebuilt_auto_intel_goes_upstream_and_drops_fork_pin(): + # Routing fork -> upstream also drops the fork release pin, which is in a + # different tag namespace and would make the upstream resolver miss. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, FORK, "b9596-mix-abc", force_cpu = False) + assert repo == UPSTREAM + assert tag == "" + assert routed.has_intel_gpu is True + + +def test_route_to_vulkan_prebuilt_preserves_explicit_upstream_pin(): + # A pin set WITH an explicit upstream repo is already on upstream -> kept. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + _routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, UPSTREAM, "b9596", force_cpu = False) + assert repo == UPSTREAM + assert tag == "b9596" + + +def test_route_to_vulkan_prebuilt_cpu_fallback_wins(): + # --cpu-fallback suppresses Vulkan routing even for an Intel host. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, FORK, "b9596-mix-abc", force_cpu = True) + assert repo == FORK + assert tag == "b9596-mix-abc" + assert routed is host + + +def test_route_to_vulkan_prebuilt_hidden_nvidia_not_rerouted(): + # A mixed NVIDIA+Intel host that hid NVIDIA (CUDA_VISIBLE_DEVICES=""/-1): + # physical NVIDIA present but not usable. Must NOT auto-route to Vulkan, or + # Vulkan (which ignores CUDA_VISIBLE_DEVICES) could grab the reserved GPU. + host = _host( + is_linux = True, + is_x86_64 = True, + has_intel_gpu = True, + has_physical_nvidia = True, + has_usable_nvidia = False, + ) + _routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + + +def test_route_to_vulkan_prebuilt_rocm_host_not_rerouted(): + # An Intel iGPU alongside a usable ROCm GPU stays on its ROCm/fork path. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True, has_rocm = True) + _routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + + +def test_route_to_vulkan_prebuilt_non_intel_unchanged(): + host = _host(is_linux = True, is_x86_64 = True) + routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + assert routed is host + + +def test_resolve_prebuilt_intel_host_routes_to_upstream(monkeypatch, capsys): + # The --resolve-prebuilt probe must agree with the install path: an + # auto-detected Intel host resolves against upstream (Vulkan), not the fork. + monkeypatch.setattr( + ilp, "detect_host", lambda: _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + ) + seen, out = _run_resolve_capture_host(monkeypatch, capsys) + assert seen["repo"] == UPSTREAM + assert out["repo"] == UPSTREAM diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index 5138e90471..f405ebcbd1 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -448,6 +448,48 @@ def test_start_update_happy_path(monkeypatch, tmp_path): assert popen_kwargs["env"]["UNSLOTH_PROGRESS_PERCENT_STEP"] == "5" +def test_start_update_preserves_vulkan_via_env(monkeypatch, tmp_path): + # A Vulkan install (marker asset carries 'vulkan') must re-assert + # UNSLOTH_FORCE_VULKAN on update, or detect_host on a GPU box re-routes to + # CUDA/ROCm and silently replaces the Vulkan build. + install_dir = tmp_path / "llama.cpp" + binary = _write_install( + install_dir, + "b9493", + repo = "ggml-org/llama.cpp", + asset = "llama-b9493-bin-ubuntu-vulkan-x64.tar.gz", + ) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518") + + def _on_start(cmd): + _write_install( + install_dir, + "b9518", + repo = "ggml-org/llama.cpp", + asset = "llama-b9518-bin-ubuntu-vulkan-x64.tar.gz", + ) + + popen_kwargs: dict = {} + _patch_installer_popen( + monkeypatch, + lines = ["installed\n"], + on_start = _on_start, + captured_kwargs = popen_kwargs, + ) + + assert upd.start_update()["started"] is True + deadline = time.time() + 10 + while time.time() < deadline: + job = upd.get_update_status()["job"] + if job["state"] in ("success", "error"): + break + time.sleep(0.05) + assert job["state"] == "success", job + assert popen_kwargs["env"]["UNSLOTH_FORCE_VULKAN"] == "1" + + def test_start_update_reports_full_release_tag(monkeypatch, tmp_path): install_dir = tmp_path / "llama.cpp" binary = _write_install(install_dir, "b9595") diff --git a/studio/backend/tests/test_llama_cpp_vulkan_probe.py b/studio/backend/tests/test_llama_cpp_vulkan_probe.py new file mode 100644 index 0000000000..92aaab4873 --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_vulkan_probe.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Vulkan free-VRAM reader regression tests on a synthetic probe output. + +Covers the post-probe handling in +``LlamaCppBackend._get_gpu_free_memory_vulkan``: + + * integrated GPUs (probe reports is_igpu=1) leave a flat per-device host + margin matching llama.cpp's --fit-target, so context auto-sizing can't + over-commit shared RAM, and report total 0 (shared RAM is not a budget), + * discrete GPUs (is_igpu=0) keep their free untouched and pass their real + total through so the fit can reserve absolute headroom, + * an inherited ``GGML_VK_VISIBLE_DEVICES`` is passed through to ggml unchanged + (ggml applies it), not stripped or filtered in Python -- the probe reports + ggml's compact ordinal, which load_model pins with ``--device Vulkan``. + +The ggml Vulkan library is never loaded: subprocess.run is mocked to emit +the tab-separated lines the real ``_vulkan_probe.py`` would print. +""" + +from __future__ import annotations + +import subprocess +import sys +import types as _types +from pathlib import Path +from unittest import mock + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +import importlib as _importlib # noqa: E402 + + +def _maybe_stub(name: str, builder): + try: + _importlib.import_module(name) + except ImportError: + sys.modules[name] = builder() + + +def _build_loggers_stub(): + m = _types.ModuleType("loggers") + m.get_logger = lambda name: __import__("logging").getLogger(name) + return m + + +_maybe_stub("loggers", _build_loggers_stub) +_maybe_stub("structlog", lambda: _types.ModuleType("structlog")) + +from core.inference import llama_cpp as _llama_mod # noqa: E402 +from core.inference.llama_cpp import ( # noqa: E402 + LlamaCppBackend, + _llama_lib_dir, + _vulkan_lib_filename, +) + +MIB = 1024 * 1024 +GIB = 1024 * MIB + + +def _make_vulkan_install(tmp_path: Path) -> str: + """A binary whose sibling dir holds the Vulkan ggml lib, so the + reader's ``is_vulkan_backend`` sibling-file check passes.""" + bindir = tmp_path / "build" / "bin" + bindir.mkdir(parents = True) + binary = bindir / ("llama-server.exe" if sys.platform == "win32" else "llama-server") + binary.write_bytes(b"stub") + (bindir / _vulkan_lib_filename()).write_bytes(b"stub") + return str(binary) + + +def _mock_probe(rows: list[str], captured_env: dict | None = None): + """Patch subprocess.run so the _vulkan_probe.py call returns ``rows`` + (already tab-formatted), recording the env it was launched with.""" + real_run = subprocess.run + + def fake_run(cmd, *args, **kwargs): + if isinstance(cmd, list) and any("_vulkan_probe" in str(c) for c in cmd): + if captured_env is not None: + captured_env.clear() + captured_env.update(kwargs.get("env") or {}) + return subprocess.CompletedProcess( + args = cmd, returncode = 0, stdout = "\n".join(rows), stderr = "" + ) + return real_run(cmd, *args, **kwargs) + + return mock.patch("subprocess.run", side_effect = fake_run) + + +def _row( + idx: int, + free_bytes: int, + is_igpu: int, + total_bytes: int = 0, +) -> str: + return f"{idx}\t{free_bytes}\t{is_igpu}\t{total_bytes}" + + +def test_integrated_gpu_leaves_host_margin(tmp_path): + binary = _make_vulkan_install(tmp_path) + # iGPU with 30 GiB free; reserve a flat 1024 MiB (llama.cpp --fit-target). + # total stays 0: shared system RAM is not a VRAM budget for the fit. + rows = [_row(0, 30 * GIB, is_igpu = 1, total_bytes = 32 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 30 * 1024 - 1024, 0)], gpus + + +def test_discrete_gpu_free_is_untouched_and_total_passed_through(tmp_path): + binary = _make_vulkan_install(tmp_path) + # 6 GiB free on a partially occupied 24 GiB card: free is untouched and the + # real total flows through so the fit reserves absolute headroom (CUDA/ROCm + # parity) instead of the looser free*frac budget. + rows = [_row(0, 6 * GIB, is_igpu = 0, total_bytes = 24 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 6 * 1024, 24 * 1024)], gpus + + +def test_large_discrete_gpu_is_untouched(tmp_path): + binary = _make_vulkan_install(tmp_path) + # A 48 GiB discrete card stays untouched regardless of size; only the + # iGPU flag triggers the host margin, never a VRAM/RAM ratio. + rows = [_row(0, 47 * GIB, is_igpu = 0, total_bytes = 48 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 47 * 1024, 48 * 1024)], gpus + + +def test_inherited_visible_devices_mask_is_passed_through_to_probe(tmp_path, monkeypatch): + # The mask is NOT stripped or filtered in Python: ggml parses it in raw + # physical-device space while this probe reports the compact post-filter + # ordinal, so mixing spaces would be wrong. It is passed through unchanged + # so ggml applies it to the same device list the launch will enumerate. + binary = _make_vulkan_install(tmp_path) + monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", "1") + captured: dict = {} + rows = [_row(0, 23 * GIB, is_igpu = 0, total_bytes = 24 * GIB)] + with _mock_probe(rows, captured_env = captured): + LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert captured.get("GGML_VK_VISIBLE_DEVICES") == "1", captured + + +def test_vulkan_pin_args_uses_device_names_not_env_mask(): + # Pin by compact device name via --device (the space the probe reports and + # the registry names), never by writing a compact ordinal into the raw + # GGML_VK_VISIBLE_DEVICES index space. + assert LlamaCppBackend._vulkan_pin_args([0]) == ["--device", "Vulkan0"] + assert LlamaCppBackend._vulkan_pin_args([1, 2]) == ["--device", "Vulkan1,Vulkan2"] + assert LlamaCppBackend._vulkan_pin_args(None) == [] + assert LlamaCppBackend._vulkan_pin_args([]) == [] + + +def test_vulkan_only_build_is_detected(tmp_path): + binary = _make_vulkan_install(tmp_path) + assert LlamaCppBackend._is_vulkan_backend(binary) is True + + +def test_multi_backend_build_is_not_vulkan_only(tmp_path): + # A custom build that ships CUDA (or HIP) alongside Vulkan must NOT be + # treated as Vulkan-only, or its CUDA GPU would be probed/pinned as a Vulkan + # device; defer to the CUDA/HIP path instead. + binary = _make_vulkan_install(tmp_path) + cuda = "ggml-cuda.dll" if sys.platform == "win32" else "libggml-cuda.so" + (_llama_lib_dir(binary) / cuda).write_bytes(b"stub") + assert LlamaCppBackend._is_vulkan_backend(binary) is False + + +@pytest.mark.skipif(sys.platform == "win32", reason = "shell wrapper fallback is POSIX") +def test_shell_wrapper_entrypoint_resolves_to_real_lib_dir(tmp_path): + # create_exec_entrypoint falls back to a #!/bin/sh wrapper at the install root + # when it cannot symlink; _find_llama_server_binary returns that root entrypoint, + # so _llama_lib_dir must follow the wrapper's exec target to build/bin -- else + # _is_vulkan_backend misses libggml-vulkan.so and the Vulkan probe/pin silently + # never engage on a valid Vulkan install. + import os + + binary = _make_vulkan_install(tmp_path) # tmp_path/build/bin/llama-server + vulkan lib + bindir = Path(binary).parent + wrapper = tmp_path / "llama-server" + wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n') + os.chmod(wrapper, 0o755) + assert _llama_lib_dir(str(wrapper)) == bindir + assert LlamaCppBackend._is_vulkan_backend(str(wrapper)) is True + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index c16ae91467..1bcbfbf95a 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -514,6 +514,12 @@ def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path logger.info("llama update: installing", cmd = " ".join(cmd)) # Stream progress lines into job["progress"]. env = dict(os.environ, UNSLOTH_PROGRESS_PERCENT_STEP = "5") + # Preserve a Vulkan install across updates: detect_host on a CUDA/ROCm + # box would otherwise re-route and silently replace the Vulkan build. + # Re-assert it via the same env flag setup uses (mirrors + # _rocm_install_args). + if asset and "vulkan" in asset.lower(): + env["UNSLOTH_FORCE_VULKAN"] = "1" proc = subprocess.Popen( cmd, stdout = subprocess.PIPE, diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 6c75e6c394..856ba71478 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -10,6 +10,7 @@ import argparse import atexit import errno import fnmatch +import glob import hashlib import json import os @@ -265,6 +266,7 @@ class HostInfo: has_physical_nvidia: bool has_usable_nvidia: bool has_rocm: bool = False + has_intel_gpu: bool = False rocm_gfx_target: str | None = None # (major, minor) from platform.mac_ver(); None off macOS or if unparseable. # Skips a macos prebuilt whose minimum-OS exceeds this host. @@ -1482,6 +1484,24 @@ def direct_upstream_release_plan( install_kind = "windows-hip", ) ) + # Intel (or other non-NVIDIA/non-AMD) GPU: use the Vulkan prebuilt. Gate + # on no PHYSICAL NVIDIA (not just no usable one): a host that hid NVIDIA + # via CUDA_VISIBLE_DEVICES must not reach Vulkan, which ignores that mask + # and could enumerate the reserved card. Falls through to CPU below. + elif host.has_intel_gpu and not host.has_physical_nvidia: + vulkan_asset = f"llama-{release_tag}-bin-win-vulkan-x64.zip" + vulkan_url = assets.get(vulkan_asset) + if vulkan_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = vulkan_asset, + url = vulkan_url, + source_label = "upstream", + install_kind = "windows-vulkan", + ) + ) cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip" cpu_url = assets.get(cpu_asset) if cpu_url: @@ -1545,6 +1565,23 @@ def direct_upstream_release_plan( # ROCm hosts are excluded: this ggml-org path ships no per-gfx ROCm # asset, so they fall through to the empty-attempts raise (HIP source # build) rather than silently getting a CPU binary on a GPU host. + # Intel (or other non-NVIDIA/non-AMD) GPU: use the Vulkan prebuilt. The + # elif already excludes usable NVIDIA and ROCm; also require no PHYSICAL + # NVIDIA so a CUDA-hidden card isn't reached through Vulkan (CPU below). + if host.has_intel_gpu and not host.has_physical_nvidia: + vulkan_asset = f"llama-{release_tag}-bin-ubuntu-vulkan-x64.tar.gz" + vulkan_url = assets.get(vulkan_asset) + if vulkan_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = vulkan_asset, + url = vulkan_url, + source_label = "upstream", + install_kind = "linux-vulkan", + ) + ) asset_name = f"llama-{release_tag}-bin-ubuntu-x64.tar.gz" asset_url = assets.get(asset_name) if asset_url: @@ -1564,6 +1601,23 @@ def direct_upstream_release_plan( # selector returned 0 attempts and the installer fell back to a # source build on every Linux ARM64 host (DGX Spark, Ampere # Altra, GitHub-hosted ubuntu-24.04-arm runners, etc.). + # Intel (or other non-NVIDIA/non-AMD) GPU: prefer the Vulkan prebuilt, + # mirroring the x86_64 branch. Upstream ships bin-ubuntu-vulkan-arm64. + # No physical NVIDIA: don't reach a CUDA-hidden card through Vulkan. + if host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm: + vulkan_asset = f"llama-{release_tag}-bin-ubuntu-vulkan-arm64.tar.gz" + vulkan_url = assets.get(vulkan_asset) + if vulkan_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = vulkan_asset, + url = vulkan_url, + source_label = "upstream", + install_kind = "linux-vulkan", + ) + ) asset_name = f"llama-{release_tag}-bin-ubuntu-arm64.tar.gz" asset_url = assets.get(asset_name) if asset_url: @@ -3075,6 +3129,40 @@ def detect_host() -> HostInfo: # Note: amdhip64.dll presence alone is NOT treated as GPU evidence # since the HIP SDK can be installed without an AMD GPU. + # Detect an Intel GPU; gates the Vulkan prebuilt. Linux reads the DRM sysfs + # vendor id (0x8086); Windows queries the WMI video controller list. Only + # probed with no usable NVIDIA and no ROCm (matching the Vulkan branches), + # keeping the probe (notably the Windows powershell call) off that path. + has_intel_gpu = False + if not has_usable_nvidia and not has_rocm: + if is_linux: + for _vendor_file in glob.glob("/sys/class/drm/card*/device/vendor"): + try: + with open(_vendor_file) as _vf: + if _vf.read().strip().lower() == "0x8086": + has_intel_gpu = True + break + except OSError: + continue + elif is_windows: + _ps = shutil.which("powershell") or shutil.which("pwsh") + if _ps: + try: + _result = run_capture( + [ + _ps, + "-NoProfile", + "-Command", + "Get-CimInstance Win32_VideoController | " + "Select-Object -ExpandProperty Name", + ], + timeout = 15, + ) + if _result.returncode == 0 and "intel" in _result.stdout.lower(): + has_intel_gpu = True + except Exception: + pass + return HostInfo( system = system, machine = machine, @@ -3090,6 +3178,7 @@ def detect_host() -> HostInfo: has_physical_nvidia = has_physical_nvidia, has_usable_nvidia = has_usable_nvidia, has_rocm = has_rocm, + has_intel_gpu = has_intel_gpu, rocm_gfx_target = rocm_gfx_target, macos_version = macos_version, ) @@ -3126,6 +3215,7 @@ def _apply_host_overrides( has_physical_nvidia = False, has_rocm = False, rocm_gfx_target = None, + has_intel_gpu = False, ) gfx = _normalize_forwarded_gfx(override_rocm_gfx) if gfx: @@ -3866,6 +3956,23 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice "falling back to source build with HIP support" ) + # Intel (or other non-NVIDIA/non-AMD) GPU: use the Vulkan prebuilt. No + # physical NVIDIA (not just no usable one): a CUDA-hidden card must not + # be reached through Vulkan, which ignores CUDA_VISIBLE_DEVICES. + if host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm: + vulkan_name = f"llama-{llama_tag}-bin-ubuntu-vulkan-x64.tar.gz" + if vulkan_name in upstream_assets: + log(f"Intel GPU detected -- using upstream Vulkan prebuilt {vulkan_name}") + return AssetChoice( + repo = UPSTREAM_REPO, + tag = llama_tag, + name = vulkan_name, + url = upstream_assets[vulkan_name], + source_label = "upstream", + install_kind = "linux-vulkan", + ) + log("Intel GPU detected but no Vulkan prebuilt found -- falling back to CPU") + upstream_name = f"llama-{llama_tag}-bin-ubuntu-x64.tar.gz" if upstream_name not in upstream_assets: raise PrebuiltFallback("upstream Linux CPU asset was not found") @@ -3908,6 +4015,24 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice ) log("AMD ROCm detected on Windows but no HIP prebuilt found -- falling back to CPU") + # Intel (or other non-NVIDIA/non-AMD) GPU on Windows: use Vulkan. No + # physical NVIDIA so a CUDA-hidden card isn't reached through Vulkan. + if host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm: + vulkan_name = f"llama-{llama_tag}-bin-win-vulkan-x64.zip" + if vulkan_name in upstream_assets: + log( + f"Intel GPU detected on Windows -- using upstream Vulkan prebuilt {vulkan_name}" + ) + return AssetChoice( + repo = UPSTREAM_REPO, + tag = llama_tag, + name = vulkan_name, + url = upstream_assets[vulkan_name], + source_label = "upstream", + install_kind = "windows-vulkan", + ) + log("Intel GPU detected on Windows but no Vulkan prebuilt found -- falling back to CPU") + upstream_name = f"llama-{llama_tag}-bin-win-cpu-x64.zip" if upstream_name not in upstream_assets: raise PrebuiltFallback("upstream Windows CPU asset was not found") @@ -4503,6 +4628,7 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: "linux-arm64-cuda", "linux-rocm", "linux-arm64", + "linux-vulkan", }: return ["llama-server", "llama-quantize", "llama-diffusion-gemma-visual-server", "lib*.so*"] if choice.install_kind in {"macos-arm64", "macos-x64"}: @@ -4516,6 +4642,7 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: "windows-cpu", "windows-cuda", "windows-hip", + "windows-vulkan", "windows-rocm", "windows-arm64", }: @@ -5731,8 +5858,10 @@ def validate_server( "linux-cuda", "linux-arm64-cuda", "linux-rocm", + "linux-vulkan", "windows-cuda", "windows-hip", + "windows-vulkan", "windows-rocm", "macos-arm64", } @@ -6354,6 +6483,20 @@ def runtime_payload_health_groups(choice: AssetChoice) -> list[list[str]]: ["libmtmd.so*"], ["libggml-hip.so*"], ] + if choice.install_kind == "linux-vulkan": + return [ + ["libllama-common.so*"], + ["libllama.so*"], + ["libggml.so*"], + ["libggml-base.so*"], + # Match the sibling globs (linux-cuda/-rocm): x64 bundles ship + # arch-suffixed libggml-cpu-.so, arm64 may ship a bare + # libggml-cpu.so; the '-' form missed the latter and re-flagged + # the install unhealthy on every check. + ["libggml-cpu*.so*"], + ["libmtmd.so*"], + ["libggml-vulkan.so*"], + ] if choice.install_kind in {"windows-cpu", "windows-arm64"}: return [["llama.dll"]] if choice.install_kind == "windows-cuda": @@ -6373,6 +6516,8 @@ def runtime_payload_health_groups(choice: AssetChoice) -> list[list[str]]: return groups if choice.install_kind in {"windows-hip", "windows-rocm"}: return [["llama.dll"], ["*hip*.dll"]] + if choice.install_kind == "windows-vulkan": + return [["llama.dll"], ["ggml-vulkan.dll"]] return [] @@ -6654,6 +6799,89 @@ def validate_prebuilt_attempts( raise PrebuiltFallback("no prebuilt bundle passed validation") +def force_vulkan_requested() -> bool: + """Whether UNSLOTH_FORCE_VULKAN opts this host into the Vulkan llama.cpp + prebuilt instead of its detected CUDA/ROCm backend (e.g. so an AMD user can + run the Vulkan build for inference). Scoped to the llama.cpp backend; the + torch/training stack installs separately and still sees the real GPU. + """ + return os.environ.get("UNSLOTH_FORCE_VULKAN", "").strip().lower() in ( + "1", + "true", + "yes", + ) + + +def _vulkan_only_host(host: HostInfo) -> HostInfo: + """Rewrite ``host`` so the asset selectors take their Vulkan branch. + + That branch fires on ``has_intel_gpu and not nvidia and not rocm``, so clear + the CUDA/ROCm flags and raise the integrated-GPU flag. The synthetic flag + never leaves install planning -- it only routes the llama.cpp prebuilt + choice, not the torch/training stack. + """ + return dataclasses_replace( + host, + has_usable_nvidia = False, + has_physical_nvidia = False, + has_rocm = False, + has_intel_gpu = True, + ) + + +def _route_to_vulkan_prebuilt( + host: HostInfo, published_repo: str, published_release_tag: str, *, force_cpu: bool +) -> tuple[HostInfo, str, str]: + """Point a Vulkan-capable host at the upstream ggml-org Vulkan prebuilt. + + The unsloth published repo ships only CUDA/ROCm/CPU assets, so Vulkan comes + from UPSTREAM_REPO. Two triggers route here, both suppressed under + --cpu-fallback (the explicit "give me CPU" last resort wins): + * UNSLOTH_FORCE_VULKAN forces Vulkan over the detected CUDA/ROCm backend; + * an auto-detected Intel GPU with NO physical NVIDIA/ROCm -- the purpose + of the has_intel_gpu probe, since the fork manifest ships no Vulkan asset. + Applied by BOTH the install path and the --resolve-prebuilt probe so the + "is a prebuilt available" answer matches what actually gets installed. + + Returns the (possibly rewritten) host, repo, and release tag. + """ + forced = force_vulkan_requested() + # Gate auto-routing on no PHYSICAL NVIDIA, not merely no usable one: a mixed + # NVIDIA+Intel host that hides NVIDIA with CUDA_VISIBLE_DEVICES=""/-1 keeps + # has_physical_nvidia=True while has_usable_nvidia goes False. Vulkan ignores + # CUDA_VISIBLE_DEVICES, so auto-routing such a host would let it grab the + # reserved NVIDIA GPU. An explicit UNSLOTH_FORCE_VULKAN still overrides. + auto_intel = host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm + if force_cpu or not (forced or auto_intel): + return host, published_repo, published_release_tag + if host.is_macos: + if forced: + log( + "UNSLOTH_FORCE_VULKAN is set but ignored on macOS " + "(Metal is used; there is no Vulkan prebuilt)" + ) + return host, published_repo, published_release_tag + if forced: + log( + "UNSLOTH_FORCE_VULKAN is set; installing the upstream Vulkan " + "llama.cpp prebuilt instead of the detected GPU backend" + ) + # Forcing may override a detected NVIDIA/ROCm host, so normalize it to + # Vulkan-only; an auto-detected Intel host already is. + host = _vulkan_only_host(host) + else: + log("Intel GPU detected; installing the upstream Vulkan llama.cpp prebuilt") + # Swapping the fork for upstream invalidates a fork release pin: the two use + # different tag namespaces (fork b9596-mix- vs upstream b9596), so a + # pinned fork tag would make the upstream resolver query a nonexistent + # release and fall back to source. Drop it and let the upstream resolver + # pick by the requested llama tag. A pin already on an explicit upstream repo + # (repo unchanged here) is preserved. + if published_repo != UPSTREAM_REPO: + published_release_tag = "" + return host, UPSTREAM_REPO, published_release_tag + + def diffusion_visual_server_backfill_needed( install_dir: Path, host: HostInfo, choice: AssetChoice ) -> bool: @@ -6696,6 +6924,9 @@ def install_prebuilt( override_rocm_gfx = override_rocm_gfx, force_cpu = force_cpu, ) + host, published_repo, published_release_tag = _route_to_vulkan_prebuilt( + host, published_repo, published_release_tag, force_cpu = force_cpu + ) choice: AssetChoice | None = None try: with install_lock(install_lock_path(install_dir)): @@ -6708,7 +6939,9 @@ def install_prebuilt( f"no existing llama.cpp install detected at {install_dir}; performing fresh prebuilt install" ) # Single resolver: every fork host selects from the release manifest; - # an explicit ggml-org override selects by asset filename instead. + # an explicit ggml-org override selects by asset filename instead. A + # forced-Vulkan host already has published_repo pointed at + # UPSTREAM_REPO above, so the resolver takes the Vulkan asset branch. requested_tag, release_plans = resolve_simple_install_release_plans( llama_tag, host, @@ -6994,10 +7227,14 @@ def main() -> int: override_rocm_gfx = args.rocm_gfx, force_cpu = args.cpu_fallback, ) - repo = args.published_repo + # Same Vulkan routing the install path applies, so the probe's answer + # matches what would install (an Intel/forced-Vulkan host -> upstream). + host, repo, release_tag = _route_to_vulkan_prebuilt( + host, args.published_repo, args.published_release_tag or "", force_cpu = args.cpu_fallback + ) try: _requested, plans = resolve_simple_install_release_plans( - args.resolve_prebuilt, host, repo, args.published_release_tag or "" + args.resolve_prebuilt, host, repo, release_tag ) choice = plans[0].attempts[0] if plans and plans[0].attempts else None if choice is None: From 216a1fad33561ee4fcf24fd47811fbf721b46f29 Mon Sep 17 00:00:00 2001 From: alkinun Date: Thu, 9 Jul 2026 13:46:47 +0300 Subject: [PATCH 012/367] Fix Windows installer torch index override (#6972) * Fix Windows installer torch index override * Clear inherited uv index env vars for pinned installs in studio/setup.ps1 (#6898) * Harden setup.ps1 index-var clearing to truly remove vars (#6898) * Apply UV_DEFAULT_INDEX torch index fix to Linux/Mac install.sh (#6898) * Neutralize all uv index env vars for pinned torch installs (#6898) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Daniel Han Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- install.ps1 | 24 ++++++--- install.sh | 28 ++++++---- studio/setup.ps1 | 13 ++++- .../test_tokenizers_and_torch_constraint.py | 51 +++++++++++++++++++ tests/sh/test_mac_intel_compat.sh | 2 +- 5 files changed, 99 insertions(+), 19 deletions(-) diff --git a/install.ps1 b/install.ps1 index 696f4e613a..0797cd3868 100644 --- a/install.ps1 +++ b/install.ps1 @@ -469,6 +469,17 @@ function Install-UnslothStudio { param( [Parameter(Mandatory = $true)][ScriptBlock]$Command ) + # Installer-pinned index installs (torch) must beat an inherited uv mirror + # (#6898): when the command pins an index, clear every uv index env var so + # it wins, then restore in finally. Other installs keep the user's mirror. + $savedUvIndex = $null + if ($Command.ToString() -match '--default-index') { + $savedUvIndex = @{} + foreach ($n in 'UV_DEFAULT_INDEX', 'UV_INDEX_URL', 'UV_INDEX', 'UV_EXTRA_INDEX_URL') { + $savedUvIndex[$n] = [Environment]::GetEnvironmentVariable($n) + Remove-Item "Env:$n" -ErrorAction SilentlyContinue + } + } $prevEap = $ErrorActionPreference $ErrorActionPreference = "Continue" try { @@ -488,6 +499,7 @@ function Install-UnslothStudio { return [int]$LASTEXITCODE } finally { $ErrorActionPreference = $prevEap + if ($savedUvIndex) { foreach ($n in $savedUvIndex.Keys) { if ($null -ne $savedUvIndex[$n]) { Set-Item "Env:$n" $savedUvIndex[$n] } } } } } @@ -2200,7 +2212,7 @@ exit 0 # ABI-incompatible torchvision/torchaudio on AMD's per-arch index. $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (AMD ROCm)" { uv pip install --python $VenvPython --force-reinstall --index-url $ROCmIndexUrl $torchSpec $visionSpec $audioSpec } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (AMD ROCm)" { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $torchSpec $visionSpec $audioSpec } if ($torchInstallExit -ne 0) { # Transient AMD-index failure: fall back to a CPU base so the install # still completes; Studio setup retries ROCm afterwards. @@ -2209,7 +2221,7 @@ exit 0 # torch (e.g. 2.10.0+rocm on gfx110X/gfx90a) that still satisfies the CPU # torch>= range, so without it uv would keep the ROCm build and only swap # the companions -- a mismatched venv the flavor-repair block won't fix. - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (ROCm and CPU base both failed, exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2223,7 +2235,7 @@ exit 0 } else { Write-TauriLog "STEP" "Installing PyTorch" substep "installing PyTorch ($TorchIndexUrl)..." - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2306,7 +2318,7 @@ exit 0 # keeps a stale torch==X+cpu against a CUDA index and setup.ps1 then loops on # "torch cpu != required cuXXX". Reinstall the right triplet when a GPU build is # expected: CUDA from $TorchIndexUrl, ROCm from $ROCmIndexUrl (repo.amd.com gfx* - # is a PEP 503 index uv resolves via --index-url, same URL the fresh ROCm install + # is a PEP 503 index uv resolves via --default-index, same URL the fresh ROCm install # above uses). --no-torch / CPU-only hosts (expected cpu) are no-ops. if (-not $SkipTorch) { $expectedTorchTag = Get-ExpectedTorchFlavorTag -TorchIndexUrl $TorchIndexUrl -ROCmIndexUrl $ROCmIndexUrl @@ -2322,7 +2334,7 @@ exit 0 $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } substep "PyTorch flavor mismatch (installed $installedTorchTag, need ROCm) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --index-url $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct ROCm build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch (ROCm) (exit code $torchFixExit)" $torchFixExit) @@ -2331,7 +2343,7 @@ exit 0 } elseif ($expectedTorchTag -ne 'rocm') { # CUDA: stale +cpu (or wrong cuXXX) against a CUDA index -> reinstall triplet. substep "PyTorch flavor mismatch (installed $installedTorchTag, need $expectedTorchTag) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct CUDA build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch ($expectedTorchTag) (exit code $torchFixExit)" $torchFixExit) diff --git a/install.sh b/install.sh index 0acc9ec0be..3f4ea92387 100755 --- a/install.sh +++ b/install.sh @@ -159,6 +159,12 @@ run_maybe_quiet() { run_install_cmd() { _label="$1" shift + # Installer-pinned index installs (torch) must beat an inherited uv mirror + # (#6898): when we pass --default-index, neutralize every uv index env var so + # the pinned index wins. Other installs keep the user's mirror. + case " $* " in + *" --default-index "*) set -- env -u UV_DEFAULT_INDEX -u UV_INDEX_URL -u UV_INDEX -u UV_EXTRA_INDEX_URL "$@" ;; + esac if _is_verbose; then "$@" && return 0 _rc=$? @@ -2190,9 +2196,9 @@ _expected_torch_flavor_tag() { esac } -# Whether index ($1) supports a plain --index-url reinstall. pytorch.org cuXXX / +# Whether index ($1) supports a plain --default-index reinstall. pytorch.org cuXXX / # rocmX.Y AND the repo.amd.com gfx* indexes are all PEP 503 simple indexes that uv -# resolves (torch + every transitive dep) via --index-url -- the same URLs the +# resolves (torch + every transitive dep) via --default-index -- the same URLs the # fresh-install paths above already use -- so a stale wheel is auto-repairable. # Unknown/odd-mirror leaves -> no, so we warn rather than risk a wrong reinstall. _torch_index_repairable() { @@ -2744,7 +2750,7 @@ if [ "$_MIGRATED" = true ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --force-reinstall fi ;; @@ -2870,7 +2876,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "[WARN] Radeon repo lacks a compatible wheel set for this Python; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" else substep "installing PyTorch from Radeon repo (${_RADEON_BASE_URL})..." # Pass explicit wheel URLs so the matched trio is @@ -2893,18 +2899,18 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "[WARN] Radeon repo unavailable; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi else substep "[WARN] Radeon GPU detected but could not detect full ROCm version; falling back to ROCm index" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi else substep "installing PyTorch ($TORCH_INDEX_URL)..." run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi # AMD ROCm: install bitsandbytes (once, after torch, for all ROCm paths). # Gate on SKIP_TORCH=false so a user running with --no-torch on a ROCm @@ -2964,7 +2970,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --force-reinstall fi ;; @@ -2999,14 +3005,14 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then _installed_torch_ver=$("$_VENV_PY" -c "import torch; print(torch.__version__)" 2>/dev/null || true) _installed_torch_tag="" [ -n "$_installed_torch_ver" ] && _installed_torch_tag=$(_torch_flavor_tag "$_installed_torch_ver") - # Repair when flavor is wrong AND the index is plain --index-url reinstallable + # Repair when flavor is wrong AND the index is plain --default-index reinstallable # (cuXXX / rocmX.Y / repo.amd.com gfx*); an unknown mirror leaf -> warn only. if [ -n "$_installed_torch_tag" ] && [ "$_installed_torch_tag" != "$_expected_torch_tag" ] \ && [ "$(_torch_index_repairable "$TORCH_INDEX_URL")" = "yes" ]; then substep "PyTorch flavor mismatch (installed $_installed_torch_tag, need $_expected_torch_tag) -- reinstalling correct build..." run_install_cmd "reinstall PyTorch ($_expected_torch_tag)" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio _installed_torch_ver=$("$_VENV_PY" -c "import torch; print(torch.__version__)" 2>/dev/null || true) _installed_torch_tag="" @@ -3017,7 +3023,7 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then substep "[WARN] PyTorch is CPU-only but a $_expected_torch_tag GPU build was expected for this machine." "$C_WARN" substep "[WARN] Training and GPU inference will run on CPU until this is fixed." "$C_WARN" substep "[WARN] Re-run this installer, or reinstall the GPU build manually:" "$C_WARN" - substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" torchvision torchaudio --index-url $TORCH_INDEX_URL --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" + substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" torchvision torchaudio --default-index $TORCH_INDEX_URL --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" fi fi fi diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 07dcb17335..db01a1ecad 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -2621,7 +2621,18 @@ function Fast-Install { param([Parameter(ValueFromRemainingArguments=$true)]$Args_) if ($UseUv) { $VenvPy = (Get-Command python).Source - $result = & uv pip install --python $VenvPy @Args_ 2>&1 + # An explicit --index-url must win. Inherited uv index env vars otherwise + # override it and pull CPU torch over the CUDA/ROCm build (#6898), so drop + # them only for index-pinned installs; mirrors still apply elsewhere. + $saved = @{} + if (@($Args_) -contains '--index-url') { + foreach ($n in 'UV_DEFAULT_INDEX', 'UV_INDEX_URL', 'UV_INDEX', 'UV_EXTRA_INDEX_URL') { + $saved[$n] = [Environment]::GetEnvironmentVariable($n) + Remove-Item "Env:$n" -ErrorAction SilentlyContinue + } + } + try { $result = & uv pip install --python $VenvPy @Args_ 2>&1 } + finally { foreach ($n in $saved.Keys) { if ($null -ne $saved[$n]) { Set-Item "Env:$n" $saved[$n] } } } if ($LASTEXITCODE -eq 0) { return } } & python -m pip install @Args_ 2>&1 diff --git a/tests/python/test_tokenizers_and_torch_constraint.py b/tests/python/test_tokenizers_and_torch_constraint.py index 7390d7be9b..4322f0c7d6 100644 --- a/tests/python/test_tokenizers_and_torch_constraint.py +++ b/tests/python/test_tokenizers_and_torch_constraint.py @@ -14,6 +14,7 @@ _TESTS_DIR = pathlib.Path(__file__).resolve().parent.parent # tests/ _REPO_ROOT = _TESTS_DIR.parent # unsloth/ _INSTALL_SH = _REPO_ROOT / "install.sh" _INSTALL_PS1 = _REPO_ROOT / "install.ps1" +_SETUP_PS1 = _REPO_ROOT / "studio" / "setup.ps1" _NO_TORCH_RT = _REPO_ROOT / "studio" / "backend" / "requirements" / "no-torch-runtime.txt" @@ -109,6 +110,56 @@ class TestStructuralInstallPs1Unchanged: assert '"torch>=2.4,<2.11.0"' in self._ps1 +class TestInstallPs1UvDefaultIndex: + """Installer-managed torch indexes must override inherited uv defaults.""" + + _ps1 = _read(_INSTALL_PS1) + + def test_torch_installs_use_default_index(self): + assert "--default-index $TorchIndexUrl" in self._ps1 + assert "--default-index $ROCmIndexUrl" in self._ps1 + + def test_torch_installs_do_not_use_deprecated_index_url(self): + assert "--index-url $TorchIndexUrl" not in self._ps1 + assert "--index-url $ROCmIndexUrl" not in self._ps1 + + def test_torch_installs_neutralize_all_uv_index_env_vars(self): + # Extra-index vars outrank --default-index, so pinned installs must clear them. + for var in ("UV_DEFAULT_INDEX", "UV_INDEX_URL", "UV_INDEX", "UV_EXTRA_INDEX_URL"): + assert var in self._ps1 + assert 'Remove-Item "Env:$n"' in self._ps1 + + +class TestSetupPs1FastInstallIndex: + """setup.ps1 Fast-Install must neutralize inherited uv indexes when pinning.""" + + _ps1 = _read(_SETUP_PS1) + + def test_fast_install_clears_all_uv_index_env_vars(self): + for var in ("UV_DEFAULT_INDEX", "UV_INDEX_URL", "UV_INDEX", "UV_EXTRA_INDEX_URL"): + assert var in self._ps1 + # Must truly remove the vars (child sees no value), not set them empty. + assert 'Remove-Item "Env:$n"' in self._ps1 + + +class TestInstallShUvDefaultIndex: + """Linux/Mac installer torch indexes must override inherited uv defaults.""" + + _sh = _read(_INSTALL_SH) + + def test_torch_installs_use_default_index(self): + assert '--default-index "$TORCH_INDEX_URL"' in self._sh + + def test_torch_installs_do_not_use_deprecated_index_url(self): + assert '--index-url "$TORCH_INDEX_URL"' not in self._sh + + def test_torch_installs_neutralize_all_uv_index_env_vars(self): + # --default-index installs run with all uv index env vars unset via `env -u`. + assert ( + "env -u UV_DEFAULT_INDEX -u UV_INDEX_URL -u UV_INDEX -u UV_EXTRA_INDEX_URL" in self._sh + ) + + # Group 2 -- Shell snippet tests (bash subprocess, mocked python) class TestTorchConstraintShell: """Test the TORCH_CONSTRAINT block via bash with mocked python minor versions.""" diff --git a/tests/sh/test_mac_intel_compat.sh b/tests/sh/test_mac_intel_compat.sh index 3c3bbfaa5f..8a0ff4b641 100644 --- a/tests/sh/test_mac_intel_compat.sh +++ b/tests/sh/test_mac_intel_compat.sh @@ -312,7 +312,7 @@ if [ "$SKIP_TORCH" = true ]; then else echo "==> Installing PyTorch ($TORCH_INDEX_URL)..." uv pip install --python "$_VENV_PY" "torch>=2.4,<2.11.0" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi TORCH_EOF From cd9d251f157bc8a014a68f4688b961344d5d02f6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 04:10:59 -0700 Subject: [PATCH 013/367] Fix fast inference crash on compressed-tensors FP8 models (#7025) * Fix fast_gemv crash on compressed-tensors FP8 models Loading a compressed-tensors FP8 checkpoint (for example unsloth/Llama-3.2-1B-Instruct-FP8-Block) with fast_inference=False and running a forward crashed with 'Parameter object has no attribute absmax' inside fast_gemv. A compressed-tensors CompressedLinear exposes an already dequantized bf16 weight at forward time while keeping a weight_scale Parameter. The quant state resolution in get_lora_parameters/get_lora_parameters_bias fell back to that weight_scale, so a bf16 weight was routed into the bitsandbytes fast_gemv/fast_dequantize path, which expects a bitsandbytes QuantState with an absmax attribute. Only fall back to weight_scale_inv/weight_scale when the weight is still fp8. A decompressed bf16 weight then resolves to no quant state and flows through the normal bf16 path, which already handles bias and the LoRA backward. Real fp8 and bitsandbytes 4bit weights are unchanged. * Skip the fast_gemv dispatch test before importing unsloth when bitsandbytes is absent --- tests/test_fast_gemv_dispatch.py | 63 ++++++++++++++++++++++++++++++++ unsloth/kernels/utils.py | 27 ++++++++++++-- 2 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 tests/test_fast_gemv_dispatch.py diff --git a/tests/test_fast_gemv_dispatch.py b/tests/test_fast_gemv_dispatch.py new file mode 100644 index 0000000000..7758db2cd9 --- /dev/null +++ b/tests/test_fast_gemv_dispatch.py @@ -0,0 +1,63 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`get_lora_parameters` must not treat a `weight_scale` as a quant state for a weight that is +already dequantized to bf16 (e.g. a compressed-tensors layer at forward time). Otherwise the +bnb fast_gemv / fast_dequantize path reads a missing `absmax` and crashes. +""" + +from types import SimpleNamespace + +import pytest +import torch + +# unsloth.kernels.utils imports bitsandbytes unconditionally, so skip the whole module up +# front on runners without it (e.g. CPU-only) before importing unsloth, otherwise collection +# errors instead of producing a skip. Any other import error still surfaces as a failure. +pytest.importorskip("bitsandbytes") + +import unsloth # noqa: F401 (sets UNSLOTH_IS_PRESENT before transformers) +from unsloth.kernels.utils import get_lora_parameters_bias, _FP8_WEIGHT_DTYPES + +_FP8 = _FP8_WEIGHT_DTYPES[0] if _FP8_WEIGHT_DTYPES else None + + +def _proj(weight, weight_scale = None): + proj = SimpleNamespace(weight = weight, bias = None, merged = False) + if weight_scale is not None: + proj.weight_scale = weight_scale + return proj + + +def test_bf16_weight_scale_not_used_as_quant_state(): + """A bf16 weight carrying a weight_scale (compressed-tensors) -> quant state must be None.""" + proj = _proj(torch.randn(4, 4, dtype = torch.bfloat16), torch.rand(2, 2)) + W, W_quant = get_lora_parameters_bias(proj)[:2] + assert W_quant is None + + +def test_fp8_weight_keeps_scale(): + """An actual fp8 weight still resolves its weight_scale as the quant state.""" + if _FP8 is None: + pytest.skip("no float8 dtype in this torch build") + scale = torch.rand(2, 2) + proj = _proj(torch.randn(4, 4).to(_FP8), scale) + W, W_quant = get_lora_parameters_bias(proj)[:2] + assert W_quant is scale + + +def test_plain_bf16_has_no_quant_state(): + proj = _proj(torch.randn(4, 4, dtype = torch.bfloat16)) + W, W_quant = get_lora_parameters_bias(proj)[:2] + assert W_quant is None diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 43ed198a4a..1b0b5ce12e 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -282,6 +282,21 @@ def QUANT_STATE(W): return getattr(W, "quant_state", None) +# fp8 weight dtypes. A `weight_scale` / `weight_scale_inv` should only be treated as a +# quant state when the weight itself is still fp8. compressed-tensors layers expose an +# already-dequantized bf16 weight at forward time while keeping a `weight_scale` around; +# reading that as a quant state routes a bf16 weight into the bitsandbytes fast_gemv / +# fast_dequantize path, which then reads a missing `absmax` and crashes. +_FP8_WEIGHT_DTYPES = tuple( + dtype + for dtype in ( + getattr(torch, "float8_e4m3fn", None), + getattr(torch, "float8_e5m2", None), + ) + if dtype is not None +) + + def get_lora_parameters(proj): """Return (weight, weight quant_state, lora A, lora B, lora scale). With QAT enabled, also fake-quantizes the base layer and lora weights. @@ -298,9 +313,11 @@ def get_lora_parameters(proj): if weight_fake_quantizer is not None: W = weight_fake_quantizer(W) - # Get quant state for 4bit or FP8 + # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the + # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer) + # must not carry a scale as its quant state or fast_gemv will crash on it. W_quant = getattr(W, "quant_state", None) - if W_quant is None: + if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES: W_quant = getattr(base_layer, "weight_scale_inv", None) if W_quant is None: W_quant = getattr(base_layer, "weight_scale", None) @@ -349,9 +366,11 @@ def get_lora_parameters_bias(proj): ) # (proj.base_layer if hasattr(proj, "base_layer") else proj) W = base_layer.weight - # Get quant state for 4bit or FP8 + # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the + # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer) + # must not carry a scale as its quant state or fast_gemv will crash on it. W_quant = getattr(W, "quant_state", None) - if W_quant is None: + if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES: W_quant = getattr(base_layer, "weight_scale_inv", None) if W_quant is None: W_quant = getattr(base_layer, "weight_scale", None) From 534c877d2136b47b0ceec25cc45900c7f7f15e6d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 04:20:41 -0700 Subject: [PATCH 014/367] Keep native RoPE scaling when extending context; carry rope_theta for linear (#7028) * Keep native RoPE scaling when extending context; carry rope_theta for linear When max_seq_length exceeds a model's native window, the loader overwrote the model's rope_scaling with linear scaling. For models that already ship a scaled RoPE (llama3/yarn/longrope) that is far worse for long context, and on transformers v5 the linear dict omitted rope_theta (v5 keeps it under rope_parameters), so the rotary base fell back to 10000 and broke past ~8K tokens. Keep the native scaling and just widen the window; only synthesize linear for plain-RoPE models, and carry rope_theta so v5 keeps the real base. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Only preserve native llama3 when extending context; keep linear fallback otherwise The patched attention constructor (patch_llama_rope_scaling) rebuilds only linear, llama3 and longrope and its longrope branch reads a top-level original_max_position_embeddings, so preserving yarn or a nested-only longrope config would raise during construction on transformers <= 4.47.1. Keep only llama3 native; yarn/longrope/other types fall back to the linear override, still carrying rope_theta. * Correct long-context extension comment to match llama3-only preservation --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/utils/test_rope_scaling_drift.py | 33 ++++++++++++++ unsloth/models/llama.py | 61 +++++++++++++++++--------- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/tests/utils/test_rope_scaling_drift.py b/tests/utils/test_rope_scaling_drift.py index 7a738e236c..b2ec1e5a20 100644 --- a/tests/utils/test_rope_scaling_drift.py +++ b/tests/utils/test_rope_scaling_drift.py @@ -257,6 +257,39 @@ def test_recompute_helper_scales_on_cpu(): ), "_unsloth_recompute_inv_freq must return vanilla inv_freq when unscaled." +def test_extended_rope_scaling_keeps_llama3_and_carries_theta(): + # Long-context extension keeps native llama3, but falls back to linear for every other + # type (the patched attention constructor only rebuilds linear/llama3/longrope), and the + # linear dict carries rope_theta so transformers v5 does not fall back to base 10000. + from types import SimpleNamespace + + from unsloth.models.llama import _extended_rope_scaling + + # llama3 model: keep native scaling, do not synthesize linear. + scaling, native = _extended_rope_scaling(_make_config(LLAMA3_ROPE_SCALING), 2.0) + assert ( + scaling is None and native == "llama3" + ), "must keep native llama3 scaling instead of overwriting it with linear." + + # yarn is not rebuildable by the patcher -> keep the safe linear fallback, not native. + yarn = SimpleNamespace(rope_scaling = {"rope_type": "yarn", "factor": 2.0}, rope_theta = 500000.0) + scaling, _ = _extended_rope_scaling(yarn, 2.0) + assert scaling == { + "type": "linear", + "factor": 2.0, + "rope_theta": 500000.0, + }, f"yarn must fall back to linear (patcher cannot rebuild it), got {scaling}." + + # plain RoPE with theta only under v5 rope_parameters: linear must carry rope_theta. + v5 = SimpleNamespace(rope_parameters = {"rope_type": "default", "rope_theta": 1000000.0}) + scaling, _ = _extended_rope_scaling(v5, 2.0) + assert scaling == { + "type": "linear", + "factor": 2.0, + "rope_theta": 1000000.0, + }, f"linear override dropped rope_theta on v5 (got {scaling}); base would fall back to 10000." + + def test_extended_rotary_reads_config_factor(): # LlamaExtendedRotaryEmbedding must honor the config factor, not hardcode 8 # (Llama-3.2 uses 32); otherwise the subclass path re-drops scaling (#2405). diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 1f43f61443..05523bc27b 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1651,6 +1651,26 @@ def _rope_scaling_as_dict(rope_scaling): return {} +def _extended_rope_scaling(config, factor): + """RoPE scaling to extend a model past its native window. Keeps native llama3 as-is + (linear extension is far worse for long context); everything else gets linear. Returns + (scaling_or_None, type): None keeps llama3. The linear dict carries rope_theta so + transformers v5 (which stores it under rope_parameters) keeps the real base, not 10000. + Only llama3 is preserved because patch_llama_rope_scaling can only rebuild linear/llama3/ + longrope and its longrope branch needs a top-level original_max_position_embeddings.""" + existing = _rope_scaling_as_dict( + getattr(config, "rope_scaling", None) or getattr(config, "rope_parameters", None) or {} + ) + existing_type = existing.get("rope_type") or existing.get("type") + if existing_type == "llama3": + return None, existing_type + return { + "type": "linear", + "factor": factor, + "rope_theta": _get_rope_theta(config), + }, existing_type + + def _llama3_inv_freq_from_config( config, rope_scaling, @@ -2518,34 +2538,33 @@ class FastLlamaModel: max_seq_length = model_max_seq_length if (rope_scaling is None) and (max_seq_length > model_max_seq_length): - rope_scaling = max_seq_length / model_max_seq_length + factor = max_seq_length / model_max_seq_length if fast_inference: raise NotImplementedError( "Unsloth: Fast inference does not yet work with RoPE Scaling." ) - logger.warning_once( - f"Unsloth: {model_name} can only handle sequence lengths of at most " - f"{model_max_seq_length}.\nBut with kaiokendev's RoPE scaling of " - f"{round(rope_scaling, 3)}, it can be magically be extended to " - f"{max_seq_length}!" - ) - - # Warn RoPE scaling isn't allowed - if not has_rope_scaling: - raise RuntimeError( - f"However, {model_name} doesn't support RoPE Scaling!\n" - "Please file a feature request at https://github.com/unslothai/unsloth." + linear_scaling, native_type = _extended_rope_scaling(model_config, factor) + if linear_scaling is not None: + logger.warning_once( + f"Unsloth: {model_name} can only handle sequence lengths of at most " + f"{model_max_seq_length}.\nBut with kaiokendev's RoPE scaling of " + f"{round(factor, 3)}, it can be magically be extended to " + f"{max_seq_length}!" + ) + if not has_rope_scaling: + raise RuntimeError( + f"However, {model_name} doesn't support RoPE Scaling!\n" + "Please file a feature request at https://github.com/unslothai/unsloth." + ) + kwargs["rope_scaling"] = linear_scaling + else: + # Native llama3 scaling already handles long context; just widen the window. + logger.warning_once( + f"Unsloth: extending {model_name} to {max_seq_length} using its native " + f"{native_type} RoPE scaling." ) - - rope_scaling = { - "type": "linear", - "factor": rope_scaling, - } - - # Add to kwargs - kwargs["rope_scaling"] = rope_scaling from .loader_utils import ( check_and_disable_bitsandbytes_loading, From b5dca66cb1480b36ef738a4e62680e02cca2a65f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 04:52:30 -0700 Subject: [PATCH 015/367] scripts: refresh scan_packages allowlist baseline (#7032) * scripts: refresh scan_packages allowlist baseline Regenerate scripts/scan_packages_baseline.json against the current resolved dependency set so the blocking pip scan-packages gate matches what the scanner now finds. Refreshes evidence hashes for benign findings whose code shifted lines (unsloth-zoo mlx loader, gguf/mlx test /tmp fixtures) and adds two mainstream-library entries that were newly surfaced (torch inductor codecache base64+subprocess compile cache, torch testing common_utils socket import). Stale entries whose matching code changed and no longer triggers are dropped. All entries remain CRITICAL/HIGH findings manually judged benign; matched on (package, file, check, evidence_hash). * ci(security-audit): re-run scan when the allowlist baseline changes The security-audit pull_request trigger listed the scanners but not their allowlist baselines, so a baseline-only edit never re-ran the scan that consumes it. A refreshed baseline could therefore merge without CI confirming its evidence hashes match what the scanner finds. Add scan_packages_baseline.json and scan_npm_packages_baseline.json to the paths filter so baseline changes are validated on their own PR. --- .github/workflows/security-audit.yml | 6 +- scripts/scan_packages_baseline.json | 304 ++++++++++++--------------- 2 files changed, 140 insertions(+), 170 deletions(-) diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 0ef2ad1e9d..1275d12216 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -2,8 +2,8 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # Multi-language supply-chain audit. Triggers: -# - PRs touching any dependency manifest (Python / npm / Cargo) or -# this workflow file, +# - PRs touching any dependency manifest (Python / npm / Cargo), a +# scanner or its allowlist baseline, or this workflow file, # - push to main / pip, # - nightly @ 04:13 UTC so newly-published advisories surface even # when no PR opens, @@ -57,7 +57,9 @@ on: - 'studio/src-tauri/Cargo.lock' - 'pyproject.toml' - 'scripts/scan_packages.py' + - 'scripts/scan_packages_baseline.json' - 'scripts/scan_npm_packages.py' + - 'scripts/scan_npm_packages_baseline.json' - '.github/workflows/security-audit.yml' push: branches: [main, pip] diff --git a/scripts/scan_packages_baseline.json b/scripts/scan_packages_baseline.json index 1d34cfb66d..3582517d31 100644 --- a/scripts/scan_packages_baseline.json +++ b/scripts/scan_packages_baseline.json @@ -39,7 +39,7 @@ "file": "botocore/utils.py", "check": "Reads credential paths AND makes network calls", "severity": "CRITICAL", - "evidence": "Creds: L3551: CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'boto', 'cache')) | L3721: return os.path.expanduser(os.path.join('~', '.aws', 'login', 'cache'))\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", + "evidence": "Creds: L3551: CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'boto', 'cache')) | L3719: return os.path.expanduser(os.path.join('~', '.aws', 'login', 'cache'))\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", "evidence_hash": "2d691bc373ab872aad23c744104596ba6d0d9f3b35aa101c7edbff4429b174c1" }, { @@ -55,23 +55,23 @@ "file": "datasets/utils/file_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L443: while True: sha256:feba37d77721aa658e1786d2e4b67de76fefe1ceeb3ce8529d361c5241778eea", - "evidence_hash": "2e458563dec752d0a9896c9685d368d9906867110db315ab751e3eb6ec63f51c" + "evidence": "L441: while True: sha256:ce92e38c17c524815e1f9055be77235028c1e68e41b45cbfe9c8f1b867a205da", + "evidence_hash": "cb36281d28a975d101121c0702ee05eeee470879520d39a8be552129333f514d" }, { "package": "datasets", "file": "datasets/utils/file_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L441: while True: sha256:ce92e38c17c524815e1f9055be77235028c1e68e41b45cbfe9c8f1b867a205da", - "evidence_hash": "cb36281d28a975d101121c0702ee05eeee470879520d39a8be552129333f514d" + "evidence": "L443: while True: sha256:feba37d77721aa658e1786d2e4b67de76fefe1ceeb3ce8529d361c5241778eea", + "evidence_hash": "2e458563dec752d0a9896c9685d368d9906867110db315ab751e3eb6ec63f51c" }, { "package": "diffusers", "file": "diffusers/utils/import_utils.py", "check": "Downloads and executes remote code", "severity": "CRITICAL", - "evidence": "L1015: return importlib.import_module(\".\" + module_name, self.__name__)", + "evidence": "L1052: return importlib.import_module(\".\" + module_name, self.__name__)", "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" }, { @@ -79,7 +79,7 @@ "file": "diffusers/utils/testing_utils.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L233: value = os.environ[key]\nNetwork: L688: response = requests.get(arry, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L709: response = requests.get(url, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L728: image = PIL.Image.open(requests.get(image, stream=True, timeout=DIFFUSERS_REQUEST_TIMEOUT).raw)", + "evidence": "Env: L236: value = os.environ[key]\nNetwork: L691: response = requests.get(arry, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L712: response = requests.get(url, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L731: image = PIL.Image.open(requests.get(image, stream=True, timeout=DIFFUSERS_REQUEST_TIMEOUT).raw)", "evidence_hash": "671190a6106c6ee9674e5e5942dc0940e1d2f8c78d5faf674413c2345b783fd9" }, { @@ -90,12 +90,20 @@ "evidence": "Archive: L317: a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w')\nNetwork: L330: x['SocketType'] = _socket = socket.socket()", "evidence_hash": "894862e547cf91b90cd6e4b495db3fb05b7490ef0d63de7e795a7e3d9447d850" }, + { + "package": "fastapi", + "file": "fastapi/routing.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L586: while True: sha256:251135b5ebfdd1248916449f32262575e003ef64382501c65b7e4061d67bda45", + "evidence_hash": "365aef4449c8089753d9398417cd76ab762cef547d75db70d87bca9c0b550ab5" + }, { "package": "fastmcp-slim", "file": "fastmcp/cli/apps_dev.py", "check": "Creates archive with sensitive data AND makes network calls", "severity": "CRITICAL", - "evidence": "Archive: L1340: with tarfile.open(fileobj=io.BytesIO(data), mode=\"r:gz\") as tar:\nNetwork: L1291: with httpx.Client(timeout=30.0) as client: | L1305: with httpx.Client(timeout=30.0) as client: | L1335: with httpx.Client(timeout=30.0) as client: | L1537: client = httpx.AsyncClient(\nL1538: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1539: ) | L1701: async with httpx.AsyncClient(trust_env=False) as client: | L1769: with socket.socket(family, socket.SOCK_STREAM) as s:", + "evidence": "Archive: L1353: with tarfile.open(fileobj=io.BytesIO(data), mode=\"r:gz\") as tar:\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", "evidence_hash": "73a7a72013e9f800627ea07e6dbc3beeb8c905a6a5480c8fd896f0063173d25c" }, { @@ -103,8 +111,8 @@ "file": "fastmcp/cli/apps_dev.py", "check": "Enumerates filesystem AND makes network calls", "severity": "CRITICAL", - "evidence": "FS: L624: history.replaceState(null, \"\", url); sha256:fd8dbfa8af4dea2ce43f4d441f3f81239de341b76a2eb0a33c446f6757ce5f43\nNetwork: L1291: with httpx.Client(timeout=30.0) as client: | L1305: with httpx.Client(timeout=30.0) as client: | L1335: with httpx.Client(timeout=30.0) as client: | L1537: client = httpx.AsyncClient(\nL1538: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1539: ) | L1701: async with httpx.AsyncClient(trust_env=False) as client: | L1769: with socket.socket(family, socket.SOCK_STREAM) as s:", - "evidence_hash": "6ada4a9111213bdee5ea24c70a72ec4acdc8ffe0de4a01fd9835bc261ccab8f8" + "evidence": "FS: L637: history.replaceState(null, \"\", url); sha256:17068ba5bfed62c3a3007ec8bf3e0ea41ef6529b9e6112064d9afb3be9231436\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", + "evidence_hash": "e5325edfada6499540e6f0c24a0868979d275522e2b6a180aa9b5dd3280681b4" }, { "package": "fonttools", @@ -132,19 +140,35 @@ }, { "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", + "file": "huggingface_hub/_sandbox.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L3746: while True: sha256:0c73ed1a7447120b112c063b14e720c6695bc11d00eb6b912cd0f10dc3e29b31", - "evidence_hash": "22f50b930e44146c5350bb99e6e6ebb09feea9bf1e899e407bedc4ffaf06721b" + "evidence": "L1179: while True: sha256:33ceddf9e42aae207e891e97808c518e92a0b27ab60e4326256717bfb25a3a38", + "evidence_hash": "802fd41d8bb17bf425e99d128c0351c820103a5efb74690a4086e542a71437b8" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/_sandbox.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L83: d=/tmp/.sbx-server\nL84: if command -v wget >/dev/null 2>&1; then wget -q --header \"Authorization: Bearer $SBX_DL_TOKEN\" -O \"$d\" \"$SBX_SERVER_URL\"\nL85: elif command -v curl >/dev/null 2>&1; then curl -fsSL -H \"Authorization: Bearer $SBX_DL_TOKEN\" -o \"$d\" \"$SBX_SERVER_URL\"\nL86: else cp \"$SBX_SERVER_MOUNT/sbx-server\" \"$d\"; fi\nL87: chmod +x \"$d\"", + "evidence_hash": "6908a3fe328fa94ee22a119998d6ad07cfa1ba4efa2628acf240f4204fd76e22" }, { "package": "huggingface-hub", "file": "huggingface_hub/hf_api.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L4600: while True: sha256:f4a851312a1832efe1b435aa1275a82184e19cc3f47e2cd244373d56c11de272", - "evidence_hash": "dc8fcf44788e32f42d1cc2eb0e2deb55eb2dbf2c3a55909a7d503e450f45e602" + "evidence": "L4613: while True: sha256:f764b6ca3118b23c7c0e670e77178c022a6905f825d7df6e528545fa10aae8f6", + "evidence_hash": "9c85d50c227285fa8dc69512999cbb082258cda4b299c7d0e0f69f5aff7accd4" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/hf_api.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L3746: while True: sha256:0c73ed1a7447120b112c063b14e720c6695bc11d00eb6b912cd0f10dc3e29b31", + "evidence_hash": "22f50b930e44146c5350bb99e6e6ebb09feea9bf1e899e407bedc4ffaf06721b" }, { "package": "huggingface-hub", @@ -159,8 +183,8 @@ "file": "huggingface_hub/utils/_http.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L443: while True: sha256:0ab4fed32d3af10f361963371f681923481377508a405b5d8770cef75f859168", - "evidence_hash": "1484f6b92f41c427ba8cbc7c4695a94975fea683dfa83aa510b4b0e982be4721" + "evidence": "L462: while True: sha256:c75d1ee228cf7703a8c28551d649395a1f89f69a3aba69413f5bbcbd10c31958", + "evidence_hash": "d4d5f83fed39b87898cf776d5dad0bf1a6388a932f5fb7997d1070b50e46213e" }, { "package": "huggingface-hub", @@ -218,6 +242,22 @@ "evidence": "L5: import socket sha256:915068303029fa5806199f256fb74504c65f253f9aee8ea23d8e384bb772b1c7", "evidence_hash": "30be130f165f418dfd37b144c5ae333de184b95f828ab8bd4010a67b84a5f814" }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L3355: os.dup2(conn.fileno(), i) | L3387: \"test needs os.dup2()\") | L3405: os.dup2(fd, newfd) | L19: import socket sha256:26a745abdc7e89da28ab943394234d8ccb415e805477c3cc1f7d4766341a4c4c", + "evidence_hash": "a6b9bb85e9bb6682ab0dea4f95fd9266e8802f118c76d86dd87f7ab5864872cf" + }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L3521: os.dup2(conn.fileno(), i) | L3553: \"test needs os.dup2()\") | L3571: os.dup2(fd, newfd) | L20: import socket sha256:07d2933301c0dbeeb6e42381687827d8dd7cfd7471986c559ca64283d5ae6e24", + "evidence_hash": "db1f4ca69865ec3911d7450fe11d212b817139deda21cd7a4ee32d547a8dc452" + }, { "package": "numba", "file": "numba/pycc/decorators.py", @@ -231,7 +271,7 @@ "file": "numba/tests/support.py", "check": "Reverse shell / bind shell pattern", "severity": "CRITICAL", - "evidence": "L1021: os.dup2(w, fd) | L1026: os.dup2(save, fd)", + "evidence": "L1016: os.dup2(w, fd) | L1021: os.dup2(save, fd)", "evidence_hash": "fea7aa03d48bf0f4386302fa444984c4f5dfc772cfec3f1df199fd33a52eec10" }, { @@ -495,16 +535,16 @@ "file": "sklearn/datasets/_openml.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L100: while True: sha256:270363bb66980201e477f9b94886e4023f7a3d21b5ce026b7603a8c249a50c5b", - "evidence_hash": "53edbe07c312d459068d38e537b5114e65685ac3d4487b0423fa4542b5df20fe" + "evidence": "L100: while True: sha256:1f05a1b4fdd843b309634f583cb5e919866ef38ec5aa0b7d8a66ac8820655594", + "evidence_hash": "69597a64e5670a0f9a3c2aafc0bde4160f6170a9e2dc38f2c413cfa8d22ad193" }, { "package": "scikit-learn", "file": "sklearn/datasets/_openml.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L100: while True: sha256:1f05a1b4fdd843b309634f583cb5e919866ef38ec5aa0b7d8a66ac8820655594", - "evidence_hash": "69597a64e5670a0f9a3c2aafc0bde4160f6170a9e2dc38f2c413cfa8d22ad193" + "evidence": "L100: while True: sha256:270363bb66980201e477f9b94886e4023f7a3d21b5ce026b7603a8c249a50c5b", + "evidence_hash": "53edbe07c312d459068d38e537b5114e65685ac3d4487b0423fa4542b5df20fe" }, { "package": "scikit-learn", @@ -642,6 +682,14 @@ "evidence": "Base64: L1211: content = base64.b64decode(data)\nSubprocess: L2692: subprocess.run(\nL2693: cmd.split(), capture_output=True, text=True, check=True\nL2694: ) | L2995: cmd_output = subprocess.run(\nL2996: (\"openssl\", \"sha512\", filename), capture_output=True, text=True\nL2997: ) | L3707: out = subprocess.check_output(\nL3708: [\"ldd\", os.path.join(search, file)]\nL3709: ) | L3791: jobs.append(functools.partial(subprocess.check_call, cmd)) | L3876: subprocess.check_call(\nL3877: shlex.split(halide_cmd_gen.get_command_line())\nL3878: ) | L4336: subprocess.check_output(\nL4337: cmd_parts, stderr=subprocess.STDOUT, env=os.environ\nL4338: ) | L4591: output = subprocess.check_output(\nL4592: cmd_parts,\nL4593: stderr=subprocess.STDOUT,\nL4594: text=True,\nL4595: env=os.environ,\nL4596: )", "evidence_hash": "c09774087b702a6c5d6e2e85d9239c7c241ec938fbe9c0153e8f0b5c0710389b" }, + { + "package": "torch", + "file": "torch/_inductor/codecache.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L1727: content = base64.b64decode(data)\nSubprocess: L3270: subprocess.run(\nL3271: cmd, capture_output=True, text=True, check=True\nL3272: ) | L3583: cmd_output = subprocess.run(\nL3584: (\"openssl\", \"sha512\", filename), capture_output=True, text=True\nL3585: ) | L4338: out = subprocess.check_output(\nL4339: [\"ldd\", os.path.join(search, file)]\nL4340: ) | L4422: jobs.append(functools.partial(subprocess.check_call, cmd)) | L4507: subprocess.check_call(\nL4508: shlex.split(halide_cmd_gen.get_command_line())\nL4509: ) | L4992: subprocess.check_output(\nL4993: cmd_parts, stderr=subprocess.STDOUT, env=os.environ\nL4994: ) | L5247: output = subprocess.check_output(\nL5248: cmd_parts,\nL5249: stderr=subprocess.STDOUT,\nL5250: text=True,\nL5251: env=os.environ,\nL5252: )", + "evidence_hash": "87f77b5f51cb84fe9950fdeeb90fe8710e1b863100e90b5e2cfb228a725bee06" + }, { "package": "torch", "file": "torch/ao/__init__.py", @@ -695,7 +743,7 @@ "file": "torch/testing/_internal/common_utils.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L4770: env = os.environ.copy()\nNetwork: L4832: with request.urlopen(url, timeout=15) as f1, open(path, 'wb' if binary else 'w') as f2: | L4850: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:", + "evidence": "Env: L4900: env = os.environ.copy()\nNetwork: L4962: with request.urlopen(url, timeout=15) as f1, open(path, 'wb' if binary else 'w') as f2: | L4980: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:", "evidence_hash": "704a851b9d68c9b885b9e15538bd7e96f03875503b618fe6f126c4438edd7386" }, { @@ -706,6 +754,14 @@ "evidence": "L32: import socket sha256:89faaaa8bc908e02dad73fd59b2b481fa91189c84b39b556c2766e71d2783bf3", "evidence_hash": "3d23d77ace91812a07cb9508cf352185d154176e8e8c8b9b28fa92cdbcfe0d53" }, + { + "package": "torch", + "file": "torch/testing/_internal/common_utils.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L32: import socket sha256:ba439cbf568b194872f1d974c02b0487e51f677b67e379400522d0992600bd2d", + "evidence_hash": "88e98b227573997f86eedea8e885a407b0dd549d46d4a3f0b840ec5aafe66865" + }, { "package": "torchvision", "file": "torchvision/datasets/utils.py", @@ -743,8 +799,8 @@ "file": "transformers/testing_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L1663: while True: sha256:969e911d30c37a279ad915fb8c3d2d0a3f5705a7eb82ae6e00687388b68bbe65", - "evidence_hash": "2aa8e94baa805d599720a16afee6f08976482e301333e619e6c343389498ad15" + "evidence": "L1577: while True: sha256:2c6152f9da685f728e58d39dfc1827bc794f52606f56983bf38b5c6d0857cd5b", + "evidence_hash": "cdada67f3327237f00838a6750a4908dfaf76b9ab30c1352495c340d4fbd15c9" }, { "package": "transformers", @@ -759,15 +815,15 @@ "file": "transformers/testing_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L1577: while True: sha256:2c6152f9da685f728e58d39dfc1827bc794f52606f56983bf38b5c6d0857cd5b", - "evidence_hash": "cdada67f3327237f00838a6750a4908dfaf76b9ab30c1352495c340d4fbd15c9" + "evidence": "L1699: while True: sha256:969e911d30c37a279ad915fb8c3d2d0a3f5705a7eb82ae6e00687388b68bbe65", + "evidence_hash": "2aa8e94baa805d599720a16afee6f08976482e301333e619e6c343389498ad15" }, { "package": "transformers", "file": "transformers/testing_utils.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L284: value = os.environ[key] | L300: value = os.environ[key] | L2129: env = os.environ.copy() | L2251: for k in list(os.environ.keys()):\nNetwork: L2561: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:", + "evidence": "Env: L288: value = os.environ[key] | L304: value = os.environ[key] | L2165: env = os.environ.copy() | L2287: for k in list(os.environ.keys()):\nNetwork: L2597: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:", "evidence_hash": "73ff16aee09cf163fb3a7a04dfa2cf610595bde2f19460a579397695f728e3f4" }, { @@ -799,16 +855,16 @@ "file": "trl/extras/vllm_client.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L146: while True: sha256:2beedc742e1f085eaa10fd3bc40be97d2331d21887ef1b9ccdfa2150a184edfe", - "evidence_hash": "1540dffaaa053780e953e04c11d9c6b9c74b91cb60f3e6d87451ba7fe7db46db" + "evidence": "L152: while True: sha256:93e7d409e300af445376e6defbe2d0241aa19ecf63ed41b780fbb91c7d09856f", + "evidence_hash": "208838617172de61bca201d2a1bbeb5aa5aaa55feb1a1069cf39214673a7d6d1" }, { "package": "trl", "file": "trl/extras/vllm_client.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L152: while True: sha256:93e7d409e300af445376e6defbe2d0241aa19ecf63ed41b780fbb91c7d09856f", - "evidence_hash": "208838617172de61bca201d2a1bbeb5aa5aaa55feb1a1069cf39214673a7d6d1" + "evidence": "L146: while True: sha256:2beedc742e1f085eaa10fd3bc40be97d2331d21887ef1b9ccdfa2150a184edfe", + "evidence_hash": "1540dffaaa053780e953e04c11d9c6b9c74b91cb60f3e6d87451ba7fe7db46db" }, { "package": "trl", @@ -866,6 +922,14 @@ "evidence": "Crypto: L294: r\"|\\b(?:xprv|xpub|bc1|0x[a-fA-F0-9]{40})\\b\",\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", "evidence_hash": "278ff15b0b702d37d7f0b30a1e55a31bf2b11883685718a47478fbb5ce7f5212" }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", | L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", sha256:78268349021e21bedcd2eaaa5b4a71b0de1d52e023ada914dfdc09515ee1aad8", + "evidence_hash": "590fe1c96c442fbea5eb8642650257bc0b0199e919b9bacdb11dfa767b6fe839" + }, { "package": "unsloth-zoo", "file": "tests/security/fixtures/_build.py", @@ -919,16 +983,16 @@ "file": "tests/test_mlx_save_export_regressions.py", "check": "Writes to /tmp and executes (staged dropper)", "severity": "CRITICAL", - "evidence": "L164: temporary_location=\"/tmp/ignored\", sha256:78837e80d48e872ef191aaacfe5e1c621a98a20df486a70a41d1a932d074a5b3", - "evidence_hash": "dd11376e664d0d7e7f4cc4baf57eacd4b7ae7b03222dce3912ce68b63dbfca1e" + "evidence": "L165: temporary_location=\"/tmp/ignored\", sha256:9f8502377b19666288b28399633dfc6740a64d0cb70ad1615e38b1269f94bf37", + "evidence_hash": "b7262d6e58f2ebad961dd3e64ca6c32bba356b5044d7a642d7dbd36a58cb6c81" }, { "package": "unsloth-zoo", "file": "tests/test_quantize_gguf_q2_k_l.py", "check": "Writes to /tmp and executes (staged dropper)", "severity": "CRITICAL", - "evidence": "L67: input_gguf=\"/tmp/in.gguf\", sha256:32532cadc357beee1009f4e86481bdbe60a0b7bf47f6bb022b05ec1b8e15aed0", - "evidence_hash": "49f5b67379de17178f21a9bc93b79d6b94a70ecbdd16de86574934aac30a071d" + "evidence": "L67: input_gguf=\"/tmp/in.gguf\", sha256:06789b55e8f31426c233f37ff7d3729cc9e1f61c0829abd2c00c39216c63c7ad", + "evidence_hash": "ad4913d9099eb9b70e09d6860b242eb5f48c67e46d9bf4ae35c1c38a267d753b" }, { "package": "unsloth-zoo", @@ -951,7 +1015,7 @@ "file": "unsloth_zoo/llama_cpp.py", "check": "Creates archive with sensitive data AND makes network calls", "severity": "CRITICAL", - "evidence": "Archive: L938: with tarfile.open(archive_path, \"r:gz\") as archive:\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2862: check = requests.get(llama_cpp_chat_file, timeout = 5)", + "evidence": "Archive: L938: with tarfile.open(archive_path, \"r:gz\") as archive:\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2873: check = requests.get(llama_cpp_chat_file, timeout = 5)", "evidence_hash": "b9f3b1652349fa8ef9ac2d1715978aca1e1632165851a00a2698dd47189e410c" }, { @@ -959,7 +1023,7 @@ "file": "unsloth_zoo/llama_cpp.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L125: keynames = \"\\n\" + \"\\n\".join(os.environ.keys()) | L683: token = os.environ.get(\"GH_TOKEN\") or os.environ.get(\"GITHUB_TOKEN\")\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2862: check = requests.get(llama_cpp_chat_file, timeout = 5)", + "evidence": "Env: L125: keynames = \"\\n\" + \"\\n\".join(os.environ.keys()) | L683: token = os.environ.get(\"GH_TOKEN\") or os.environ.get(\"GITHUB_TOKEN\")\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2873: check = requests.get(llama_cpp_chat_file, timeout = 5)", "evidence_hash": "9cd0b1bb59c7eb1d814d7636dfd167c34f265eb7c4521a9d88b2bdcfd535b926" }, { @@ -1002,6 +1066,14 @@ "evidence": "Obfusc: L87: __import__(name)\nExec: L735: exec(\"\"\"exec _code_ in _globs_, _locs_\"\"\")", "evidence_hash": "3cb7d8247dea7dd3d7b21ededc0181c58c50099aeb73c9138a286f3d1ad92d4f" }, + { + "package": "cffi", + "file": "cffi/_cffi_gen_src.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L52: compiled = compile(source=pysrc, filename=filename, mode='exec')\nExec: L53: exec(compiled, globs, globs)", + "evidence_hash": "c429e4c977a61db6b7c717b5a552fce74eda622213e49eb5467a3782fd746fb9" + }, { "package": "cffi", "file": "cffi/setuptools_ext.py", @@ -1127,7 +1199,7 @@ "file": "numba/tests/support.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L879: __import__(modname)\nExec: L813: eval(co, globs, ns)", + "evidence": "Obfusc: L874: __import__(modname)\nExec: L808: eval(co, globs, ns)", "evidence_hash": "649a7d750f903478243b0bcb9e8020521b505fc7fedc5b696ec01f4efc096109" }, { @@ -1159,7 +1231,7 @@ "file": "numba/tests/test_np_functions.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L7106: exec(compile(funcstr, '', 'exec'), globals(), dct)\nExec: L7106: exec(compile(funcstr, '', 'exec'), globals(), dct)", + "evidence": "Obfusc: L7118: exec(compile(funcstr, '', 'exec'), globals(), dct)\nExec: L7118: exec(compile(funcstr, '', 'exec'), globals(), dct)", "evidence_hash": "9e81164131d16056fb56ad3cd11b8d129d1ff4f5855031e8b501e0335d5c14ed" }, { @@ -1175,16 +1247,16 @@ "file": "numpy/testing/_private/utils.py", "check": "Anti-analysis/sandbox evasion + suspicious behavior", "severity": "HIGH", - "evidence": "Anti: L2777: original_trace = sys.gettrace() | L2779: sys.settrace(None) | L2782: sys.settrace(original_trace)\nSubprocess: L1478: output = subprocess.run(cmd, capture_output=True, text=True)\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", - "evidence_hash": "27468a6828101c6c026ae25aca8aa90ef485fd62b2c8f0967479edae9c965844" + "evidence": "Anti: L2788: original_trace = sys.gettrace() | L2790: sys.settrace(None) | L2793: sys.settrace(original_trace)\nSubprocess: L1486: output = subprocess.run(cmd, capture_output=True, text=True) | L2889: res = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True,\nL2890: errors=\"replace\", **kwargs)\nExec: L1352: exec(astr, dict) | L1640: exec(code, globs, locs)", + "evidence_hash": "9c6961817e5b1751e572dfe0858286703bb835870ecdfd6a7a9fdd8372a5dd2b" }, { "package": "numpy", "file": "numpy/testing/_private/utils.py", "check": "Anti-analysis/sandbox evasion + suspicious behavior", "severity": "HIGH", - "evidence": "Anti: L2788: original_trace = sys.gettrace() | L2790: sys.settrace(None) | L2793: sys.settrace(original_trace)\nSubprocess: L1486: output = subprocess.run(cmd, capture_output=True, text=True) | L2889: res = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True,\nL2890: errors=\"replace\", **kwargs)\nExec: L1352: exec(astr, dict) | L1640: exec(code, globs, locs)", - "evidence_hash": "9c6961817e5b1751e572dfe0858286703bb835870ecdfd6a7a9fdd8372a5dd2b" + "evidence": "Anti: L2777: original_trace = sys.gettrace() | L2779: sys.settrace(None) | L2782: sys.settrace(original_trace)\nSubprocess: L1478: output = subprocess.run(cmd, capture_output=True, text=True)\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", + "evidence_hash": "27468a6828101c6c026ae25aca8aa90ef485fd62b2c8f0967479edae9c965844" }, { "package": "numpy", @@ -1199,7 +1271,7 @@ "file": "PIL/Image.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L422: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), []) | L490: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), [])\nExec: L3772: def eval(image: Image, *args: Callable[[int], float]) -> Image:", + "evidence": "Obfusc: L422: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), []) | L490: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), [])\nExec: L3776: def eval(image: Image, *args: Callable[[int], float]) -> Image:", "evidence_hash": "c2c1e7ae44e15862caf8de549d09db7b35e93282450f07ef61aaf5450a408c13" }, { @@ -1255,7 +1327,7 @@ "file": "setuptools/_distutils/compilers/C/base.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L1286: __import__(module_name)\nExec: L1113: if lib_type not in eval(expected):", + "evidence": "Obfusc: L1287: __import__(module_name)\nExec: L1114: if lib_type not in eval(expected):", "evidence_hash": "368651e9818ed2d1bb009027d3bcfbf94ae30639c0882a6c2bddde97b8c4f1e5" }, { @@ -1271,7 +1343,7 @@ "file": "setuptools/tests/config/test_pyprojecttoml.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L364: \"setup.py\": \"__import__('setuptools').setup(include_package_data=False)\",\nExec: L98: \"__main__.py\": \"def exec(): print('hello')\",", + "evidence": "Obfusc: L387: \"setup.py\": \"__import__('setuptools').setup(include_package_data=False)\",\nExec: L98: \"__main__.py\": \"def exec(): print('hello')\",", "evidence_hash": "067d41014f72a61d8b4adf25f3659d1f66a0e909f732223f48837aa7684df4e6" }, { @@ -1279,7 +1351,7 @@ "file": "setuptools/tests/test_editable_install.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L120: SETUP_SCRIPT_STUB = \"__import__('setuptools').setup()\"\nExec: L449: exec(finder, loc, loc)", + "evidence": "Obfusc: L120: SETUP_SCRIPT_STUB = \"__import__('setuptools').setup()\"\nExec: L447: exec(finder, loc, loc)", "evidence_hash": "a78d7f5af7eb4ba92656cda258c195b92f6337c585c97d0823e47a9d4a2eb15d" }, { @@ -1322,12 +1394,20 @@ "evidence": "Obfusc: L919: c = compile(funcstr, filename, 'exec')\nExec: L163: module = eval(import_command) | L170: exec(import_command, {}, namespace) | L903: exec(ln, {}, namespace) | L909: exec(ln, {}, namespace) | L920: exec(c, namespace, funclocals)", "evidence_hash": "ab4f5819576a70038301668b8f3e4a781c4b757b146117d5d93eab1896a5a6cd" }, + { + "package": "tensorboard", + "file": "tensorboard/plugins/projector/tf_projector_plugin/projector_binary.js", + "check": "Python wheel ships large JS bundle (uncommon; manually review)", + "severity": "HIGH", + "evidence": "sha256: 53c38430766be25dc672a30846ac3b9eba86aee35eb0746785ec012647c7d9a2", + "evidence_hash": "2c6384e8115a6d5dacf1f84d8f724832d8dc59feb442bb98ffae0857c0ccb381" + }, { "package": "torch", "file": "torch/_dynamo/bytecode_debugger.py", "check": "Anti-analysis/sandbox evasion + suspicious behavior", "severity": "HIGH", - "evidence": "Anti: L1048: self._old_trace = sys.gettrace() | L1049: sys.settrace(self._settrace_callback) | L1106: sys.settrace(self._old_trace)\nExec: L683: result = eval(arg, frame_globals, eval_locals) | L708: result = eval(cmd, frame_globals, eval_locals) | L716: exec(cmd, frame_globals, eval_locals)", + "evidence": "Anti: L1052: self._old_trace = sys.gettrace() | L1053: sys.settrace(self._settrace_callback) | L1113: sys.settrace(self._old_trace)\nExec: L684: result = eval(arg, frame_globals, eval_locals) | L709: result = eval(cmd, frame_globals, eval_locals) | L717: exec(cmd, frame_globals, eval_locals)", "evidence_hash": "dc2afd1769d357c15b69802bd2799fafa059c0b1dcdd4937528fb5b601962f1b" }, { @@ -1343,7 +1423,7 @@ "file": "torch/fx/experimental/rewriter.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L46: code = compile(dest_ast, \"\", \"exec\")\nExec: L49: exec(code, globals_dict)", + "evidence": "Obfusc: L44: code = compile(dest_ast, \"\", \"exec\")\nExec: L47: exec(code, globals_dict)", "evidence_hash": "76374f96feed416eec390458843621f33524cfb8d93ef0f3eb4cb1b47d0ad748" }, { @@ -1359,7 +1439,7 @@ "file": "torch/package/package_importer.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L602: def __import__(self, name, globals=None, locals=None, fromlist=(), level=0):\nExec: L412: exec(code, ns)", + "evidence": "Obfusc: L599: def __import__(self, name, globals=None, locals=None, fromlist=(), level=0):\nExec: L412: exec(code, ns)", "evidence_hash": "c7c0650f0c74a086d224112f77ee76634b8f47afc047ce27fee8c7fc45560512" }, { @@ -1391,7 +1471,7 @@ "file": "tests/test_mlx_trainer_internals.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L430: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L408: def eval(self):", + "evidence": "Obfusc: L1158: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L1136: def eval(self):", "evidence_hash": "c409327ef6420cc0c7224506fcb82b11bbc9838a6f2f97c9c2cfc00a40c4cdbf" }, { @@ -1407,7 +1487,7 @@ "file": "unsloth_zoo/compiler.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L1013: _mod = __import__(model_location, fromlist=items) | L4291: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4294: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4294: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4291: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L612: if eval(_dtype) is not None: | L613: dtype = eval(_dtype) | L955: _modeling_file = eval(model_location) | L1255: f = eval(f\"{model_location}.{module}\") | L1563: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L1564: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L2699: exec(f\"import {parent}\", locals(), globals()) | L2830: dir(eval(parent)), | L2834: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L2908: module = eval(f\"modeling_file.{module}\") | L2935: inner_class = eval(f\"modeling_file.{inner_class}\") | L3065: exec(f\"from timm.layers.norm_act import {norm}\") | L3073: forward = eval(norm).forward | L3079: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L3096: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L3104: forward = eval(block).forward | L3110: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L3385: exec(f\"import {model_location}\", globals()) | L3388: modeling_file = eval(model_location) | L3401: exec(\nL3402: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL3403: ) | L3405: exec(\nL3406: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL3407: globals(),\nL3408: locals(),\nL3409: ) | L3560: source = eval(f\"modeling_file.{module}\") | L3574: source = eval(f\"modeling_file.{module}\") | L3675: source = eval(f\"modeling_file.{module}\") | L3713: source = eval(f\"{model_location}.{module}\") | L3784: source = eval(f\"{model_location}.{module}\") | L3832: source = eval(f\"{model_location}.{module}\") | L4054: source = eval(f\"{model_location}.{module}\") | L4065: exec(\nL4066: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL4067: globals(),\nL4068: ) | L4131: source = eval(f\"{model_location}.{module}\") | L4172: module_cls = eval(f\"{model_location}.{module}\") | L4209: module_cls = eval(f\"{model_location}.{module}\") | L4276: exec(\nL4277: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL4278: globals(),\nL4279: ) | L4341: exec(inner_training_loop, globals()) | L4349: function = eval(f\"{model_location}.{module}\") | L4427: function = eval(f\"{model_location}.{module}\") | L4562: source = eval(f\"{model_location}.torch\") | L4569: function = eval(f\"source.nn.{module}\") | L4628: exec(\nL4629: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4630: globals(),\nL4631: locals(),\nL4632: ) | L4634: exec(\nL4635: f\"{model_location}.nn.{module}.forward = forward\",\nL4636: globals(),\nL4637: locals(),\nL4638: ) | L4642: exec(\nL4643: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4644: globals(),\nL4645: locals(),\nL4646: ) | L4648: exec(\nL4649: f\"combined_module.nn.{module}.forward = forward\",\nL4650: globals(),\nL4651: locals(),\nL4652: ) | L4669: exec(\nL4670: f\"{model_location}.{module} = combined_module.{module}\",\nL4671: globals(),\nL4672: locals(),\nL4673: ) | L4683: check_dicts = dir(eval(f\"{model_location}\")) | L4685: item = eval(f\"{model_location}.{check}\") | L4695: exec(\nL4696: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL4697: globals(),\nL4698: locals(),\nL4699: )", + "evidence": "Obfusc: L1013: _mod = __import__(model_location, fromlist=items) | L4295: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4298: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4298: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4295: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L612: if eval(_dtype) is not None: | L613: dtype = eval(_dtype) | L955: _modeling_file = eval(model_location) | L1255: f = eval(f\"{model_location}.{module}\") | L1563: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L1564: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L2699: exec(f\"import {parent}\", locals(), globals()) | L2830: dir(eval(parent)), | L2834: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L2908: module = eval(f\"modeling_file.{module}\") | L2935: inner_class = eval(f\"modeling_file.{inner_class}\") | L3065: exec(f\"from timm.layers.norm_act import {norm}\") | L3073: forward = eval(norm).forward | L3079: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L3096: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L3104: forward = eval(block).forward | L3110: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L3389: exec(f\"import {model_location}\", globals()) | L3392: modeling_file = eval(model_location) | L3405: exec(\nL3406: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL3407: ) | L3409: exec(\nL3410: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL3411: globals(),\nL3412: locals(),\nL3413: ) | L3564: source = eval(f\"modeling_file.{module}\") | L3578: source = eval(f\"modeling_file.{module}\") | L3679: source = eval(f\"modeling_file.{module}\") | L3717: source = eval(f\"{model_location}.{module}\") | L3788: source = eval(f\"{model_location}.{module}\") | L3836: source = eval(f\"{model_location}.{module}\") | L4058: source = eval(f\"{model_location}.{module}\") | L4069: exec(\nL4070: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL4071: globals(),\nL4072: ) | L4135: source = eval(f\"{model_location}.{module}\") | L4176: module_cls = eval(f\"{model_location}.{module}\") | L4213: module_cls = eval(f\"{model_location}.{module}\") | L4280: exec(\nL4281: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL4282: globals(),\nL4283: ) | L4345: exec(inner_training_loop, globals()) | L4353: function = eval(f\"{model_location}.{module}\") | L4431: function = eval(f\"{model_location}.{module}\") | L4566: source = eval(f\"{model_location}.torch\") | L4573: function = eval(f\"source.nn.{module}\") | L4632: exec(\nL4633: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4634: globals(),\nL4635: locals(),\nL4636: ) | L4638: exec(\nL4639: f\"{model_location}.nn.{module}.forward = forward\",\nL4640: globals(),\nL4641: locals(),\nL4642: ) | L4646: exec(\nL4647: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4648: globals(),\nL4649: locals(),\nL4650: ) | L4652: exec(\nL4653: f\"combined_module.nn.{module}.forward = forward\",\nL4654: globals(),\nL4655: locals(),\nL4656: ) | L4673: exec(\nL4674: f\"{model_location}.{module} = combined_module.{module}\",\nL4675: globals(),\nL4676: locals(),\nL4677: ) | L4687: check_dicts = dir(eval(f\"{model_location}\")) | L4689: item = eval(f\"{model_location}.{check}\") | L4699: exec(\nL4700: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL4701: globals(),\nL4702: locals(),\nL4703: )", "evidence_hash": "ec1875fd32d00fe885e566ebda75163e46e838ca31020abb57e0991892c2bdf7" }, { @@ -1423,8 +1503,8 @@ "file": "unsloth_zoo/mlx/loader.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L2218: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L140: mx.eval(model.parameters()) | L176: mx.eval(model.parameters()) | L2022: model.eval() | L2605: mx.eval(model.parameters()) | L2721: mx.eval(module.weight) | L4030: mx.eval(model.parameters()) | L4058: mx.eval(model.parameters()) | L4178: mx.eval(model.parameters())", - "evidence_hash": "9b29dade82912216c8b4808aa293b79749aa80ef1d2be35edd93bec7632810f1" + "evidence": "Obfusc: L2869: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L148: mx.eval(model.parameters()) | L180: mx.eval(model.parameters()) | L732: mx.eval(model.parameters()) | L733: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L799: mx.eval(model.parameters()) | L802: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L2673: model.eval() | L3256: mx.eval(model.parameters()) | L3372: mx.eval(module.weight) | L5666: mx.eval(model.parameters()) | L5716: mx.eval(model.parameters()) | L5859: mx.eval(model.parameters())", + "evidence_hash": "7b44760032c5df6d379ccfdd0bff3d23f857f64e08210fa0fba8d2881d457634" }, { "package": "unsloth-zoo", @@ -1439,7 +1519,7 @@ "file": "unsloth_zoo/saving_utils.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L3241: module = __import__('transformers', fromlist=[model_class_name])\nExec: L3123: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3169: exec(save_pretrained, globals(), functions)", + "evidence": "Obfusc: L4015: module = __import__('transformers', fromlist=[model_class_name])\nExec: L3897: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3943: exec(save_pretrained, globals(), functions)", "evidence_hash": "530b2383acd9fe8330aa65cd0bf86164aaacd47770e7c8d0752195bee36396ec" }, { @@ -1449,118 +1529,6 @@ "severity": "HIGH", "evidence": "Obfusc: L836: code = compile(module, \"\", \"exec\")\nExec: L736: exec(code, globs, locs)", "evidence_hash": "5c0992c90f05c772abd94d00784f157de337e1f8567f8b3aee1b15e46c96cd5d" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3355: os.dup2(conn.fileno(), i) | L3387: \"test needs os.dup2()\") | L3405: os.dup2(fd, newfd) | L19: import socket sha256:26a745abdc7e89da28ab943394234d8ccb415e805477c3cc1f7d4766341a4c4c", - "evidence_hash": "a6b9bb85e9bb6682ab0dea4f95fd9266e8802f118c76d86dd87f7ab5864872cf" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", | L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", sha256:78268349021e21bedcd2eaaa5b4a71b0de1d52e023ada914dfdc09515ee1aad8", - "evidence_hash": "590fe1c96c442fbea5eb8642650257bc0b0199e919b9bacdb11dfa767b6fe839" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3521: os.dup2(conn.fileno(), i) | L3553: \"test needs os.dup2()\") | L3571: os.dup2(fd, newfd) | L20: import socket sha256:07d2933301c0dbeeb6e42381687827d8dd7cfd7471986c559ca64283d5ae6e24", - "evidence_hash": "db1f4ca69865ec3911d7450fe11d212b817139deda21cd7a4ee32d547a8dc452" - }, - { - "package": "fastapi", - "file": "fastapi/routing.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L586: while True: sha256:bef9ea429314fad39e063895a37dc5cfe9b04561f3d1acbb3c99abb4e92e6cfe", - "evidence_hash": "b15773e1bc249713156a349278ea60f7c0e3dd7d537affe929ab51089e1942bb" - }, - { - "package": "tensorboard", - "file": "tensorboard/plugins/projector/tf_projector_plugin/projector_binary.js", - "check": "Python wheel ships large JS bundle (uncommon; manually review)", - "severity": "HIGH", - "evidence": "sha256: 53c38430766be25dc672a30846ac3b9eba86aee35eb0746785ec012647c7d9a2", - "evidence_hash": "2c6384e8115a6d5dacf1f84d8f724832d8dc59feb442bb98ffae0857c0ccb381" - }, - { - "package": "fastapi", - "file": "fastapi/routing.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L586: while True: sha256:251135b5ebfdd1248916449f32262575e003ef64382501c65b7e4061d67bda45", - "evidence_hash": "365aef4449c8089753d9398417cd76ab762cef547d75db70d87bca9c0b550ab5" - }, - { - "package": "fastmcp-slim", - "file": "fastmcp/cli/apps_dev.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L637: history.replaceState(null, \"\", url); sha256:17068ba5bfed62c3a3007ec8bf3e0ea41ef6529b9e6112064d9afb3be9231436\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", - "evidence_hash": "e5325edfada6499540e6f0c24a0868979d275522e2b6a180aa9b5dd3280681b4" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/_sandbox.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L1179: while True: sha256:33ceddf9e42aae207e891e97808c518e92a0b27ab60e4326256717bfb25a3a38", - "evidence_hash": "802fd41d8bb17bf425e99d128c0351c820103a5efb74690a4086e542a71437b8" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/_sandbox.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L83: d=/tmp/.sbx-server\nL84: if command -v wget >/dev/null 2>&1; then wget -q --header \"Authorization: Bearer $SBX_DL_TOKEN\" -O \"$d\" \"$SBX_SERVER_URL\"\nL85: elif command -v curl >/dev/null 2>&1; then curl -fsSL -H \"Authorization: Bearer $SBX_DL_TOKEN\" -o \"$d\" \"$SBX_SERVER_URL\"\nL86: else cp \"$SBX_SERVER_MOUNT/sbx-server\" \"$d\"; fi\nL87: chmod +x \"$d\"", - "evidence_hash": "6908a3fe328fa94ee22a119998d6ad07cfa1ba4efa2628acf240f4204fd76e22" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L4613: while True: sha256:f764b6ca3118b23c7c0e670e77178c022a6905f825d7df6e528545fa10aae8f6", - "evidence_hash": "9c85d50c227285fa8dc69512999cbb082258cda4b299c7d0e0f69f5aff7accd4" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/utils/_http.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L462: while True: sha256:c75d1ee228cf7703a8c28551d649395a1f89f69a3aba69413f5bbcbd10c31958", - "evidence_hash": "d4d5f83fed39b87898cf776d5dad0bf1a6388a932f5fb7997d1070b50e46213e" - }, - { - "package": "cffi", - "file": "cffi/_cffi_gen_src.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L52: compiled = compile(source=pysrc, filename=filename, mode='exec')\nExec: L53: exec(compiled, globs, globs)", - "evidence_hash": "c429e4c977a61db6b7c717b5a552fce74eda622213e49eb5467a3782fd746fb9" - }, - { - "package": "multiprocess", - "file": "multiprocess/forkserver.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L6: import socket sha256:6c707119169286c9a798e2c8d13a48614e481d8a503950916fd4ffb4c94d3182", - "evidence_hash": "50fec0f0522a8e4e636bf348b752002d7935d8455af31fb78c6f11e2eba19f6d" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3569: os.dup2(conn.fileno(), i) | L3601: \"test needs os.dup2()\") | L3619: os.dup2(fd, newfd) | L20: import socket sha256:c824dc0f409f242420c3fbb324790c53cb3078d2c8b07ee8f2a05694b01c2946", - "evidence_hash": "3878a2b430c175dbc5877a95195bfe52f9588ff73fb74e2261ed5e33087915ad" } ] } From fb5dc91bb4f33f8a4c5a41c89cbe88c93394e970 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 05:09:16 -0700 Subject: [PATCH 016/367] Studio: remove dead direct_linux_release_plan path (#7030) parse_direct_linux_release_bundle and direct_linux_release_plan are no longer reached by any live code path. Fork Linux installs resolve through _fork_manifest_release_plans -> _linux_published_attempts, and the upstream (ggml-org) path uses direct_upstream_release_plan. The dead parser also called _resolve_linux_bundle_profile, which no longer exists, so its CUDA branch would raise NameError if ever executed. Drop both functions and the obsolete TestDirectLinuxNvidiaCpuGate; its live equivalent TestLinuxPublishedAttemptsNvidiaCpuGate already covers the NVIDIA no-silent-CPU behaviour. --- studio/install_llama_prebuilt.py | 156 ------------------- tests/studio/install/test_selection_logic.py | 62 -------- 2 files changed, 218 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 856ba71478..40caebc040 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1286,162 +1286,6 @@ def synthetic_checksums_for_release( ) -def parse_direct_linux_release_bundle( - repo: str, release: dict[str, Any] -) -> PublishedReleaseBundle | None: - release_tag = release.get("tag_name") - if not isinstance(release_tag, str) or not release_tag: - return None - - assets = release_asset_map(release) - artifacts: list[PublishedLlamaArtifact] = [] - inferred_labels: list[str] = [] - - linux_asset_re = re.compile( - r"^app-(?P