Merge remote-tracking branch 'origin/main' into studio-tokenless-llama-prebuilt
This commit is contained in:
Daniel Han
commit
b1e20dc893
15 changed files with 1629 additions and 113 deletions
57
.github/scripts/assert-llama-loads.sh
vendored
Executable file
57
.github/scripts/assert-llama-loads.sh
vendored
Executable file
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
#
|
||||
# Assert Studio installed a llama.cpp that loads and runs on THIS macOS. Tests
|
||||
# the contract that matters (binaries load and their minimum-OS is <= this host)
|
||||
# instead of the old "did install.sh fall back to a source build?" grep, since a
|
||||
# source build with a correct deployment target is a valid outcome.
|
||||
set -uo pipefail
|
||||
|
||||
UNSLOTH_HOME="${STUDIO_HOME:-$HOME/.unsloth}"
|
||||
LLAMA_DIR="${LLAMA_CPP_DIR:-$UNSLOTH_HOME/llama.cpp}"
|
||||
BIN_DIR="$LLAMA_DIR/build/bin"
|
||||
|
||||
fail() {
|
||||
echo "::error::$*"
|
||||
if [ -f logs/install.log ]; then
|
||||
echo "---- install.log (llama.cpp lines) ----"
|
||||
grep -E "llama-prebuilt|llama\.cpp|macos prebuilt|falling back" logs/install.log | tail -80 || true
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
SERVER="$(find "$LLAMA_DIR" -type f -name 'llama-server' 2>/dev/null | head -1)"
|
||||
QUANT="$(find "$LLAMA_DIR" -type f -name 'llama-quantize' 2>/dev/null | head -1)"
|
||||
[ -n "$SERVER" ] || fail "llama-server not found under $LLAMA_DIR after install"
|
||||
[ -n "$QUANT" ] || fail "llama-quantize not found under $LLAMA_DIR after install"
|
||||
|
||||
HOST_VER="$(sw_vers -productVersion 2>/dev/null || echo '0')"
|
||||
HOST_MAJOR="${HOST_VER%%.*}"
|
||||
|
||||
# Static minimum-OS check on every Mach-O we ship. vtool ships with the Xcode
|
||||
# command line tools, which GitHub macOS runners always have; if it is somehow
|
||||
# missing we skip the static check and rely on the runtime launch below.
|
||||
if command -v vtool >/dev/null 2>&1; then
|
||||
while IFS= read -r macho; do
|
||||
[ -n "$macho" ] || continue
|
||||
minos="$(vtool -show-build "$macho" 2>/dev/null | awk '/minos/{print $2; exit}')"
|
||||
[ -n "$minos" ] || continue
|
||||
min_major="${minos%%.*}"
|
||||
if [ "$min_major" -gt "$HOST_MAJOR" ] 2>/dev/null; then
|
||||
fail "$(basename "$macho") is built for macOS $minos but this runner is macOS $HOST_VER (prebuilt is newer than the host)"
|
||||
fi
|
||||
done < <(find "$BIN_DIR" -type f \( -name '*.dylib' -o -name 'llama-server' -o -name 'llama-quantize' \) 2>/dev/null)
|
||||
fi
|
||||
|
||||
# Runtime launch: --version forces dyld to load every linked dylib (including
|
||||
# libggml-metal.dylib). A missing Metal symbol or too-new binary fails here.
|
||||
if ! "$SERVER" --version >/tmp/llama-server-version.txt 2>&1; then
|
||||
echo "---- llama-server --version output ----"
|
||||
cat /tmp/llama-server-version.txt || true
|
||||
fail "llama-server failed to launch on macOS $HOST_VER (dyld load / symbol error)"
|
||||
fi
|
||||
|
||||
echo "llama.cpp load validation passed on macOS $HOST_VER"
|
||||
echo " server: $SERVER"
|
||||
sed -n '1,4p' /tmp/llama-server-version.txt 2>/dev/null || true
|
||||
9
.github/workflows/studio-mac-api-smoke.yml
vendored
9
.github/workflows/studio-mac-api-smoke.yml
vendored
|
|
@ -89,13 +89,8 @@ jobs:
|
|||
set -o pipefail
|
||||
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
|
||||
|
||||
- name: Assert install.sh used the Mac llama.cpp prebuilt
|
||||
run: |
|
||||
if grep -q "falling back to source build" logs/install.log; then
|
||||
echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
|
||||
grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
|
||||
exit 1
|
||||
fi
|
||||
- name: Assert llama.cpp loads on this macOS
|
||||
run: bash .github/scripts/assert-llama-loads.sh
|
||||
|
||||
- name: Install pyjwt for the JWT-expiry forge test
|
||||
run: pip install 'pyjwt>=2.6'
|
||||
|
|
|
|||
27
.github/workflows/studio-mac-inference-smoke.yml
vendored
27
.github/workflows/studio-mac-inference-smoke.yml
vendored
|
|
@ -114,13 +114,8 @@ jobs:
|
|||
set -o pipefail
|
||||
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
|
||||
|
||||
- name: Assert install.sh used the Mac llama.cpp prebuilt
|
||||
run: |
|
||||
if grep -q "falling back to source build" logs/install.log; then
|
||||
echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
|
||||
grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
|
||||
exit 1
|
||||
fi
|
||||
- name: Assert llama.cpp loads on this macOS
|
||||
run: bash .github/scripts/assert-llama-loads.sh
|
||||
|
||||
- name: Install OpenAI + Anthropic Python SDKs
|
||||
run: pip install 'openai>=1.50' 'anthropic>=0.40'
|
||||
|
|
@ -369,13 +364,8 @@ jobs:
|
|||
set -o pipefail
|
||||
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
|
||||
|
||||
- name: Assert install.sh used the Mac llama.cpp prebuilt
|
||||
run: |
|
||||
if grep -q "falling back to source build" logs/install.log; then
|
||||
echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
|
||||
grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
|
||||
exit 1
|
||||
fi
|
||||
- name: Assert llama.cpp loads on this macOS
|
||||
run: bash .github/scripts/assert-llama-loads.sh
|
||||
|
||||
- name: Reset auth + boot Studio (API-only, default tool policy)
|
||||
# We deliberately use the API-only mode rather than
|
||||
|
|
@ -760,13 +750,8 @@ jobs:
|
|||
set -o pipefail
|
||||
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
|
||||
|
||||
- name: Assert install.sh used the Mac llama.cpp prebuilt
|
||||
run: |
|
||||
if grep -q "falling back to source build" logs/install.log; then
|
||||
echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
|
||||
grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
|
||||
exit 1
|
||||
fi
|
||||
- name: Assert llama.cpp loads on this macOS
|
||||
run: bash .github/scripts/assert-llama-loads.sh
|
||||
|
||||
- name: Install OpenAI + Anthropic Python SDKs
|
||||
run: pip install 'openai>=1.50' 'anthropic>=0.40'
|
||||
|
|
|
|||
80
.github/workflows/studio-mac-install-matrix.yml
vendored
Normal file
80
.github/workflows/studio-mac-install-matrix.yml
vendored
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
|
||||
# Proves Studio's llama.cpp install loads on every supported macOS. The heavy
|
||||
# app smokes stay single-OS; this matrix covers the OS-version dimension cheaply
|
||||
# (install.sh + binary-load assert). Regression guard for the macOS-version
|
||||
# selection in studio/install_llama_prebuilt.py.
|
||||
|
||||
name: Mac Studio Install Matrix CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'studio/install_llama_prebuilt.py'
|
||||
- 'studio/setup.sh'
|
||||
- 'install.sh'
|
||||
- '.github/scripts/assert-llama-loads.sh'
|
||||
- '.github/workflows/studio-mac-install-matrix.yml'
|
||||
push:
|
||||
branches: [main, pip]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
install-load:
|
||||
name: Install + load (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 25
|
||||
continue-on-error: ${{ matrix.experimental }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-14 # Apple Silicon, macOS 14 Sonoma
|
||||
experimental: false
|
||||
- os: macos-15 # Apple Silicon, macOS 15 Sequoia
|
||||
experimental: false
|
||||
- os: macos-26 # Apple Silicon, macOS 26 Tahoe
|
||||
experimental: false
|
||||
- os: macos-15-intel # Intel x86_64, macOS 15 (informational)
|
||||
experimental: true
|
||||
- os: macos-26-intel # Intel x86_64, macOS 26 (last Intel macOS)
|
||||
experimental: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install Studio (--local, --no-torch)
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
mkdir -p logs
|
||||
set -o pipefail
|
||||
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
|
||||
|
||||
- name: Assert llama.cpp loads on this macOS
|
||||
run: bash .github/scripts/assert-llama-loads.sh
|
||||
|
||||
- name: Upload install log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mac-install-matrix-${{ matrix.os }}-log
|
||||
path: logs/install.log
|
||||
retention-days: 7
|
||||
9
.github/workflows/studio-mac-ui-smoke.yml
vendored
9
.github/workflows/studio-mac-ui-smoke.yml
vendored
|
|
@ -89,13 +89,8 @@ jobs:
|
|||
set -o pipefail
|
||||
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
|
||||
|
||||
- name: Assert install.sh used the Mac llama.cpp prebuilt
|
||||
run: |
|
||||
if grep -q "falling back to source build" logs/install.log; then
|
||||
echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
|
||||
grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
|
||||
exit 1
|
||||
fi
|
||||
- name: Assert llama.cpp loads on this macOS
|
||||
run: bash .github/scripts/assert-llama-loads.sh
|
||||
|
||||
- name: Install Playwright + Chromium
|
||||
# No --with-deps on Mac: that flag installs Linux apt packages.
|
||||
|
|
|
|||
17
.github/workflows/studio-mac-update-smoke.yml
vendored
17
.github/workflows/studio-mac-update-smoke.yml
vendored
|
|
@ -67,21 +67,8 @@ jobs:
|
|||
set -o pipefail
|
||||
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
|
||||
|
||||
- name: Assert install.sh used the Mac llama.cpp prebuilt
|
||||
run: |
|
||||
# Mac install must take the prebuilt path. Source-build
|
||||
# fallback here is an Unsloth bug.
|
||||
if grep -q "falling back to source build" logs/install.log; then
|
||||
echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
|
||||
grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -qE "prebuilt installed and validated|prebuilt up to date and validated|bin-macos-arm64" logs/install.log; then
|
||||
echo "::error::no Mac prebuilt llama.cpp marker in install.log."
|
||||
grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
|
||||
exit 1
|
||||
fi
|
||||
echo "install.sh installed the Mac prebuilt llama.cpp"
|
||||
- name: Assert llama.cpp loads on this macOS
|
||||
run: bash .github/scripts/assert-llama-loads.sh
|
||||
|
||||
- name: First update should be a no-op (prebuilt already validated)
|
||||
env:
|
||||
|
|
|
|||
102
studio/backend/tests/test_gguf_routing.py
Normal file
102
studio/backend/tests/test_gguf_routing.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""
|
||||
Tests for GGUF routing in detect_gguf_model.
|
||||
|
||||
Regression test for the bug where a .gguf file temporarily appears
|
||||
inaccessible on Windows during llama-server process teardown, causing
|
||||
is_file() to return False and the model to be routed to the transformers
|
||||
backend instead of llama-server.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
# Stub structlog before importing backend modules (mirrors other tests in this suite)
|
||||
if "structlog" not in sys.modules:
|
||||
|
||||
class _DummyLogger:
|
||||
def __getattr__(self, _):
|
||||
return lambda *a, **k: None
|
||||
|
||||
sys.modules["structlog"] = types.SimpleNamespace(
|
||||
get_logger = lambda *a, **k: _DummyLogger(),
|
||||
BoundLogger = _DummyLogger,
|
||||
)
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from utils.models.model_config import detect_gguf_model
|
||||
|
||||
|
||||
def test_detects_gguf_file_normally(tmp_path):
|
||||
"""Normal case: .gguf file exists and is accessible."""
|
||||
gguf = tmp_path / "gpt-oss-20b-MXFP4.gguf"
|
||||
gguf.write_bytes(b"")
|
||||
result = detect_gguf_model(str(gguf))
|
||||
assert result is not None
|
||||
assert result.endswith("gpt-oss-20b-MXFP4.gguf")
|
||||
|
||||
|
||||
def test_detects_gguf_when_stat_raises_oserror(tmp_path):
|
||||
"""
|
||||
Regression: on Windows, both is_file() and exists() call stat() internally.
|
||||
During the brief lock window after llama-server is killed, stat() raises
|
||||
OSError, causing both to return False. detect_gguf_model must still route
|
||||
to llama-server based on the file extension alone.
|
||||
"""
|
||||
gguf = tmp_path / "gpt-oss-20b-MXFP4.gguf"
|
||||
gguf.write_bytes(b"")
|
||||
|
||||
original_stat = Path.stat
|
||||
|
||||
def flaky_stat(self, **kwargs):
|
||||
if self.suffix.lower() == ".gguf":
|
||||
raise OSError("file temporarily inaccessible (Windows lock window)")
|
||||
return original_stat(self, **kwargs)
|
||||
|
||||
with patch.object(Path, "stat", flaky_stat):
|
||||
result = detect_gguf_model(str(gguf))
|
||||
|
||||
assert result is not None, (
|
||||
"detect_gguf_model returned None when stat() raised OSError. "
|
||||
"This causes the model to fall through to the transformers backend."
|
||||
)
|
||||
|
||||
|
||||
def test_does_not_detect_mmproj_as_main_model(tmp_path):
|
||||
"""mmproj files must never be returned as the primary model."""
|
||||
mmproj = tmp_path / "mmproj-model-f16.gguf"
|
||||
mmproj.write_bytes(b"")
|
||||
result = detect_gguf_model(str(mmproj))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_detects_gguf_in_directory(tmp_path):
|
||||
"""Directory containing a .gguf file is resolved to that file."""
|
||||
gguf = tmp_path / "model-Q4_K_M.gguf"
|
||||
gguf.write_bytes(b"")
|
||||
result = detect_gguf_model(str(tmp_path))
|
||||
assert result is not None
|
||||
assert result.endswith("model-Q4_K_M.gguf")
|
||||
|
||||
|
||||
def test_directory_named_like_gguf_scans_inside(tmp_path):
|
||||
"""A directory named *.gguf resolves the real .gguf inside, not itself."""
|
||||
gguf_dir = tmp_path / "mymodel.gguf"
|
||||
gguf_dir.mkdir()
|
||||
inner = gguf_dir / "model-Q4_K_M.gguf"
|
||||
inner.write_bytes(b"")
|
||||
result = detect_gguf_model(str(gguf_dir))
|
||||
assert result is not None
|
||||
assert result.endswith("model-Q4_K_M.gguf")
|
||||
|
||||
|
||||
def test_returns_none_for_non_gguf_path(tmp_path):
|
||||
"""Non-.gguf paths with no .gguf files inside return None."""
|
||||
result = detect_gguf_model(str(tmp_path))
|
||||
assert result is None
|
||||
|
|
@ -1156,13 +1156,18 @@ def detect_gguf_model(path: str) -> Optional[str]:
|
|||
p = Path(path)
|
||||
|
||||
# Case 1: direct .gguf file
|
||||
if p.suffix.lower() == ".gguf" and p.is_file():
|
||||
if p.suffix.lower() == ".gguf":
|
||||
if _is_mmproj(p.name):
|
||||
return None
|
||||
# Use absolute (not resolve) to preserve symlink names -- e.g.
|
||||
# Ollama .studio_links/model.gguf -> blobs/sha256-... should
|
||||
# keep the readable symlink name, not the opaque blob hash.
|
||||
return str(p.absolute())
|
||||
# Extension is authoritative: don't gate on is_file()/exists(), which
|
||||
# can fail in the Windows lock window after llama-server is killed.
|
||||
try:
|
||||
is_dir = p.is_dir()
|
||||
except OSError:
|
||||
is_dir = False # stat() unavailable in the lock window
|
||||
if not is_dir:
|
||||
return str(p.absolute()) # absolute() keeps symlink names readable
|
||||
# Directory named "*.gguf": fall through to the dir scan below.
|
||||
|
||||
# Case 2: directory containing .gguf files (skip mmproj)
|
||||
if p.is_dir():
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ import {
|
|||
Edit03Icon,
|
||||
Globe02Icon,
|
||||
HelpCircleIcon,
|
||||
Logout01Icon,
|
||||
Logout05Icon,
|
||||
Search01Icon,
|
||||
PowerIcon,
|
||||
PencilEdit02Icon,
|
||||
|
|
@ -796,7 +796,7 @@ export function AppSidebar() {
|
|||
void navigate({ to: "/login" });
|
||||
}}
|
||||
>
|
||||
<HugeiconsIcon icon={Logout01Icon} strokeWidth={1.75} className="size-icon" />
|
||||
<HugeiconsIcon icon={Logout05Icon} strokeWidth={1.75} className="size-icon" />
|
||||
<span>{t("shell.navigation.logOut")}</span>
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => setShutdownOpen(true)}>
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import re
|
|||
import shutil
|
||||
import site
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
|
|
@ -160,6 +161,14 @@ DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS = env_int(
|
|||
2,
|
||||
minimum = 1,
|
||||
)
|
||||
# Deeper macOS-only walk-back: upstream can ship a run of prebuilts built for a
|
||||
# newer macOS than the host, only caught at validate time, so an older host must
|
||||
# skip the whole run. Free on new hosts (first plan validates, extras unused).
|
||||
DEFAULT_MAX_MACOS_RELEASE_FALLBACKS = env_int(
|
||||
"UNSLOTH_LLAMA_MAX_MACOS_RELEASE_FALLBACKS",
|
||||
16,
|
||||
minimum = 1,
|
||||
)
|
||||
FORCE_COMPILE_DEFAULT_REF = os.environ.get("UNSLOTH_LLAMA_FORCE_COMPILE_REF", "master")
|
||||
|
||||
DIRECT_LINUX_BUNDLE_PROFILES: dict[str, dict[str, Any]] = {
|
||||
|
|
@ -213,6 +222,71 @@ DIRECT_LINUX_BUNDLE_PROFILES: dict[str, dict[str, Any]] = {
|
|||
},
|
||||
}
|
||||
|
||||
# Lowest CUDA major we ship prebuilts for, and the highest major we probe for
|
||||
# installed runtime libraries. Detection and runtime-line derivation are
|
||||
# generated per major so a new toolkit (cuda14, ...) needs no code change while
|
||||
# llama.cpp keeps the cudart64_<major>.dll / libcudart.so.<major> naming.
|
||||
_MIN_CUDA_MAJOR = 12
|
||||
_MAX_PROBE_CUDA_MAJOR = 19
|
||||
|
||||
# Last ggml-org release whose Windows win-cuda-13 build is still sub-13.3
|
||||
# (cuda-13.1, b9360, 2026-05-27). Upstream bumped win-cuda-13 to 13.3 at b9365
|
||||
# and now ships only cuda-12.4 + cuda-13.3. cuda-12.4 predates Blackwell (ggml
|
||||
# compiles sm_120 only at toolkit >= 12.8), so a Blackwell host on a 13.1/13.2
|
||||
# driver is gated off 13.3 and would drop to a CPU-only 12.4 build. b9360 is
|
||||
# immutable, so we pin its cuda-13.1 build (plus paired cudart) as a GPU
|
||||
# fallback for exactly those hosts. See unslothai/unsloth#5887.
|
||||
_PINNED_BLACKWELL_FALLBACK_TAG = "b9360"
|
||||
_PINNED_BLACKWELL_FALLBACK_RUNTIME = "13.1"
|
||||
_PINNED_BLACKWELL_DRIVER_FLOOR = (13, 1)
|
||||
_BLACKWELL_MIN_SM = 120
|
||||
# ggml compiles Blackwell sm_120 only at toolkit >= 12.8, so an in-release
|
||||
# windows-cuda build at or above this already covers Blackwell and makes the
|
||||
# older pinned 13.1 fallback unnecessary (cuda-12.4 is below it).
|
||||
_BLACKWELL_MIN_TOOLKIT = (12, 8)
|
||||
_PINNED_BLACKWELL_LLAMA_SHA256 = (
|
||||
"31ddb8b42d7ab4a47cab8c48c397519f580ca502df7e73f3ab396eacc16c8e8d"
|
||||
)
|
||||
_PINNED_BLACKWELL_CUDART_SHA256 = (
|
||||
"f96935e7e385e3b2d0189239077c10fe8fd7e95690fea4afec455b1b6c7e3f18"
|
||||
)
|
||||
|
||||
|
||||
def _cuda_runtime_lines_for_major(major: int) -> list[str]:
|
||||
"""Runtime lines a driver of this CUDA major can use, newest major first
|
||||
down to the minimum we ship. A driver runs its own major and any older one
|
||||
(backward compatibility)."""
|
||||
return [f"cuda{m}" for m in range(major, _MIN_CUDA_MAJOR - 1, -1)]
|
||||
|
||||
|
||||
def _resolve_linux_bundle_profile(bundle_profile: str) -> "dict[str, Any] | None":
|
||||
"""Profile (runtime line + sm coverage) for a linux-x64-cuda<major>-<class>
|
||||
bundle. Known majors use their published coverage; an unknown future major
|
||||
reuses the newest known major's coverage for the same class as a forward
|
||||
default, with the post-build GPU smoke test as the backstop."""
|
||||
known = DIRECT_LINUX_BUNDLE_PROFILES.get(bundle_profile)
|
||||
if known is not None:
|
||||
return known
|
||||
m = re.fullmatch(
|
||||
r"cuda(?P<major>\d+)-(?P<klass>older|newer|portable)", bundle_profile
|
||||
)
|
||||
if not m:
|
||||
return None
|
||||
base_key = max(
|
||||
(
|
||||
k
|
||||
for k, v in DIRECT_LINUX_BUNDLE_PROFILES.items()
|
||||
if v["coverage_class"] == m.group("klass")
|
||||
),
|
||||
key = lambda k: int(re.match(r"cuda(\d+)-", k).group(1)),
|
||||
default = None,
|
||||
)
|
||||
if base_key is None:
|
||||
return None
|
||||
profile = dict(DIRECT_LINUX_BUNDLE_PROFILES[base_key])
|
||||
profile["runtime_line"] = f"cuda{m.group('major')}"
|
||||
return profile
|
||||
|
||||
|
||||
@dataclass
|
||||
class HostInfo:
|
||||
|
|
@ -231,6 +305,9 @@ class HostInfo:
|
|||
has_usable_nvidia: bool
|
||||
has_rocm: bool = False
|
||||
rocm_gfx_target: str | None = None
|
||||
# (major, minor) from platform.mac_ver(); None off macOS or if unparseable.
|
||||
# Skips a macos prebuilt whose minimum-OS exceeds this host.
|
||||
macos_version: tuple[int, int] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -757,6 +834,26 @@ def windows_cuda_asset_aliases(
|
|||
return aliases
|
||||
|
||||
|
||||
def _published_windows_cuda_runtime(
|
||||
upstream_assets: dict[str, str], major: int, driver: tuple[int, int] | None
|
||||
) -> str | None:
|
||||
"""Highest cuda-<major>.<minor> published upstream that `driver` can run by
|
||||
default CUDA compatibility, i.e. (major, minor) <= driver. None if nothing
|
||||
qualifies. Gating on the driver (not just the major) keeps a 13.3 build off
|
||||
a driver that only advertises 13.1, where it would otherwise rely on the
|
||||
unguaranteed minor-version-compatibility path."""
|
||||
if driver is None:
|
||||
return None
|
||||
best: int | None = None
|
||||
for name in upstream_assets:
|
||||
m = re.search(r"-bin-win-cuda-(\d+)\.(\d+)-x64\.zip$", name)
|
||||
if m and int(m.group(1)) == major:
|
||||
minor = int(m.group(2))
|
||||
if (major, minor) <= driver and (best is None or minor > best):
|
||||
best = minor
|
||||
return f"{major}.{best}" if best is not None else None
|
||||
|
||||
|
||||
def format_byte_count(num_bytes: float) -> str:
|
||||
units = ["B", "KiB", "MiB", "GiB", "TiB"]
|
||||
value = float(num_bytes)
|
||||
|
|
@ -1341,7 +1438,7 @@ def parse_direct_linux_release_bundle(
|
|||
inferred_labels: list[str] = []
|
||||
|
||||
linux_asset_re = re.compile(
|
||||
r"^app-(?P<label>.+)-(?P<target>linux-x64(?:-cpu)?|linux-x64-(?:cuda12|cuda13)-(?:older|newer|portable))\.tar\.gz$"
|
||||
r"^app-(?P<label>.+)-(?P<target>linux-x64(?:-cpu)?|linux-x64-cuda\d+-(?:older|newer|portable))\.tar\.gz$"
|
||||
)
|
||||
for asset_name in sorted(assets):
|
||||
match = linux_asset_re.fullmatch(asset_name)
|
||||
|
|
@ -1366,7 +1463,7 @@ def parse_direct_linux_release_bundle(
|
|||
continue
|
||||
|
||||
bundle_profile = target.removeprefix("linux-x64-")
|
||||
profile = DIRECT_LINUX_BUNDLE_PROFILES.get(bundle_profile)
|
||||
profile = _resolve_linux_bundle_profile(bundle_profile)
|
||||
if profile is None:
|
||||
continue
|
||||
artifacts.append(
|
||||
|
|
@ -1526,6 +1623,11 @@ def direct_upstream_release_plan(
|
|||
torch_preference.selection_log,
|
||||
)
|
||||
)
|
||||
# Blackwell on a 13.1/13.2 driver: prefer the pinned cuda-13.1 GPU
|
||||
# build over the CPU-only cuda-12.4 the in-release gating leaves.
|
||||
pinned = _pinned_windows_cuda_fallback(host, attempts)
|
||||
if pinned is not None:
|
||||
attempts.insert(0, pinned)
|
||||
elif host.has_rocm:
|
||||
lemonade_choice = resolve_lemonade_rocm_choice(
|
||||
host, "windows", "windows-hip", llama_tag = requested_tag
|
||||
|
|
@ -1666,6 +1768,14 @@ def resolve_simple_install_release_plans(
|
|||
requested_tag == "latest" and not published_release_tag
|
||||
)
|
||||
release_limit = max(1, max_release_fallbacks)
|
||||
# macOS may need to walk past a run of too-new prebuilts. Only when the host
|
||||
# version is known; otherwise keep the default (cannot tell up front).
|
||||
if (
|
||||
host.is_macos
|
||||
and allow_older_release_fallback
|
||||
and host.macos_version is not None
|
||||
):
|
||||
release_limit = max(release_limit, DEFAULT_MAX_MACOS_RELEASE_FALLBACKS)
|
||||
plans: list[InstallReleasePlan] = []
|
||||
last_error: PrebuiltFallback | None = None
|
||||
|
||||
|
|
@ -1885,8 +1995,8 @@ def linux_runtime_dirs_for_required_libraries(
|
|||
|
||||
def detected_linux_runtime_lines() -> tuple[list[str], dict[str, list[str]]]:
|
||||
line_requirements = {
|
||||
"cuda13": ["libcudart.so.13", "libcublas.so.13"],
|
||||
"cuda12": ["libcudart.so.12", "libcublas.so.12"],
|
||||
f"cuda{m}": [f"libcudart.so.{m}", f"libcublas.so.{m}"]
|
||||
for m in range(_MAX_PROBE_CUDA_MAJOR, _MIN_CUDA_MAJOR - 1, -1)
|
||||
}
|
||||
detected: list[str] = []
|
||||
runtime_dirs: dict[str, list[str]] = {}
|
||||
|
|
@ -2888,6 +2998,8 @@ def detect_host() -> HostInfo:
|
|||
is_x86_64 = machine in {"x86_64", "amd64"}
|
||||
is_arm64 = machine in {"arm64", "aarch64"}
|
||||
|
||||
macos_version = parse_macos_version(platform.mac_ver()[0]) if is_macos else None
|
||||
|
||||
nvidia_smi = shutil.which("nvidia-smi")
|
||||
driver_cuda_version = None
|
||||
compute_caps: list[str] = []
|
||||
|
|
@ -3063,6 +3175,7 @@ def detect_host() -> HostInfo:
|
|||
has_usable_nvidia = has_usable_nvidia,
|
||||
has_rocm = has_rocm,
|
||||
rocm_gfx_target = rocm_gfx_target,
|
||||
macos_version = macos_version,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -3081,17 +3194,21 @@ def compatible_linux_runtime_lines(host: HostInfo) -> list[str]:
|
|||
if not host.driver_cuda_version:
|
||||
return []
|
||||
major, _minor = host.driver_cuda_version
|
||||
if major >= 13:
|
||||
return ["cuda13", "cuda12"]
|
||||
if major >= 12:
|
||||
return ["cuda12"]
|
||||
return []
|
||||
if major < _MIN_CUDA_MAJOR:
|
||||
return []
|
||||
return _cuda_runtime_lines_for_major(major)
|
||||
|
||||
|
||||
def windows_runtime_line_info() -> dict[str, tuple[str, ...]]:
|
||||
# Generated per CUDA major (newest first) so a new toolkit is detected
|
||||
# without a code change while the cudart64_<major>.dll naming holds.
|
||||
return {
|
||||
"cuda13": ("cudart64_13*.dll", "cublas64_13*.dll", "cublasLt64_13*.dll"),
|
||||
"cuda12": ("cudart64_12*.dll", "cublas64_12*.dll", "cublasLt64_12*.dll"),
|
||||
f"cuda{m}": (
|
||||
f"cudart64_{m}*.dll",
|
||||
f"cublas64_{m}*.dll",
|
||||
f"cublasLt64_{m}*.dll",
|
||||
)
|
||||
for m in range(_MAX_PROBE_CUDA_MAJOR, _MIN_CUDA_MAJOR - 1, -1)
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -3108,12 +3225,13 @@ def detected_windows_runtime_lines() -> tuple[list[str], dict[str, list[str]]]:
|
|||
|
||||
|
||||
def compatible_windows_runtime_lines(host: HostInfo) -> list[str]:
|
||||
driver_runtime = pick_windows_cuda_runtime(host)
|
||||
if driver_runtime == "13.1":
|
||||
return ["cuda13", "cuda12"]
|
||||
if driver_runtime == "12.4":
|
||||
return ["cuda12"]
|
||||
return []
|
||||
if not host.driver_cuda_version:
|
||||
return []
|
||||
major, minor = host.driver_cuda_version
|
||||
# cuda12 prebuilts need a 12.4+ driver; cuda13+ any minor of the major.
|
||||
if major < _MIN_CUDA_MAJOR or (major == _MIN_CUDA_MAJOR and minor < 4):
|
||||
return []
|
||||
return _cuda_runtime_lines_for_major(major)
|
||||
|
||||
|
||||
def runtime_line_from_cuda_version(cuda_version: str | None) -> str | None:
|
||||
|
|
@ -3190,7 +3308,6 @@ def windows_cuda_attempts(
|
|||
selection_preamble: Iterable[str] = (),
|
||||
) -> list[AssetChoice]:
|
||||
selection_log = list(selection_preamble)
|
||||
runtime_by_line = {"cuda12": "12.4", "cuda13": "13.1"}
|
||||
driver_runtime = pick_windows_cuda_runtime(host)
|
||||
detected_runtime_lines, runtime_dirs = detected_windows_runtime_lines()
|
||||
compatible_runtime_lines = compatible_windows_runtime_lines(host)
|
||||
|
|
@ -3233,12 +3350,7 @@ def windows_cuda_attempts(
|
|||
selection_log.append(
|
||||
"windows_cuda_selection: detected CUDA runtime DLLs were incompatible with the reported driver"
|
||||
)
|
||||
fallback_runtime_lines = (
|
||||
["cuda13", "cuda12"]
|
||||
if driver_runtime == "13.1"
|
||||
else (["cuda12"] if driver_runtime == "12.4" else [])
|
||||
)
|
||||
normal_runtime_lines = fallback_runtime_lines
|
||||
normal_runtime_lines = compatible_runtime_lines
|
||||
|
||||
runtime_order: list[str] = []
|
||||
if preferred_runtime_line and preferred_runtime_line in normal_runtime_lines:
|
||||
|
|
@ -3262,6 +3374,13 @@ def windows_cuda_attempts(
|
|||
for runtime_line in normal_runtime_lines
|
||||
if runtime_line not in runtime_order
|
||||
)
|
||||
# Keep every driver-compatible line reachable as a fallback, so a line gated
|
||||
# out by the driver version still drops to an older major (cuda13 -> cuda12).
|
||||
runtime_order.extend(
|
||||
runtime_line
|
||||
for runtime_line in compatible_runtime_lines
|
||||
if runtime_line not in runtime_order
|
||||
)
|
||||
selection_log.append(
|
||||
"windows_cuda_selection: normal_runtime_order="
|
||||
+ (",".join(normal_runtime_lines) if normal_runtime_lines else "none")
|
||||
|
|
@ -3273,7 +3392,18 @@ def windows_cuda_attempts(
|
|||
|
||||
attempts: list[AssetChoice] = []
|
||||
for runtime_line in runtime_order:
|
||||
runtime = runtime_by_line[runtime_line]
|
||||
major = int(runtime_line.removeprefix("cuda"))
|
||||
# Track whatever minor llama.cpp actually ships for this major
|
||||
# (cuda13 -> 13.1, 13.3, ...). Skip the line when the release has no
|
||||
# matching asset instead of guessing a now-missing name.
|
||||
runtime = _published_windows_cuda_runtime(
|
||||
upstream_assets, major, host.driver_cuda_version
|
||||
)
|
||||
if runtime is None:
|
||||
selection_log.append(
|
||||
f"windows_cuda_selection: no driver-supported asset for {runtime_line}"
|
||||
)
|
||||
continue
|
||||
selected_name = None
|
||||
asset_url = None
|
||||
for candidate_name in windows_cuda_upstream_asset_names(llama_tag, runtime):
|
||||
|
|
@ -3328,6 +3458,110 @@ def windows_cuda_attempts(
|
|||
return attempts
|
||||
|
||||
|
||||
def _windows_cuda_attempt_covers_blackwell(attempt: AssetChoice) -> bool:
|
||||
"""True if an in-release windows-cuda attempt is built with a toolkit that
|
||||
covers Blackwell sm_120 (>= 12.8), read from its asset name's CUDA minor."""
|
||||
if attempt.install_kind != "windows-cuda":
|
||||
return False
|
||||
m = re.search(r"-bin-win-cuda-(\d+)\.(\d+)-x64\.zip$", attempt.name)
|
||||
return (
|
||||
m is not None and (int(m.group(1)), int(m.group(2))) >= _BLACKWELL_MIN_TOOLKIT
|
||||
)
|
||||
|
||||
|
||||
def _pinned_windows_cuda_fallback(
|
||||
host: HostInfo, existing_cuda_attempts: list[AssetChoice]
|
||||
) -> AssetChoice | None:
|
||||
"""Pinned GPU fallback for a Blackwell host the in-release build gates off.
|
||||
Upstream stopped publishing a sub-13.3 Windows cuda13 build after b9360, and
|
||||
cuda-12.4 cannot offload sm_120, so a 13.1/13.2 driver would land on CPU.
|
||||
b9360's cuda-13.1 build is immutable and runs on those drivers. Returns None
|
||||
(dormant) whenever the in-release selection already offers a Blackwell-capable
|
||||
build (toolkit >= 12.8, e.g. a runnable cuda13/cuda14), so it self-disables
|
||||
once upstream ships a driver-runnable build again.
|
||||
|
||||
The b9360 binary reuses the current release's source tree and convert scripts
|
||||
and is recorded via binary_release_tag, the same binary/source split used for
|
||||
the lemonade prebuilt."""
|
||||
if not (host.is_windows and host.is_x86_64 and host.has_usable_nvidia):
|
||||
return None
|
||||
driver = host.driver_cuda_version
|
||||
if driver is None or driver < _PINNED_BLACKWELL_DRIVER_FLOOR:
|
||||
return None
|
||||
caps = normalize_compute_caps(host.compute_caps)
|
||||
if not caps or int(caps[-1]) < _BLACKWELL_MIN_SM:
|
||||
return None
|
||||
if any(
|
||||
_windows_cuda_attempt_covers_blackwell(attempt)
|
||||
for attempt in existing_cuda_attempts
|
||||
):
|
||||
return None
|
||||
tag = _PINNED_BLACKWELL_FALLBACK_TAG
|
||||
runtime = _PINNED_BLACKWELL_FALLBACK_RUNTIME
|
||||
base = (
|
||||
f"https://github.com/{UPSTREAM_REPO}/releases/download/"
|
||||
f"{urllib.parse.quote(tag, safe = '')}"
|
||||
)
|
||||
name = f"llama-{tag}-bin-win-cuda-{runtime}-x64.zip"
|
||||
cudart_name = f"cudart-llama-bin-win-cuda-{runtime}-x64.zip"
|
||||
return AssetChoice(
|
||||
repo = UPSTREAM_REPO,
|
||||
tag = tag,
|
||||
name = name,
|
||||
url = f"{base}/{name}",
|
||||
source_label = "upstream",
|
||||
install_kind = "windows-cuda",
|
||||
runtime_line = "cuda13",
|
||||
runtime_name = cudart_name,
|
||||
runtime_url = f"{base}/{cudart_name}",
|
||||
expected_sha256 = _PINNED_BLACKWELL_LLAMA_SHA256,
|
||||
runtime_sha256 = _PINNED_BLACKWELL_CUDART_SHA256,
|
||||
selection_log = [
|
||||
f"windows_cuda_selection: pinned {tag} cuda-{runtime} Blackwell GPU "
|
||||
f"fallback (in-release cuda13 gated off by driver "
|
||||
f"{driver[0]}.{driver[1]})"
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _augment_checksums_with_pin(
|
||||
checksums: ApprovedReleaseChecksums, pin: AssetChoice
|
||||
) -> ApprovedReleaseChecksums:
|
||||
"""Add the pin's own verified hashes to a copy of the approved checksums so
|
||||
apply_approved_hashes keeps it on the published path (b9360 is not in the
|
||||
release manifest)."""
|
||||
artifacts = dict(checksums.artifacts)
|
||||
if pin.expected_sha256:
|
||||
artifacts[pin.name] = ApprovedArtifactHash(
|
||||
asset_name = pin.name,
|
||||
sha256 = pin.expected_sha256,
|
||||
repo = pin.repo,
|
||||
kind = "prebuilt",
|
||||
)
|
||||
if pin.runtime_name and pin.runtime_sha256:
|
||||
artifacts[pin.runtime_name] = ApprovedArtifactHash(
|
||||
asset_name = pin.runtime_name,
|
||||
sha256 = pin.runtime_sha256,
|
||||
repo = pin.repo,
|
||||
kind = "prebuilt",
|
||||
)
|
||||
return dataclasses_replace(checksums, artifacts = artifacts)
|
||||
|
||||
|
||||
def _with_pinned_windows_cuda_fallback(
|
||||
host: HostInfo,
|
||||
attempts: list[AssetChoice],
|
||||
checksums: ApprovedReleaseChecksums,
|
||||
) -> tuple[list[AssetChoice], ApprovedReleaseChecksums]:
|
||||
"""Insert the Blackwell pin ahead of the Windows CUDA attempts and keep it
|
||||
through apply_approved_hashes, or return the inputs unchanged when dormant.
|
||||
Gives the published install path the same GPU fallback as the simple path."""
|
||||
pin = _pinned_windows_cuda_fallback(host, attempts)
|
||||
if pin is None:
|
||||
return attempts, checksums
|
||||
return [pin, *attempts], _augment_checksums_with_pin(checksums, pin)
|
||||
|
||||
|
||||
def published_windows_cuda_attempts(
|
||||
host: HostInfo,
|
||||
release: PublishedReleaseBundle,
|
||||
|
|
@ -3335,13 +3569,26 @@ def published_windows_cuda_attempts(
|
|||
selection_preamble: Iterable[str] = (),
|
||||
) -> list[AssetChoice]:
|
||||
selection_log = list(release.selection_log) + list(selection_preamble)
|
||||
runtime_by_line = {"cuda12": "12.4", "cuda13": "13.1"}
|
||||
# Seed the runtime-line ordering from the real published windows-cuda minors
|
||||
# (their names encode the minor), so a future CUDA major published here is
|
||||
# ordered too instead of a hardcoded cuda12/cuda13 pair. Keys mirror the
|
||||
# upstream naming so windows_cuda_attempts can match them; fall back to the
|
||||
# long-standing default when the release lists no windows-cuda asset.
|
||||
published_minors: list[str] = []
|
||||
for artifact in release.artifacts:
|
||||
if artifact.install_kind != "windows-cuda":
|
||||
continue
|
||||
m = re.search(r"-bin-win-cuda-(\d+\.\d+)-x64\.zip$", artifact.asset_name)
|
||||
if m:
|
||||
published_minors.append(m.group(1))
|
||||
if not published_minors:
|
||||
published_minors = ["12.4", "13.1"]
|
||||
runtime_order = windows_cuda_attempts(
|
||||
host,
|
||||
release.upstream_tag,
|
||||
{
|
||||
f"llama-{release.upstream_tag}-bin-win-cuda-{runtime}-x64.zip": "published"
|
||||
for runtime in runtime_by_line.values()
|
||||
f"llama-{release.upstream_tag}-bin-win-cuda-{minor}-x64.zip": "published"
|
||||
for minor in published_minors
|
||||
},
|
||||
preferred_runtime_line,
|
||||
selection_log,
|
||||
|
|
@ -3370,11 +3617,20 @@ def published_windows_cuda_attempts(
|
|||
asset_url = release.assets.get(artifact.asset_name)
|
||||
if not asset_url:
|
||||
continue
|
||||
# See windows_cuda_attempts: pair the cudart bundle.
|
||||
am = re.search(r"-bin-win-cuda-(\d+)\.(\d+)-x64\.zip$", artifact.asset_name)
|
||||
# Gate the real published minor against the driver, so a published
|
||||
# windows-cuda artifact can never bypass the driver-version gate.
|
||||
if (
|
||||
am is not None
|
||||
and host.driver_cuda_version is not None
|
||||
and (int(am.group(1)), int(am.group(2))) > host.driver_cuda_version
|
||||
):
|
||||
continue
|
||||
# See windows_cuda_attempts: pair the cudart bundle for the real minor.
|
||||
runtime_archive_name: str | None = None
|
||||
runtime_archive_url: str | None = None
|
||||
if artifact.asset_name.startswith("llama-"):
|
||||
runtime = runtime_by_line[runtime_line]
|
||||
if am is not None and artifact.asset_name.startswith("llama-"):
|
||||
runtime = f"{am.group(1)}.{am.group(2)}"
|
||||
cudart_name = f"cudart-llama-bin-win-cuda-{runtime}-x64.zip"
|
||||
cudart_url = release.assets.get(cudart_name)
|
||||
if cudart_url and cudart_url != asset_url:
|
||||
|
|
@ -3927,18 +4183,23 @@ def resolve_release_asset_choice(
|
|||
torch_preference.selection_log,
|
||||
)
|
||||
if published_attempts:
|
||||
pin_attempts, pin_checksums = _with_pinned_windows_cuda_fallback(
|
||||
host, published_attempts, checksums
|
||||
)
|
||||
try:
|
||||
return apply_approved_hashes(published_attempts, checksums)
|
||||
return apply_approved_hashes(pin_attempts, pin_checksums)
|
||||
except PrebuiltFallback as exc:
|
||||
log(
|
||||
"published Windows CUDA assets ignored for install planning: "
|
||||
f"{release.repo}@{release.release_tag} ({exc})"
|
||||
)
|
||||
upstream_assets = github_release_assets(UPSTREAM_REPO, llama_tag)
|
||||
return apply_approved_hashes(
|
||||
upstream_attempts, upstream_checksums = _with_pinned_windows_cuda_fallback(
|
||||
host,
|
||||
resolve_windows_cuda_choices(host, llama_tag, upstream_assets),
|
||||
checksums,
|
||||
)
|
||||
return apply_approved_hashes(upstream_attempts, upstream_checksums)
|
||||
|
||||
published_choice: AssetChoice | None = None
|
||||
if host.is_windows and host.is_x86_64:
|
||||
|
|
@ -5010,6 +5271,189 @@ def linux_runtime_dirs(binary_path: Path) -> list[str]:
|
|||
return linux_runtime_dirs_for_required_libraries(missing)
|
||||
|
||||
|
||||
# macOS prebuilt compatibility. Upstream macos prebuilts built on a newer macOS
|
||||
# (e.g. minos=26, referencing Metal-4 symbols) fail dyld load on macOS 14/15. We
|
||||
# read the host macOS version and each binary's minimum-OS so selection can skip
|
||||
# a too-new prebuilt and walk back to the newest release that runs on this host.
|
||||
# Mach-O constants (Apple mach-o/fat.h, mach-o/loader.h, mach/machine.h).
|
||||
_MACHO_FAT_MAGICS = {0xCAFEBABE, 0xCAFEBABF} # universal binary (32/64-bit fat)
|
||||
_LC_VERSION_MIN_MACOSX = 0x24 # legacy min-macOS load command
|
||||
_LC_BUILD_VERSION = 0x32 # modern platform+minos+sdk load command
|
||||
_MACHO_PLATFORM_MACOS = 1 # LC_BUILD_VERSION platform id for macOS (iOS=2, ...)
|
||||
# CPU types (base | ABI64); used to pick the host slice in a fat binary.
|
||||
_CPU_TYPE_X86_64 = 0x01000007
|
||||
_CPU_TYPE_ARM64 = 0x0100000C
|
||||
|
||||
|
||||
def parse_macos_version(value: str | None) -> tuple[int, int] | None:
|
||||
"""Parse a macOS product version string into (major, minor).
|
||||
|
||||
Handles "14.7.1", "15.5", "26.0" and bare "26". Returns None when the
|
||||
value is empty or cannot be parsed (callers then defer to runtime
|
||||
validation rather than rejecting every prebuilt)."""
|
||||
if not value:
|
||||
return None
|
||||
match = re.match(r"\s*(\d+)(?:\.(\d+))?", str(value))
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(1)), int(match.group(2) or 0)
|
||||
|
||||
|
||||
def host_supports_macos_minos(host: HostInfo, minos: tuple[int, int] | None) -> bool:
|
||||
"""True if a prebuilt requiring `minos` can load on this host. Unknown host
|
||||
version or unknown minos -> True: let runtime validation decide instead of
|
||||
rejecting a binary we cannot reason about."""
|
||||
if minos is None or host.macos_version is None:
|
||||
return True
|
||||
return host.macos_version >= minos
|
||||
|
||||
|
||||
def _macho_slice_minos(data: bytes, offset: int) -> tuple[int, int] | None:
|
||||
"""Minimum macOS for a single thin Mach-O at `offset`, via LC_BUILD_VERSION
|
||||
(platform macOS) or the legacy LC_VERSION_MIN_MACOSX. None if absent."""
|
||||
if offset + 4 > len(data):
|
||||
return None
|
||||
magic = struct.unpack_from(">I", data, offset)[0]
|
||||
if magic in (0xFEEDFACE, 0xFEEDFACF):
|
||||
endian, is64 = ">", magic == 0xFEEDFACF
|
||||
elif magic in (0xCEFAEDFE, 0xCFFAEDFE):
|
||||
endian, is64 = "<", magic == 0xCFFAEDFE
|
||||
else:
|
||||
return None
|
||||
header_size = 32 if is64 else 28
|
||||
if offset + header_size > len(data):
|
||||
return None
|
||||
ncmds = struct.unpack_from(endian + "I", data, offset + 16)[0]
|
||||
cursor = offset + header_size
|
||||
for _ in range(ncmds):
|
||||
if cursor + 8 > len(data):
|
||||
break
|
||||
cmd, cmdsize = struct.unpack_from(endian + "II", data, cursor)
|
||||
if cmdsize < 8:
|
||||
break
|
||||
if cmd == _LC_BUILD_VERSION and cursor + 16 <= len(data):
|
||||
platform_id, minos = struct.unpack_from(endian + "II", data, cursor + 8)
|
||||
if platform_id == _MACHO_PLATFORM_MACOS:
|
||||
return (minos >> 16) & 0xFFFF, (minos >> 8) & 0xFF
|
||||
elif cmd == _LC_VERSION_MIN_MACOSX and cursor + 12 <= len(data):
|
||||
version = struct.unpack_from(endian + "I", data, cursor + 8)[0]
|
||||
return (version >> 16) & 0xFFFF, (version >> 8) & 0xFF
|
||||
cursor += cmdsize
|
||||
return None
|
||||
|
||||
|
||||
def macho_minimum_macos(
|
||||
path: Path, host: HostInfo | None = None
|
||||
) -> tuple[int, int] | None:
|
||||
"""Minimum macOS (major, minor) a Mach-O binary or dylib requires.
|
||||
|
||||
Pure-Python so it works on consumer Macs without the Xcode command line
|
||||
tools (otool/vtool). For universal binaries it prefers the host-arch slice,
|
||||
else the highest minos found. Returns None for non-Mach-O files or when no
|
||||
version load command is present."""
|
||||
try:
|
||||
data = path.read_bytes()
|
||||
except Exception:
|
||||
return None
|
||||
if len(data) < 8:
|
||||
return None
|
||||
magic = struct.unpack_from(">I", data, 0)[0]
|
||||
if magic in _MACHO_FAT_MAGICS:
|
||||
is64 = magic == 0xCAFEBABF
|
||||
nfat = struct.unpack_from(">I", data, 4)[0]
|
||||
entry = 8
|
||||
slices: list[tuple[int, tuple[int, int]]] = []
|
||||
for _ in range(nfat):
|
||||
if is64:
|
||||
if entry + 32 > len(data):
|
||||
break
|
||||
cputype = struct.unpack_from(">I", data, entry)[0]
|
||||
slice_offset = struct.unpack_from(">Q", data, entry + 8)[0]
|
||||
entry += 32
|
||||
else:
|
||||
if entry + 20 > len(data):
|
||||
break
|
||||
cputype = struct.unpack_from(">I", data, entry)[0]
|
||||
slice_offset = struct.unpack_from(">I", data, entry + 8)[0]
|
||||
entry += 20
|
||||
minos = _macho_slice_minos(data, slice_offset)
|
||||
if minos is not None:
|
||||
slices.append((cputype, minos))
|
||||
if not slices:
|
||||
return None
|
||||
if host is not None:
|
||||
want = (
|
||||
_CPU_TYPE_ARM64
|
||||
if host.is_arm64
|
||||
else (_CPU_TYPE_X86_64 if host.is_x86_64 else None)
|
||||
)
|
||||
for cputype, minos in slices:
|
||||
if cputype == want:
|
||||
return minos
|
||||
return max(minos for _cputype, minos in slices)
|
||||
return _macho_slice_minos(data, 0)
|
||||
|
||||
|
||||
def looks_like_macos_incompatibility(text: str) -> bool:
|
||||
"""True when dyld output means a prebuilt needs a newer macOS than the host
|
||||
(the runtime backstop for cases the static minos scan cannot read)."""
|
||||
if not text:
|
||||
return False
|
||||
if "built for macOS" in text and "newer than running OS" in text:
|
||||
return True
|
||||
return "Symbol not found" in text and "MTLResidency" in text
|
||||
|
||||
|
||||
def macos_binary_minos_issues(
|
||||
binaries: Iterable[Path],
|
||||
install_dir: Path,
|
||||
host: HostInfo,
|
||||
) -> list[str]:
|
||||
"""Issue strings for every installed Mach-O whose minimum macOS exceeds the
|
||||
host. Scans the given executables plus every bundled .dylib next to them --
|
||||
the dyld failure originates in libggml-metal.dylib, not the executable."""
|
||||
candidates: list[Path] = list(binaries)
|
||||
bin_dir = install_dir / "build" / "bin"
|
||||
if bin_dir.is_dir():
|
||||
candidates.extend(sorted(bin_dir.rglob("*.dylib")))
|
||||
|
||||
issues: list[str] = []
|
||||
seen: set[Path] = set()
|
||||
for path in candidates:
|
||||
try:
|
||||
resolved = path.resolve()
|
||||
except Exception:
|
||||
resolved = path
|
||||
if resolved in seen or not path.exists():
|
||||
continue
|
||||
seen.add(resolved)
|
||||
minos = macho_minimum_macos(path, host)
|
||||
if minos is not None and not host_supports_macos_minos(host, minos):
|
||||
issues.append(
|
||||
f"{path.name}: built for macOS {minos[0]}.{minos[1]} > "
|
||||
f"host macOS {host.macos_version[0]}.{host.macos_version[1]}"
|
||||
)
|
||||
return issues
|
||||
|
||||
|
||||
def preflight_macos_installed_binaries(
|
||||
binaries: Iterable[Path],
|
||||
install_dir: Path,
|
||||
host: HostInfo,
|
||||
) -> None:
|
||||
"""Reject a macos prebuilt whose minimum-OS is newer than the host so the
|
||||
release walk-back advances to the newest compatible release. No-op when the
|
||||
host macOS version is unknown (runtime validation remains the backstop)."""
|
||||
if not host.is_macos or host.macos_version is None:
|
||||
return
|
||||
issues = macos_binary_minos_issues(binaries, install_dir, host)
|
||||
if issues:
|
||||
raise PrebuiltFallback(
|
||||
"macos prebuilt requires a newer macOS than this host:\n"
|
||||
+ "\n".join(issues)
|
||||
)
|
||||
|
||||
|
||||
def preflight_linux_installed_binaries(
|
||||
binaries: Iterable[Path],
|
||||
install_dir: Path,
|
||||
|
|
@ -5168,10 +5612,17 @@ def validate_quantize(
|
|||
or not quantized_path.exists()
|
||||
or quantized_path.stat().st_size == 0
|
||||
):
|
||||
combined = result.stdout + ("\n" + result.stderr if result.stderr else "")
|
||||
# Backstop for prebuilts the static minos scan could not read: a dyld
|
||||
# "built for macOS N" / missing Metal symbol failure means this binary
|
||||
# needs a newer macOS than the host, so fall back to an older release.
|
||||
prefix = (
|
||||
"macos prebuilt requires a newer macOS than this host: "
|
||||
if looks_like_macos_incompatibility(combined)
|
||||
else ""
|
||||
)
|
||||
raise PrebuiltFallback(
|
||||
"llama-quantize validation failed:\n"
|
||||
+ result.stdout
|
||||
+ ("\n" + result.stderr if result.stderr else "")
|
||||
prefix + "llama-quantize validation failed:\n" + combined
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -5589,6 +6040,14 @@ def resolve_install_release_plans(
|
|||
requested_tag == "latest" and not published_release_tag
|
||||
)
|
||||
release_limit = max(1, max_release_fallbacks)
|
||||
# macOS may need to walk past a run of too-new prebuilts. Only when the host
|
||||
# version is known; otherwise keep the default (cannot tell up front).
|
||||
if (
|
||||
host.is_macos
|
||||
and allow_older_release_fallback
|
||||
and host.macos_version is not None
|
||||
):
|
||||
release_limit = max(release_limit, DEFAULT_MAX_MACOS_RELEASE_FALLBACKS)
|
||||
plans: list[InstallReleasePlan] = []
|
||||
last_error: PrebuiltFallback | None = None
|
||||
|
||||
|
|
@ -5973,6 +6432,7 @@ def validate_prebuilt_choice(
|
|||
choice, host, install_dir, work_dir
|
||||
)
|
||||
preflight_linux_installed_binaries((server_path, quantize_path), install_dir, host)
|
||||
preflight_macos_installed_binaries((server_path, quantize_path), install_dir, host)
|
||||
ensure_repo_shape(install_dir)
|
||||
write_prebuilt_metadata(
|
||||
install_dir,
|
||||
|
|
|
|||
|
|
@ -1084,6 +1084,15 @@ else
|
|||
_IS_MACOS_ARM64=true
|
||||
fi
|
||||
|
||||
# macOS: pin a low deployment target so the source build loads on
|
||||
# older macOS too (else a macOS 26 host stamps minos=26). Set before
|
||||
# CPU_FALLBACK_CMAKE_ARGS copies CMAKE_ARGS so both paths inherit it.
|
||||
if [ "$_HOST_SYSTEM" = "Darwin" ]; then
|
||||
_MACOS_DEPLOYMENT_TARGET="${UNSLOTH_MACOS_DEPLOYMENT_TARGET:-13.3}"
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_OSX_DEPLOYMENT_TARGET=${_MACOS_DEPLOYMENT_TARGET}"
|
||||
export MACOSX_DEPLOYMENT_TARGET="${_MACOS_DEPLOYMENT_TARGET}"
|
||||
fi
|
||||
|
||||
if command -v ccache &>/dev/null; then
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache"
|
||||
fi
|
||||
|
|
|
|||
31
studio/src-tauri/Cargo.lock
generated
31
studio/src-tauri/Cargo.lock
generated
|
|
@ -2731,15 +2731,14 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.76"
|
||||
version = "0.10.80"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
|
||||
checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"cfg-if",
|
||||
"foreign-types 0.3.2",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"openssl-macros",
|
||||
"openssl-sys",
|
||||
]
|
||||
|
|
@ -2763,9 +2762,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
|||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.112"
|
||||
version = "0.9.116"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
|
||||
checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
|
|
@ -2978,7 +2977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
|
||||
dependencies = [
|
||||
"phf_shared 0.10.0",
|
||||
"rand 0.8.5",
|
||||
"rand 0.8.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2988,7 +2987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared 0.11.3",
|
||||
"rand 0.8.5",
|
||||
"rand 0.8.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3361,9 +3360,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha 0.3.1",
|
||||
|
|
@ -3382,9 +3381,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.0"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8"
|
||||
checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
|
||||
dependencies = [
|
||||
"chacha20",
|
||||
"getrandom 0.4.2",
|
||||
|
|
@ -3765,9 +3764,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
|||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.10"
|
||||
version = "0.103.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
|
|
@ -4450,9 +4449,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.45"
|
||||
version = "0.4.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973"
|
||||
checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
|
|
@ -5380,7 +5379,7 @@ dependencies = [
|
|||
"log",
|
||||
"open",
|
||||
"process-wrap",
|
||||
"rand 0.10.0",
|
||||
"rand 0.10.1",
|
||||
"regex",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
|
|
|
|||
322
tests/studio/install/test_macos_version_compat.py
Normal file
322
tests/studio/install/test_macos_version_compat.py
Normal file
|
|
@ -0,0 +1,322 @@
|
|||
"""Tests for the host-macOS-version-aware llama.cpp prebuilt selection added
|
||||
for the Mac "Failing CI" fix.
|
||||
|
||||
Covers: parse_macos_version, host_supports_macos_minos, the pure-Python Mach-O
|
||||
minimum-OS parser (macho_minimum_macos), the dyld-incompatibility classifier,
|
||||
the install preflight that rejects a too-new prebuilt, and the deeper macOS
|
||||
release walk-back in resolve_simple_install_release_plans.
|
||||
|
||||
No GPU, no network, no torch, no real Mach-O toolchain required -- the Mach-O
|
||||
samples are synthesized in-process and all I/O is monkeypatched.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
PACKAGE_ROOT = Path(__file__).resolve().parents[3]
|
||||
MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py"
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"studio_install_llama_prebuilt_macos", MODULE_PATH
|
||||
)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
ILP = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = ILP
|
||||
SPEC.loader.exec_module(ILP)
|
||||
|
||||
HostInfo = ILP.HostInfo
|
||||
PrebuiltFallback = ILP.PrebuiltFallback
|
||||
|
||||
_CPU_TYPE_ARM64 = 0x0100000C
|
||||
_CPU_TYPE_X86_64 = 0x01000007
|
||||
|
||||
|
||||
def make_macos_host(macos_version, *, arm64 = True):
|
||||
return HostInfo(
|
||||
system = "Darwin",
|
||||
machine = "arm64" if arm64 else "x86_64",
|
||||
is_windows = False,
|
||||
is_linux = False,
|
||||
is_macos = True,
|
||||
is_x86_64 = not arm64,
|
||||
is_arm64 = arm64,
|
||||
nvidia_smi = None,
|
||||
driver_cuda_version = None,
|
||||
compute_caps = [],
|
||||
visible_cuda_devices = None,
|
||||
has_physical_nvidia = False,
|
||||
has_usable_nvidia = False,
|
||||
macos_version = macos_version,
|
||||
)
|
||||
|
||||
|
||||
def thin_macho(minos = (14, 0), *, cputype = _CPU_TYPE_ARM64, build_version = True):
|
||||
"""Synthesize a minimal little-endian 64-bit Mach-O carrying a macOS
|
||||
minimum-version load command."""
|
||||
encoded = (minos[0] << 16) | (minos[1] << 8)
|
||||
if build_version:
|
||||
# LC_BUILD_VERSION: cmd, cmdsize, platform(=1 macOS), minos, sdk, ntools
|
||||
load_command = struct.pack("<6I", 0x32, 24, 1, encoded, encoded, 0)
|
||||
else:
|
||||
# LC_VERSION_MIN_MACOSX: cmd, cmdsize, version, sdk
|
||||
load_command = struct.pack("<4I", 0x24, 16, encoded, encoded)
|
||||
header = struct.pack("<8I", 0xFEEDFACF, cputype, 0, 0x2, 1, len(load_command), 0, 0)
|
||||
return header + load_command
|
||||
|
||||
|
||||
def fat_macho(slices):
|
||||
"""Synthesize a big-endian universal binary from (cputype, thin_bytes)."""
|
||||
header = struct.pack(">2I", 0xCAFEBABE, len(slices))
|
||||
data_offset = 8 + 20 * len(slices)
|
||||
arch_entries = b""
|
||||
body = b""
|
||||
for cputype, thin in slices:
|
||||
offset = data_offset + len(body)
|
||||
arch_entries += struct.pack(">5I", cputype, 0, offset, len(thin), 0)
|
||||
body += thin
|
||||
return header + arch_entries + body
|
||||
|
||||
|
||||
class TestParseMacosVersion:
|
||||
@pytest.mark.parametrize(
|
||||
"value, expected",
|
||||
[
|
||||
("14.7.1", (14, 7)),
|
||||
("15.5", (15, 5)),
|
||||
("26.0", (26, 0)),
|
||||
("26", (26, 0)),
|
||||
("13", (13, 0)),
|
||||
("", None),
|
||||
(None, None),
|
||||
("not-a-version", None),
|
||||
],
|
||||
)
|
||||
def test_parse(self, value, expected):
|
||||
assert ILP.parse_macos_version(value) == expected
|
||||
|
||||
|
||||
class TestHostSupportsMacosMinos:
|
||||
def test_older_host_rejects_newer_prebuilt(self):
|
||||
assert not ILP.host_supports_macos_minos(make_macos_host((14, 0)), (26, 0))
|
||||
|
||||
def test_same_version_supported(self):
|
||||
assert ILP.host_supports_macos_minos(make_macos_host((26, 0)), (26, 0))
|
||||
|
||||
def test_newer_host_supports_older_prebuilt(self):
|
||||
assert ILP.host_supports_macos_minos(make_macos_host((15, 5)), (14, 0))
|
||||
|
||||
def test_unknown_host_defers_to_runtime(self):
|
||||
assert ILP.host_supports_macos_minos(make_macos_host(None), (26, 0))
|
||||
|
||||
def test_unknown_minos_defers_to_runtime(self):
|
||||
assert ILP.host_supports_macos_minos(make_macos_host((14, 0)), None)
|
||||
|
||||
|
||||
class TestMachoMinimumMacos:
|
||||
def test_build_version_thin(self, tmp_path):
|
||||
path = tmp_path / "lib.dylib"
|
||||
path.write_bytes(thin_macho((26, 0)))
|
||||
assert ILP.macho_minimum_macos(path) == (26, 0)
|
||||
|
||||
def test_legacy_version_min_thin(self, tmp_path):
|
||||
path = tmp_path / "lib.dylib"
|
||||
path.write_bytes(thin_macho((14, 0), build_version = False))
|
||||
assert ILP.macho_minimum_macos(path) == (14, 0)
|
||||
|
||||
def test_universal_prefers_host_arch_slice(self, tmp_path):
|
||||
# arm64 slice needs macOS 14, x86_64 slice needs macOS 26.
|
||||
path = tmp_path / "fat"
|
||||
path.write_bytes(
|
||||
fat_macho(
|
||||
[
|
||||
(_CPU_TYPE_ARM64, thin_macho((14, 0), cputype = _CPU_TYPE_ARM64)),
|
||||
(_CPU_TYPE_X86_64, thin_macho((26, 0), cputype = _CPU_TYPE_X86_64)),
|
||||
]
|
||||
)
|
||||
)
|
||||
assert ILP.macho_minimum_macos(path, make_macos_host((14, 0))) == (14, 0)
|
||||
assert ILP.macho_minimum_macos(path, make_macos_host((26, 0), arm64 = False)) == (
|
||||
26,
|
||||
0,
|
||||
)
|
||||
|
||||
def test_non_macho_returns_none(self, tmp_path):
|
||||
path = tmp_path / "script.sh"
|
||||
path.write_bytes(b'#!/bin/sh\nexec real "$@"\n')
|
||||
assert ILP.macho_minimum_macos(path) is None
|
||||
|
||||
def test_missing_file_returns_none(self, tmp_path):
|
||||
assert ILP.macho_minimum_macos(tmp_path / "nope") is None
|
||||
|
||||
|
||||
class TestLooksLikeMacosIncompatibility:
|
||||
def test_built_for_newer_os(self):
|
||||
assert ILP.looks_like_macos_incompatibility(
|
||||
"dyld: ... (built for macOS 26.0 which is newer than running OS)"
|
||||
)
|
||||
|
||||
def test_metal_residency_symbol(self):
|
||||
assert ILP.looks_like_macos_incompatibility(
|
||||
"Symbol not found: _OBJC_CLASS_$_MTLResidencySetDescriptor"
|
||||
)
|
||||
|
||||
def test_benign_error(self):
|
||||
assert not ILP.looks_like_macos_incompatibility("some unrelated failure")
|
||||
|
||||
def test_empty(self):
|
||||
assert not ILP.looks_like_macos_incompatibility("")
|
||||
|
||||
|
||||
class TestPreflightMacosInstalledBinaries:
|
||||
def _install_dir(self, tmp_path, dylib_minos):
|
||||
bin_dir = tmp_path / "build" / "bin"
|
||||
bin_dir.mkdir(parents = True)
|
||||
(bin_dir / "libggml-metal.dylib").write_bytes(thin_macho(dylib_minos))
|
||||
server = tmp_path / "llama-server"
|
||||
server.write_bytes(thin_macho(dylib_minos))
|
||||
quantize = tmp_path / "llama-quantize"
|
||||
quantize.write_bytes(thin_macho(dylib_minos))
|
||||
return tmp_path, (server, quantize)
|
||||
|
||||
def test_rejects_too_new_dylib(self, tmp_path):
|
||||
install_dir, binaries = self._install_dir(tmp_path, (26, 0))
|
||||
with pytest.raises(PrebuiltFallback, match = "newer macOS"):
|
||||
ILP.preflight_macos_installed_binaries(
|
||||
binaries, install_dir, make_macos_host((14, 0))
|
||||
)
|
||||
|
||||
def test_accepts_compatible_prebuilt(self, tmp_path):
|
||||
install_dir, binaries = self._install_dir(tmp_path, (14, 0))
|
||||
# Must not raise on a macOS 15 host.
|
||||
ILP.preflight_macos_installed_binaries(
|
||||
binaries, install_dir, make_macos_host((15, 5))
|
||||
)
|
||||
|
||||
def test_skips_when_host_version_unknown(self, tmp_path):
|
||||
install_dir, binaries = self._install_dir(tmp_path, (26, 0))
|
||||
# Unknown host version -> defer to runtime validation, do not raise.
|
||||
ILP.preflight_macos_installed_binaries(
|
||||
binaries, install_dir, make_macos_host(None)
|
||||
)
|
||||
|
||||
def test_noop_on_non_macos_host(self, tmp_path):
|
||||
install_dir, binaries = self._install_dir(tmp_path, (26, 0))
|
||||
linux_host = HostInfo(
|
||||
system = "Linux",
|
||||
machine = "x86_64",
|
||||
is_windows = False,
|
||||
is_linux = True,
|
||||
is_macos = False,
|
||||
is_x86_64 = True,
|
||||
is_arm64 = False,
|
||||
nvidia_smi = None,
|
||||
driver_cuda_version = None,
|
||||
compute_caps = [],
|
||||
visible_cuda_devices = None,
|
||||
has_physical_nvidia = False,
|
||||
has_usable_nvidia = False,
|
||||
)
|
||||
ILP.preflight_macos_installed_binaries(binaries, install_dir, linux_host)
|
||||
|
||||
|
||||
def _fake_macos_releases(tags):
|
||||
return [
|
||||
{
|
||||
"tag_name": tag,
|
||||
"assets": [
|
||||
{
|
||||
"name": f"llama-{tag}-bin-macos-arm64.tar.gz",
|
||||
"browser_download_url": f"https://example.com/{tag}.tar.gz",
|
||||
}
|
||||
],
|
||||
}
|
||||
for tag in tags
|
||||
]
|
||||
|
||||
|
||||
class TestMacosReleaseWalkback:
|
||||
"""A known-version macOS host must generate enough older-release plans to
|
||||
walk back past a run of too-new prebuilts; unknown-version and non-macOS
|
||||
hosts keep the conservative 2-release default."""
|
||||
|
||||
TAGS = [f"b{n}" for n in range(9437, 9400, -1)] # 37 newest-first releases
|
||||
|
||||
def _patch_releases(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ILP,
|
||||
"iter_release_payloads_by_time",
|
||||
lambda repo, published_release_tag, requested_tag, host = None: (
|
||||
_fake_macos_releases(self.TAGS)
|
||||
),
|
||||
)
|
||||
|
||||
def test_known_macos_host_walks_back_deeper(self, monkeypatch):
|
||||
self._patch_releases(monkeypatch)
|
||||
_tag, plans = ILP.resolve_simple_install_release_plans(
|
||||
"latest",
|
||||
make_macos_host((14, 0)),
|
||||
"ggml-org/llama.cpp",
|
||||
"",
|
||||
)
|
||||
assert len(plans) == ILP.DEFAULT_MAX_MACOS_RELEASE_FALLBACKS
|
||||
assert len(plans) > ILP.DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS
|
||||
|
||||
def test_unknown_macos_host_uses_default(self, monkeypatch):
|
||||
self._patch_releases(monkeypatch)
|
||||
_tag, plans = ILP.resolve_simple_install_release_plans(
|
||||
"latest",
|
||||
make_macos_host(None),
|
||||
"ggml-org/llama.cpp",
|
||||
"",
|
||||
)
|
||||
assert len(plans) == ILP.DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS
|
||||
|
||||
|
||||
class TestForwardsBackwardsCompat:
|
||||
"""The gate is host >= prebuilt minos with no hardcoded version, so it holds
|
||||
for older and future macOS alike. Emulate the walk-back over a release set
|
||||
spanning several minos tiers and assert each host takes the newest release
|
||||
it can load."""
|
||||
|
||||
# Newest first: future 27 builds, current 26 builds, an old 14 tier, a 13.
|
||||
RELEASES = [
|
||||
("b9600", (27, 0)),
|
||||
("b9450", (26, 0)),
|
||||
("b9415", (14, 0)),
|
||||
("b8300", (13, 0)),
|
||||
]
|
||||
|
||||
def _select(self, tmp_path, host_version):
|
||||
for tag, minos in self.RELEASES:
|
||||
bin_dir = tmp_path / tag / "build" / "bin"
|
||||
bin_dir.mkdir(parents = True)
|
||||
(bin_dir / "libggml-metal.dylib").write_bytes(thin_macho(minos))
|
||||
try:
|
||||
ILP.preflight_macos_installed_binaries(
|
||||
(), tmp_path / tag, make_macos_host(host_version)
|
||||
)
|
||||
return tag
|
||||
except PrebuiltFallback:
|
||||
continue
|
||||
return None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"host_version, expected",
|
||||
[
|
||||
((13, 0), "b8300"), # older host takes the older prebuilt
|
||||
((14, 7), "b9415"), # backwards: skip 26/27, take newest that loads
|
||||
((15, 5), "b9415"),
|
||||
((26, 0), "b9450"), # unchanged: newest <= host
|
||||
((27, 1), "b9600"), # forwards: future host takes the future build
|
||||
],
|
||||
)
|
||||
def test_selects_newest_loadable(self, tmp_path, host_version, expected):
|
||||
assert self._select(tmp_path, host_version) == expected
|
||||
|
||||
def test_host_below_prebuilt_floor_falls_through(self, tmp_path):
|
||||
# macOS 12 is below every prebuilt -> nothing matches -> source build.
|
||||
assert self._select(tmp_path, (12, 0)) is None
|
||||
|
|
@ -52,6 +52,9 @@ compatible_windows_runtime_lines = (
|
|||
runtime_line_from_cuda_version = INSTALL_LLAMA_PREBUILT.runtime_line_from_cuda_version
|
||||
apply_approved_hashes = INSTALL_LLAMA_PREBUILT.apply_approved_hashes
|
||||
linux_cuda_choice_from_release = INSTALL_LLAMA_PREBUILT.linux_cuda_choice_from_release
|
||||
parse_direct_linux_release_bundle = (
|
||||
INSTALL_LLAMA_PREBUILT.parse_direct_linux_release_bundle
|
||||
)
|
||||
windows_cuda_attempts = INSTALL_LLAMA_PREBUILT.windows_cuda_attempts
|
||||
resolve_upstream_asset_choice = INSTALL_LLAMA_PREBUILT.resolve_upstream_asset_choice
|
||||
resolve_requested_install_tag = INSTALL_LLAMA_PREBUILT.resolve_requested_install_tag
|
||||
|
|
@ -74,6 +77,14 @@ windows_cuda_upstream_asset_names = (
|
|||
INSTALL_LLAMA_PREBUILT.windows_cuda_upstream_asset_names
|
||||
)
|
||||
env_int = INSTALL_LLAMA_PREBUILT.env_int
|
||||
direct_upstream_release_plan = INSTALL_LLAMA_PREBUILT.direct_upstream_release_plan
|
||||
_pinned_windows_cuda_fallback = INSTALL_LLAMA_PREBUILT._pinned_windows_cuda_fallback
|
||||
CudaRuntimePreference = INSTALL_LLAMA_PREBUILT.CudaRuntimePreference
|
||||
published_windows_cuda_attempts = INSTALL_LLAMA_PREBUILT.published_windows_cuda_attempts
|
||||
_windows_cuda_attempt_covers_blackwell = (
|
||||
INSTALL_LLAMA_PREBUILT._windows_cuda_attempt_covers_blackwell
|
||||
)
|
||||
resolve_release_asset_choice = INSTALL_LLAMA_PREBUILT.resolve_release_asset_choice
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -397,6 +408,44 @@ class TestCompatibleLinuxRuntimeLines:
|
|||
host = make_host(driver_cuda_version = (13, 0))
|
||||
assert compatible_linux_runtime_lines(host) == ["cuda13", "cuda12"]
|
||||
|
||||
def test_future_major_derives_lines(self):
|
||||
# A future major (14.x) offers cuda14 first, then older majors.
|
||||
host = make_host(driver_cuda_version = (14, 0))
|
||||
assert compatible_linux_runtime_lines(host) == ["cuda14", "cuda13", "cuda12"]
|
||||
|
||||
|
||||
class TestParseDirectLinuxReleaseBundle:
|
||||
def _release(self, *targets):
|
||||
names = [f"app-bTEST-linux-x64-{t}.tar.gz" for t in targets]
|
||||
return {
|
||||
"tag_name": "bTEST",
|
||||
"assets": [
|
||||
{"name": n, "browser_download_url": "https://x/" + n} for n in names
|
||||
],
|
||||
}
|
||||
|
||||
def _cuda_artifact(self, bundle):
|
||||
return [a for a in bundle.artifacts if a.install_kind == "linux-cuda"][0]
|
||||
|
||||
def test_parses_known_cuda13_bundle(self):
|
||||
bundle = parse_direct_linux_release_bundle(
|
||||
"unslothai/llama.cpp", self._release("cuda13-newer")
|
||||
)
|
||||
assert bundle is not None
|
||||
assert self._cuda_artifact(bundle).runtime_line == "cuda13"
|
||||
|
||||
def test_parses_future_cuda_major_with_forward_profile(self):
|
||||
# A future major name parses and inherits the newest known major's
|
||||
# coverage for the same class as a forward default.
|
||||
bundle = parse_direct_linux_release_bundle(
|
||||
"unslothai/llama.cpp", self._release("cuda14-newer")
|
||||
)
|
||||
assert bundle is not None
|
||||
art = self._cuda_artifact(bundle)
|
||||
assert art.runtime_line == "cuda14"
|
||||
assert art.coverage_class == "newer"
|
||||
assert art.max_sm == 120 # inherited from cuda13-newer
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# G. pick_windows_cuda_runtime + compatible_windows_runtime_lines
|
||||
|
|
@ -442,6 +491,10 @@ class TestCompatibleWindowsRuntimeLines:
|
|||
host = make_host(driver_cuda_version = (13, 0))
|
||||
assert compatible_windows_runtime_lines(host) == ["cuda13", "cuda12"]
|
||||
|
||||
def test_future_major_derives_lines(self):
|
||||
host = make_host(driver_cuda_version = (14, 0))
|
||||
assert compatible_windows_runtime_lines(host) == ["cuda14", "cuda13", "cuda12"]
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# H. runtime_line_from_cuda_version
|
||||
|
|
@ -1777,12 +1830,23 @@ class TestWindowsCudaAttempts:
|
|||
assert result[0].runtime_line == "cuda13"
|
||||
assert result[1].runtime_line == "cuda12"
|
||||
|
||||
def test_driver_13_0_cuda13_dlls_selects_cuda13_asset(self, monkeypatch):
|
||||
def test_driver_below_published_minor_is_gated_to_cuda12(self, monkeypatch):
|
||||
# A 13.0 driver cannot run a 13.1 build (forward minor), so it is gated
|
||||
# out of cuda13 and falls back to the cuda12 build it can run, even when
|
||||
# only the cuda13 runtime libs are detected.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (13, 0))
|
||||
assets = self._upstream("13.1", "12.4")
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert len(result) == 1
|
||||
assert result[0].runtime_line == "cuda12"
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-12.4-x64.zip"
|
||||
|
||||
def test_driver_at_published_minor_selects_cuda13(self, monkeypatch):
|
||||
# A 13.1 driver matches the published 13.1 build exactly.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (13, 1))
|
||||
assets = self._upstream("13.1", "12.4")
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert result[0].runtime_line == "cuda13"
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.1-x64.zip"
|
||||
|
||||
|
|
@ -1885,6 +1949,464 @@ class TestWindowsCudaAttempts:
|
|||
assert attempt.runtime_url is None
|
||||
assert attempt.runtime_name is None
|
||||
|
||||
def test_tracks_upstream_cuda13_minor_bump(self, monkeypatch):
|
||||
# ggml-org bumped the published Windows cuda13 build 13.1 -> 13.3; the
|
||||
# selector must follow it instead of the old hardcoded 13.1 (#5861).
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (13, 3))
|
||||
assets = self._upstream("13.3", "12.4")
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert result[0].runtime_line == "cuda13"
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip"
|
||||
|
||||
def test_cuda13_minor_bump_pairs_matching_cudart(self, monkeypatch):
|
||||
# The paired cudart bundle must track the same bumped minor.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (13, 3))
|
||||
assets = {
|
||||
f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip": "https://example.com/llama-13.3",
|
||||
"cudart-llama-bin-win-cuda-13.3-x64.zip": "https://example.com/cudart-13.3",
|
||||
f"llama-{self.TAG}-bin-win-cuda-12.4-x64.zip": "https://example.com/llama-12.4",
|
||||
"cudart-llama-bin-win-cuda-12.4-x64.zip": "https://example.com/cudart-12.4",
|
||||
}
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip"
|
||||
assert result[0].runtime_name == "cudart-llama-bin-win-cuda-13.3-x64.zip"
|
||||
|
||||
def test_driver_below_published_minor_does_not_get_newer_build(self, monkeypatch):
|
||||
# ggml-org ships only cuda-13.3; a 13.1 driver cannot run it (forward
|
||||
# minor), so it is gated to the cuda-12.4 build instead of an
|
||||
# unguaranteed 13.3. A 13.3 driver still gets 13.3 (see other tests).
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (13, 1))
|
||||
assets = self._upstream("13.3", "12.4")
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert result[0].runtime_line == "cuda12"
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-12.4-x64.zip"
|
||||
|
||||
def test_tracks_future_cuda13_minor(self, monkeypatch):
|
||||
# A later within-major bump (13.4) is tracked the same as 13.3.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (13, 4))
|
||||
assets = self._upstream("13.4", "12.4")
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.4-x64.zip"
|
||||
|
||||
def test_new_cuda_major_selected_when_published(self, monkeypatch):
|
||||
# A new CUDA major (14.x) driver picks the published cuda14 build.
|
||||
mock_windows_runtime(monkeypatch, ["cuda14", "cuda13", "cuda12"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (14, 0))
|
||||
assets = self._upstream("14.0", "13.3", "12.4")
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert result[0].runtime_line == "cuda14"
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-14.0-x64.zip"
|
||||
|
||||
def test_new_cuda_major_degrades_to_published_cuda13(self, monkeypatch):
|
||||
# A 14.x driver with no cuda14 build runs the newest published cuda13
|
||||
# build via backward compatibility.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (14, 0))
|
||||
assets = self._upstream("13.3", "12.4")
|
||||
result = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# N.1b. _pinned_windows_cuda_fallback -- pinned b9360 cuda-13.1 Blackwell fallback
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestPinnedBlackwellCudaFallback:
|
||||
"""A Blackwell host on a 13.1/13.2 driver, gated off the in-release 13.3
|
||||
build, gets the pinned immutable b9360 cuda-13.1 GPU build instead of the
|
||||
CPU-only cuda-12.4 drop. The pin is dormant for everyone else."""
|
||||
|
||||
TAG = "b8508"
|
||||
|
||||
def _win_host(self, driver, caps):
|
||||
return make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = driver,
|
||||
compute_caps = caps,
|
||||
)
|
||||
|
||||
def test_pin_offered_for_driver_13_1_blackwell(self):
|
||||
pin = _pinned_windows_cuda_fallback(self._win_host((13, 1), ["120"]), [])
|
||||
assert pin is not None
|
||||
assert pin.tag == "b9360"
|
||||
assert pin.runtime_line == "cuda13"
|
||||
assert pin.name == "llama-b9360-bin-win-cuda-13.1-x64.zip"
|
||||
assert pin.runtime_name == "cudart-llama-bin-win-cuda-13.1-x64.zip"
|
||||
assert pin.url.endswith("/b9360/llama-b9360-bin-win-cuda-13.1-x64.zip")
|
||||
assert pin.runtime_url.endswith("/b9360/cudart-llama-bin-win-cuda-13.1-x64.zip")
|
||||
assert pin.install_kind == "windows-cuda"
|
||||
assert pin.expected_sha256 and len(pin.expected_sha256) == 64
|
||||
assert pin.runtime_sha256 and len(pin.runtime_sha256) == 64
|
||||
|
||||
def test_pin_offered_for_driver_13_2(self):
|
||||
assert (
|
||||
_pinned_windows_cuda_fallback(self._win_host((13, 2), ["120"]), [])
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_pin_offered_for_sm121_variant(self):
|
||||
# sm_121 is Blackwell-family and also needs toolkit >= 12.8.
|
||||
assert (
|
||||
_pinned_windows_cuda_fallback(self._win_host((13, 1), ["121"]), [])
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_pin_uses_max_of_multi_gpu_caps(self):
|
||||
assert (
|
||||
_pinned_windows_cuda_fallback(self._win_host((13, 1), ["86", "120"]), [])
|
||||
is not None
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("sm", ["89", "90", "100"])
|
||||
def test_pin_not_offered_to_non_blackwell(self, sm):
|
||||
# Ada/Hopper run the cuda-12.4 build fine; the pin must not fire.
|
||||
assert _pinned_windows_cuda_fallback(self._win_host((13, 1), [sm]), []) is None
|
||||
|
||||
def test_pin_not_offered_to_driver_13_0(self):
|
||||
# 13.0 cannot run the 13.1 build (forward minor); residual CPU gap.
|
||||
assert (
|
||||
_pinned_windows_cuda_fallback(self._win_host((13, 0), ["120"]), []) is None
|
||||
)
|
||||
|
||||
def test_pin_not_offered_below_floor(self):
|
||||
assert (
|
||||
_pinned_windows_cuda_fallback(self._win_host((12, 8), ["120"]), []) is None
|
||||
)
|
||||
|
||||
def test_pin_not_offered_without_driver(self):
|
||||
assert _pinned_windows_cuda_fallback(self._win_host(None, ["120"]), []) is None
|
||||
|
||||
def test_pin_not_offered_on_linux(self):
|
||||
host = make_host(
|
||||
system = "Linux",
|
||||
machine = "x86_64",
|
||||
driver_cuda_version = (13, 1),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
assert _pinned_windows_cuda_fallback(host, []) is None
|
||||
|
||||
def test_pin_dormant_when_cuda13_attempt_present(self, monkeypatch):
|
||||
# A runnable in-release cuda13 build makes the pin unnecessary.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
host = self._win_host((13, 1), ["120"])
|
||||
assets = {
|
||||
f"llama-{self.TAG}-bin-win-cuda-13.1-x64.zip": "https://example.com/13.1",
|
||||
f"llama-{self.TAG}-bin-win-cuda-12.4-x64.zip": "https://example.com/12.4",
|
||||
}
|
||||
existing = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert any(a.runtime_line == "cuda13" for a in existing)
|
||||
assert _pinned_windows_cuda_fallback(host, existing) is None
|
||||
|
||||
def _win_cuda_attempt(self, minor):
|
||||
major = minor.split(".")[0]
|
||||
return AssetChoice(
|
||||
repo = UPSTREAM_REPO,
|
||||
tag = self.TAG,
|
||||
name = f"llama-{self.TAG}-bin-win-cuda-{minor}-x64.zip",
|
||||
url = "https://example.com/x",
|
||||
source_label = "upstream",
|
||||
install_kind = "windows-cuda",
|
||||
runtime_line = f"cuda{major}",
|
||||
)
|
||||
|
||||
def test_pin_dormant_when_runnable_cuda14_present(self, monkeypatch):
|
||||
# A future Blackwell host with an in-release cuda14 build (no cuda13)
|
||||
# must not get the older b9360 13.1 pin ahead of the runnable cuda14.
|
||||
mock_windows_runtime(monkeypatch, ["cuda14", "cuda12"])
|
||||
host = self._win_host((14, 0), ["120"])
|
||||
assets = {
|
||||
f"llama-{self.TAG}-bin-win-cuda-14.0-x64.zip": "https://example.com/14.0",
|
||||
f"llama-{self.TAG}-bin-win-cuda-12.4-x64.zip": "https://example.com/12.4",
|
||||
}
|
||||
existing = windows_cuda_attempts(host, self.TAG, assets, None)
|
||||
assert any(a.runtime_line == "cuda14" for a in existing)
|
||||
assert _pinned_windows_cuda_fallback(host, existing) is None
|
||||
|
||||
def test_pin_dormant_when_runnable_cuda12_8_present(self):
|
||||
# A cuda-12.8 build also covers Blackwell, so the pin defers to it.
|
||||
host = self._win_host((13, 1), ["120"])
|
||||
existing = [self._win_cuda_attempt("12.8")]
|
||||
assert _pinned_windows_cuda_fallback(host, existing) is None
|
||||
|
||||
def test_pin_fires_when_only_cuda12_4_present(self):
|
||||
# cuda-12.4 does not cover Blackwell, so the pin still fires.
|
||||
host = self._win_host((13, 1), ["120"])
|
||||
existing = [self._win_cuda_attempt("12.4")]
|
||||
assert _pinned_windows_cuda_fallback(host, existing) is not None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"minor, covers",
|
||||
[
|
||||
("12.4", False),
|
||||
("12.8", True),
|
||||
("13.1", True),
|
||||
("13.3", True),
|
||||
("14.0", True),
|
||||
],
|
||||
)
|
||||
def test_attempt_covers_blackwell(self, minor, covers):
|
||||
assert (
|
||||
_windows_cuda_attempt_covers_blackwell(self._win_cuda_attempt(minor))
|
||||
is covers
|
||||
)
|
||||
|
||||
def test_attempt_covers_blackwell_ignores_non_cuda_kind(self):
|
||||
cpu = AssetChoice(
|
||||
repo = UPSTREAM_REPO,
|
||||
tag = self.TAG,
|
||||
name = f"llama-{self.TAG}-bin-win-cpu-x64.zip",
|
||||
url = "https://example.com/x",
|
||||
source_label = "upstream",
|
||||
install_kind = "windows-cpu",
|
||||
)
|
||||
assert _windows_cuda_attempt_covers_blackwell(cpu) is False
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# N.1c. direct_upstream_release_plan -- pinned Blackwell fallback ordering
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestDirectUpstreamBlackwellPin:
|
||||
"""End to end: the pin lands ahead of cuda-12.4 on the simple/upstream path
|
||||
a Blackwell Windows host actually uses, and stays absent once a runnable
|
||||
in-release cuda13 build exists."""
|
||||
|
||||
TAG = "b9365"
|
||||
|
||||
def _release(self):
|
||||
names = [
|
||||
f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip",
|
||||
"cudart-llama-bin-win-cuda-13.3-x64.zip",
|
||||
f"llama-{self.TAG}-bin-win-cuda-12.4-x64.zip",
|
||||
"cudart-llama-bin-win-cuda-12.4-x64.zip",
|
||||
f"llama-{self.TAG}-bin-win-cpu-x64.zip",
|
||||
]
|
||||
return {
|
||||
"tag_name": self.TAG,
|
||||
"assets": [
|
||||
{"name": n, "browser_download_url": f"https://example.com/{n}"}
|
||||
for n in names
|
||||
],
|
||||
}
|
||||
|
||||
def _no_torch(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
INSTALL_LLAMA_PREBUILT,
|
||||
"detect_torch_cuda_runtime_preference",
|
||||
lambda host: CudaRuntimePreference(runtime_line = None, selection_log = []),
|
||||
)
|
||||
|
||||
def test_blackwell_13_1_prepends_pin(self, monkeypatch):
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
self._no_torch(monkeypatch)
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (13, 1),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
plan = direct_upstream_release_plan(
|
||||
self._release(), host, UPSTREAM_REPO, "latest"
|
||||
)
|
||||
order = [(a.tag, a.runtime_line or a.install_kind) for a in plan.attempts]
|
||||
assert order == [
|
||||
("b9360", "cuda13"),
|
||||
(self.TAG, "cuda12"),
|
||||
(self.TAG, "windows-cpu"),
|
||||
]
|
||||
assert plan.attempts[0].name == "llama-b9360-bin-win-cuda-13.1-x64.zip"
|
||||
# Direct/upstream path stays unverified-by-manifest (no approved hashes).
|
||||
assert plan.approved_checksums.artifacts == {}
|
||||
|
||||
def test_blackwell_13_3_no_pin(self, monkeypatch):
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
self._no_torch(monkeypatch)
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (13, 3),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
plan = direct_upstream_release_plan(
|
||||
self._release(), host, UPSTREAM_REPO, "latest"
|
||||
)
|
||||
assert "b9360" not in [a.tag for a in plan.attempts]
|
||||
assert plan.attempts[0].tag == self.TAG
|
||||
assert plan.attempts[0].runtime_line == "cuda13"
|
||||
assert plan.attempts[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# N.1d. published_windows_cuda_attempts -- version-dynamic ordering seed
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestPublishedWindowsCudaAttemptsDynamicMajor:
|
||||
"""The published-path ordering seed is derived from the release's real
|
||||
published minors, so a future CUDA major published here is selectable
|
||||
instead of being hidden by a hardcoded cuda12/cuda13 seed."""
|
||||
|
||||
TAG = "b8508"
|
||||
|
||||
def _win_cuda_artifact(self, minor, runtime_line):
|
||||
return make_artifact(
|
||||
f"llama-{self.TAG}-bin-win-cuda-{minor}-x64.zip",
|
||||
install_kind = "windows-cuda",
|
||||
runtime_line = runtime_line,
|
||||
max_sm = 120,
|
||||
)
|
||||
|
||||
def _release(self, minors_lines):
|
||||
artifacts = [self._win_cuda_artifact(m, line) for m, line in minors_lines]
|
||||
return make_release(artifacts, upstream_tag = self.TAG)
|
||||
|
||||
def test_future_cuda14_published_is_selected(self, monkeypatch):
|
||||
# With the dynamic seed a 14.x driver reaches a published cuda14 build;
|
||||
# the old hardcoded cuda12/cuda13 seed would never order it (the cuda14
|
||||
# line would be skipped for want of a 14.x asset in the seed).
|
||||
mock_windows_runtime(monkeypatch, ["cuda14", "cuda13", "cuda12"])
|
||||
release = self._release(
|
||||
[("14.0", "cuda14"), ("13.3", "cuda13"), ("12.4", "cuda12")]
|
||||
)
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (14, 0),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
result = published_windows_cuda_attempts(host, release, None)
|
||||
assert result[0].runtime_line == "cuda14"
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-14.0-x64.zip"
|
||||
|
||||
def test_cuda13_minor_selected_for_13_3_driver(self, monkeypatch):
|
||||
# Existing behavior unchanged: a 13.3 driver gets the real 13.3 build.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
release = self._release([("13.3", "cuda13"), ("12.4", "cuda12")])
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (13, 3),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
result = published_windows_cuda_attempts(host, release, None)
|
||||
assert result[0].runtime_line == "cuda13"
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip"
|
||||
|
||||
def test_below_minor_driver_gated_to_cuda12(self, monkeypatch):
|
||||
# A 13.1 driver is gated off a published 13.3 and falls to cuda12.
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
release = self._release([("13.3", "cuda13"), ("12.4", "cuda12")])
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (13, 1),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
result = published_windows_cuda_attempts(host, release, None)
|
||||
assert result[0].runtime_line == "cuda12"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# N.1e. resolve_release_asset_choice -- pin on the published install path
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestResolveReleaseAssetChoicePin:
|
||||
"""The published (non --simple-policy) install path reaches the same b9360
|
||||
Blackwell pin as the simple path, with its verified hash threaded."""
|
||||
|
||||
TAG = "b8508"
|
||||
|
||||
def _release(self, minors_lines):
|
||||
artifacts = [
|
||||
make_artifact(
|
||||
f"llama-{self.TAG}-bin-win-cuda-{minor}-x64.zip",
|
||||
install_kind = "windows-cuda",
|
||||
runtime_line = line,
|
||||
max_sm = 120,
|
||||
)
|
||||
for minor, line in minors_lines
|
||||
]
|
||||
assets = {}
|
||||
for minor, _line in minors_lines:
|
||||
assets[f"llama-{self.TAG}-bin-win-cuda-{minor}-x64.zip"] = (
|
||||
f"https://example.com/llama-{minor}"
|
||||
)
|
||||
assets[f"cudart-llama-bin-win-cuda-{minor}-x64.zip"] = (
|
||||
f"https://example.com/cudart-{minor}"
|
||||
)
|
||||
return make_release(artifacts, upstream_tag = self.TAG, assets = assets)
|
||||
|
||||
def _checksums(self, minors):
|
||||
names = []
|
||||
for minor in minors:
|
||||
names.append(f"llama-{self.TAG}-bin-win-cuda-{minor}-x64.zip")
|
||||
names.append(f"cudart-llama-bin-win-cuda-{minor}-x64.zip")
|
||||
return make_checksums(names)
|
||||
|
||||
def _no_torch(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
INSTALL_LLAMA_PREBUILT,
|
||||
"detect_torch_cuda_runtime_preference",
|
||||
lambda host: CudaRuntimePreference(runtime_line = None, selection_log = []),
|
||||
)
|
||||
|
||||
def test_pin_applied_on_published_path_for_13_1(self, monkeypatch):
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
self._no_torch(monkeypatch)
|
||||
release = self._release([("13.3", "cuda13"), ("12.4", "cuda12")])
|
||||
checksums = self._checksums(["12.4"]) # 13.3 gated off for a 13.1 driver
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (13, 1),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
result = resolve_release_asset_choice(host, self.TAG, release, checksums)
|
||||
assert result[0].tag == "b9360"
|
||||
assert result[0].name == "llama-b9360-bin-win-cuda-13.1-x64.zip"
|
||||
# apply_approved_hashes threaded the pin's verified hash from the
|
||||
# augmented checksums (the pin survives the approved-hash gate).
|
||||
assert result[0].expected_sha256 and len(result[0].expected_sha256) == 64
|
||||
assert result[0].runtime_sha256 and len(result[0].runtime_sha256) == 64
|
||||
assert any(a.runtime_line == "cuda12" for a in result)
|
||||
|
||||
def test_pin_dormant_on_published_path_for_13_3(self, monkeypatch):
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
self._no_torch(monkeypatch)
|
||||
release = self._release([("13.3", "cuda13"), ("12.4", "cuda12")])
|
||||
checksums = self._checksums(["13.3", "12.4"])
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (13, 3),
|
||||
compute_caps = ["120"],
|
||||
)
|
||||
result = resolve_release_asset_choice(host, self.TAG, release, checksums)
|
||||
assert "b9360" not in [a.tag for a in result]
|
||||
assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip"
|
||||
|
||||
def test_pin_not_applied_for_non_blackwell(self, monkeypatch):
|
||||
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
|
||||
self._no_torch(monkeypatch)
|
||||
release = self._release([("13.3", "cuda13"), ("12.4", "cuda12")])
|
||||
checksums = self._checksums(["12.4"])
|
||||
host = make_host(
|
||||
system = "Windows",
|
||||
machine = "AMD64",
|
||||
driver_cuda_version = (13, 1),
|
||||
compute_caps = ["89"],
|
||||
)
|
||||
result = resolve_release_asset_choice(host, self.TAG, release, checksums)
|
||||
assert "b9360" not in [a.tag for a in result]
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# N.1. apply_approved_hashes -- runtime archive checksum threading
|
||||
|
|
|
|||
|
|
@ -1216,8 +1216,6 @@ if is_openai_available():
|
|||
|
||||
# =============================================
|
||||
# Get Flash Attention v2 if Ampere (RTX 30xx, A100)
|
||||
import bitsandbytes as bnb
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
from transformers.utils.import_utils import _is_package_available
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue