+ Data
Overview
Columns
- Data
Raw
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index 5e8fe314b4..8d06c7d0e1 100755
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -65,9 +65,10 @@ def env_int(name: str, default: int, *, minimum: int | None = None) -> int:
# errors. Only use "master" temporarily when the latest release is missing
# support for a new model architecture.
DEFAULT_LLAMA_TAG = os.environ.get("UNSLOTH_LLAMA_TAG", "latest")
-# Force all installs to use mainline llama.cpp from ggml-org.
-# Previously: DEFAULT_PUBLISHED_REPO = os.environ.get("UNSLOTH_LLAMA_RELEASE_REPO", "unslothai/llama.cpp")
-DEFAULT_PUBLISHED_REPO = "ggml-org/llama.cpp"
+# Default published repo for prebuilt release resolution. Linux uses
+# Unsloth prebuilts; setup.sh/setup.ps1 pass --published-repo explicitly
+# for macOS/Windows to override with ggml-org/llama.cpp when needed.
+DEFAULT_PUBLISHED_REPO = "unslothai/llama.cpp"
DEFAULT_PUBLISHED_TAG = os.environ.get("UNSLOTH_LLAMA_RELEASE_TAG")
DEFAULT_PUBLISHED_MANIFEST_ASSET = os.environ.get(
"UNSLOTH_LLAMA_RELEASE_MANIFEST_ASSET", "llama-prebuilt-manifest.json"
@@ -89,6 +90,12 @@ GITHUB_AUTH_HOSTS = {"api.github.com", "github.com"}
RETRYABLE_HTTP_STATUS = {408, 429, 500, 502, 503, 504}
HTTP_FETCH_ATTEMPTS = 4
HTTP_FETCH_BASE_DELAY_SECONDS = 0.75
+JSON_FETCH_ATTEMPTS = 3
+DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES = env_int(
+ "UNSLOTH_LLAMA_GITHUB_RELEASE_SCAN_MAX_PAGES",
+ 5,
+ minimum = 1,
+)
SERVER_PORT_BIND_ATTEMPTS = 3
SERVER_BIND_RETRY_WINDOW_SECONDS = 5.0
TTY_PROGRESS_START_DELAY_SECONDS = 0.5
@@ -97,6 +104,58 @@ DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS = env_int(
2,
minimum = 1,
)
+FORCE_COMPILE_DEFAULT_REF = os.environ.get("UNSLOTH_LLAMA_FORCE_COMPILE_REF", "master")
+
+DIRECT_LINUX_BUNDLE_PROFILES: dict[str, dict[str, Any]] = {
+ "cuda12-older": {
+ "runtime_line": "cuda12",
+ "coverage_class": "older",
+ "supported_sms": ["70", "75", "80", "86", "89"],
+ "min_sm": 70,
+ "max_sm": 89,
+ "rank": 10,
+ },
+ "cuda12-newer": {
+ "runtime_line": "cuda12",
+ "coverage_class": "newer",
+ "supported_sms": ["86", "89", "90", "100", "120"],
+ "min_sm": 86,
+ "max_sm": 120,
+ "rank": 20,
+ },
+ "cuda12-portable": {
+ "runtime_line": "cuda12",
+ "coverage_class": "portable",
+ "supported_sms": ["70", "75", "80", "86", "89", "90", "100", "120"],
+ "min_sm": 70,
+ "max_sm": 120,
+ "rank": 30,
+ },
+ "cuda13-older": {
+ "runtime_line": "cuda13",
+ "coverage_class": "older",
+ "supported_sms": ["75", "80", "86", "89"],
+ "min_sm": 75,
+ "max_sm": 89,
+ "rank": 40,
+ },
+ "cuda13-newer": {
+ "runtime_line": "cuda13",
+ "coverage_class": "newer",
+ "supported_sms": ["86", "89", "90", "100", "120"],
+ "min_sm": 86,
+ "max_sm": 120,
+ "rank": 50,
+ },
+ "cuda13-portable": {
+ "runtime_line": "cuda13",
+ "coverage_class": "portable",
+ "supported_sms": ["75", "80", "86", "89", "90", "100", "120"],
+ "min_sm": 75,
+ "max_sm": 120,
+ "rank": 60,
+ },
+}
@dataclass
@@ -753,32 +812,48 @@ def download_bytes(
def fetch_json(url: str) -> Any:
- try:
- data = download_bytes(
- url,
- timeout = 30,
- headers = github_api_headers(url)
- if is_github_api_url(url)
- else auth_headers(url),
- )
- except urllib.error.HTTPError as exc:
- if exc.code == 403 and is_github_api_url(url):
- hint = ""
- if not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")):
- hint = "; set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits"
- raise RuntimeError(f"GitHub API returned 403 for {url}{hint}") from exc
- raise
- if not data:
- raise RuntimeError(f"downloaded empty JSON payload from {url}")
- try:
- payload = json.loads(data.decode("utf-8"))
- except (UnicodeDecodeError, json.JSONDecodeError) as exc:
- raise RuntimeError(f"downloaded invalid JSON from {url}: {exc}") from exc
- if not isinstance(payload, dict) and not isinstance(payload, list):
- raise RuntimeError(
- f"downloaded unexpected JSON type from {url}: {type(payload).__name__}"
- )
- return payload
+ attempts = JSON_FETCH_ATTEMPTS if is_github_api_url(url) else 1
+ last_decode_exc: Exception | None = None
+ for attempt in range(1, attempts + 1):
+ try:
+ data = download_bytes(
+ url,
+ timeout = 30,
+ headers = github_api_headers(url)
+ if is_github_api_url(url)
+ else auth_headers(url),
+ )
+ except urllib.error.HTTPError as exc:
+ if exc.code == 403 and is_github_api_url(url):
+ hint = ""
+ if not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")):
+ hint = (
+ "; set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits"
+ )
+ raise RuntimeError(f"GitHub API returned 403 for {url}{hint}") from exc
+ raise
+ if not data:
+ last_decode_exc = RuntimeError(f"downloaded empty JSON payload from {url}")
+ else:
+ try:
+ payload = json.loads(data.decode("utf-8"))
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ last_decode_exc = RuntimeError(
+ f"downloaded invalid JSON from {url}: {exc}"
+ )
+ else:
+ if not isinstance(payload, dict) and not isinstance(payload, list):
+ raise RuntimeError(
+ f"downloaded unexpected JSON type from {url}: {type(payload).__name__}"
+ )
+ return payload
+ if attempt >= attempts:
+ assert last_decode_exc is not None
+ raise last_decode_exc
+ log(f"json fetch failed ({attempt}/{attempts}) for {url}; retrying")
+ sleep_backoff(attempt)
+ assert last_decode_exc is not None
+ raise last_decode_exc
def download_file(url: str, destination: Path) -> None:
@@ -838,12 +913,16 @@ def download_file_verified(
url: str,
destination: Path,
*,
- expected_sha256: str,
+ expected_sha256: str | None,
label: str,
) -> None:
normalized_expected = normalize_sha256_digest(expected_sha256)
if not normalized_expected:
- raise PrebuiltFallback(f"{label} did not have a valid approved sha256")
+ download_file(url, destination)
+ log(
+ f"downloaded {label} without a published sha256; relying on install validation"
+ )
+ return
for attempt in range(1, 3):
download_file(url, destination)
@@ -898,7 +977,12 @@ def github_release(repo: str, tag: str) -> dict[str, Any]:
return payload
-def github_releases(repo: str, *, per_page: int = 100) -> list[dict[str, Any]]:
+def github_releases(
+ repo: str,
+ *,
+ per_page: int = 100,
+ max_pages: int = 0,
+) -> list[dict[str, Any]]:
releases: list[dict[str, Any]] = []
page = 1
while True:
@@ -912,6 +996,8 @@ def github_releases(repo: str, *, per_page: int = 100) -> list[dict[str, Any]]:
if len(payload) < per_page:
break
page += 1
+ if max_pages > 0 and page > max_pages:
+ break
return releases
@@ -925,6 +1011,372 @@ def latest_upstream_release_tag() -> str:
return tag
+def is_release_tag_like(value: str | None) -> bool:
+ return isinstance(value, str) and bool(re.fullmatch(r"b\d+", value.strip()))
+
+
+def release_time_sort_key(release: dict[str, Any]) -> tuple[str, int]:
+ published_at = release.get("published_at")
+ created_at = release.get("created_at")
+ release_id = release.get("id")
+ timestamp = (
+ published_at
+ if isinstance(published_at, str) and published_at
+ else created_at
+ if isinstance(created_at, str) and created_at
+ else ""
+ )
+ try:
+ normalized_id = int(release_id)
+ except (TypeError, ValueError):
+ normalized_id = 0
+ return (timestamp, normalized_id)
+
+
+def iter_release_payloads_by_time(
+ repo: str,
+ published_release_tag: str = "",
+ requested_tag: str = "",
+) -> Iterable[dict[str, Any]]:
+ if published_release_tag:
+ yield github_release(repo, published_release_tag)
+ return
+
+ if (
+ requested_tag
+ and requested_tag != "latest"
+ and is_release_tag_like(requested_tag)
+ ):
+ try:
+ yield github_release(repo, requested_tag)
+ return
+ except urllib.error.HTTPError as exc:
+ if exc.code == 404:
+ log(
+ f"release tag {requested_tag} not found in {repo}; scanning recent releases"
+ )
+ else:
+ raise
+ except Exception:
+ raise
+
+ releases = [
+ release
+ for release in github_releases(
+ repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES
+ )
+ if isinstance(release, dict)
+ and not release.get("draft")
+ and not release.get("prerelease")
+ ]
+ releases.sort(key = release_time_sort_key, reverse = True)
+ for release in releases:
+ yield release
+
+
+def direct_release_matches_request(
+ *, release_tag: str, llama_tag: str, requested_tag: str
+) -> bool:
+ if requested_tag == "latest":
+ return True
+ for candidate in (release_tag, llama_tag):
+ if refs_match(candidate, requested_tag):
+ return True
+ return False
+
+
+def synthetic_checksums_for_release(
+ repo: str, release_tag: str, upstream_tag: str
+) -> ApprovedReleaseChecksums:
+ return ApprovedReleaseChecksums(
+ repo = repo,
+ release_tag = release_tag,
+ upstream_tag = upstream_tag,
+ artifacts = {},
+ )
+
+
+def parse_direct_linux_release_bundle(
+ repo: str, release: dict[str, Any]
+) -> PublishedReleaseBundle | None:
+ release_tag = release.get("tag_name")
+ if not isinstance(release_tag, str) or not release_tag:
+ return None
+
+ assets = release_asset_map(release)
+ artifacts: list[PublishedLlamaArtifact] = []
+ inferred_labels: list[str] = []
+
+ linux_asset_re = re.compile(
+ r"^app-(?P.+)-(?Plinux-x64(?:-cpu)?|linux-x64-(?:cuda12|cuda13)-(?:older|newer|portable))\.tar\.gz$"
+ )
+ for asset_name in sorted(assets):
+ match = linux_asset_re.fullmatch(asset_name)
+ if not match:
+ continue
+ inferred_labels.append(match.group("label"))
+ target = match.group("target")
+ if target in {"linux-x64", "linux-x64-cpu"}:
+ artifacts.append(
+ PublishedLlamaArtifact(
+ asset_name = asset_name,
+ install_kind = "linux-cpu",
+ runtime_line = None,
+ coverage_class = None,
+ supported_sms = [],
+ min_sm = None,
+ max_sm = None,
+ bundle_profile = None,
+ rank = 1000,
+ )
+ )
+ continue
+
+ bundle_profile = target.removeprefix("linux-x64-")
+ profile = DIRECT_LINUX_BUNDLE_PROFILES.get(bundle_profile)
+ if profile is None:
+ continue
+ artifacts.append(
+ PublishedLlamaArtifact(
+ asset_name = asset_name,
+ install_kind = "linux-cuda",
+ runtime_line = str(profile["runtime_line"]),
+ coverage_class = str(profile["coverage_class"]),
+ supported_sms = [str(value) for value in profile["supported_sms"]],
+ min_sm = int(profile["min_sm"]),
+ max_sm = int(profile["max_sm"]),
+ bundle_profile = bundle_profile,
+ rank = int(profile["rank"]),
+ )
+ )
+
+ if not artifacts:
+ return None
+
+ upstream_tag = (
+ release_tag
+ if is_release_tag_like(release_tag)
+ else inferred_labels[0]
+ if len(set(inferred_labels)) == 1 and inferred_labels
+ else release_tag
+ )
+ selection_log = [
+ f"published_release: repo={repo}",
+ f"published_release: tag={release_tag}",
+ f"published_release: upstream_tag={upstream_tag}",
+ "published_release: direct_asset_scan=linux",
+ ]
+ return PublishedReleaseBundle(
+ repo = repo,
+ release_tag = release_tag,
+ upstream_tag = upstream_tag,
+ assets = assets,
+ manifest_asset_name = DEFAULT_PUBLISHED_MANIFEST_ASSET,
+ artifacts = artifacts,
+ selection_log = selection_log,
+ )
+
+
+def direct_linux_release_plan(
+ release: dict[str, Any],
+ host: HostInfo,
+ repo: str,
+ requested_tag: str,
+) -> InstallReleasePlan | None:
+ bundle = parse_direct_linux_release_bundle(repo, release)
+ if bundle is None:
+ return None
+ if not direct_release_matches_request(
+ release_tag = bundle.release_tag,
+ llama_tag = bundle.upstream_tag,
+ requested_tag = requested_tag,
+ ):
+ return None
+
+ attempts: list[AssetChoice] = []
+ if host.has_usable_nvidia:
+ selection = linux_cuda_choice_from_release(host, bundle)
+ if selection is not None:
+ attempts.extend(selection.attempts)
+ cpu_choice = published_asset_choice_for_kind(bundle, "linux-cpu")
+ if cpu_choice is not None:
+ attempts.append(cpu_choice)
+ if not attempts:
+ raise PrebuiltFallback("no compatible Linux prebuilt asset was found")
+ return InstallReleasePlan(
+ requested_tag = requested_tag,
+ llama_tag = bundle.upstream_tag,
+ release_tag = bundle.release_tag,
+ attempts = attempts,
+ approved_checksums = synthetic_checksums_for_release(
+ repo,
+ bundle.release_tag,
+ bundle.upstream_tag,
+ ),
+ )
+
+
+def direct_upstream_release_plan(
+ release: dict[str, Any],
+ host: HostInfo,
+ repo: str,
+ requested_tag: str,
+) -> InstallReleasePlan | None:
+ release_tag = release.get("tag_name")
+ if not isinstance(release_tag, str) or not release_tag:
+ return None
+ if not direct_release_matches_request(
+ release_tag = release_tag,
+ llama_tag = release_tag,
+ requested_tag = requested_tag,
+ ):
+ return None
+
+ assets = release_asset_map(release)
+ attempts: list[AssetChoice] = []
+ if host.is_windows and host.is_x86_64:
+ if host.has_usable_nvidia:
+ torch_preference = detect_torch_cuda_runtime_preference(host)
+ attempts.extend(
+ windows_cuda_attempts(
+ host,
+ release_tag,
+ assets,
+ torch_preference.runtime_line,
+ torch_preference.selection_log,
+ )
+ )
+ cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip"
+ cpu_url = assets.get(cpu_asset)
+ if cpu_url:
+ attempts.append(
+ AssetChoice(
+ repo = repo,
+ tag = release_tag,
+ name = cpu_asset,
+ url = cpu_url,
+ source_label = "upstream",
+ install_kind = "windows-cpu",
+ )
+ )
+ elif host.is_macos and host.is_arm64:
+ asset_name = f"llama-{release_tag}-bin-macos-arm64.tar.gz"
+ asset_url = assets.get(asset_name)
+ if asset_url:
+ attempts.append(
+ AssetChoice(
+ repo = repo,
+ tag = release_tag,
+ name = asset_name,
+ url = asset_url,
+ source_label = "upstream",
+ install_kind = "macos-arm64",
+ )
+ )
+ elif host.is_macos and host.is_x86_64:
+ asset_name = f"llama-{release_tag}-bin-macos-x64.tar.gz"
+ asset_url = assets.get(asset_name)
+ if asset_url:
+ attempts.append(
+ AssetChoice(
+ repo = repo,
+ tag = release_tag,
+ name = asset_name,
+ url = asset_url,
+ source_label = "upstream",
+ install_kind = "macos-x64",
+ )
+ )
+ elif host.is_linux and host.is_x86_64 and not host.has_usable_nvidia:
+ asset_name = f"llama-{release_tag}-bin-ubuntu-x64.tar.gz"
+ asset_url = assets.get(asset_name)
+ if asset_url:
+ attempts.append(
+ AssetChoice(
+ repo = repo,
+ tag = release_tag,
+ name = asset_name,
+ url = asset_url,
+ source_label = "upstream",
+ install_kind = "linux-cpu",
+ )
+ )
+ if not attempts:
+ raise PrebuiltFallback("no compatible upstream prebuilt asset was found")
+ return InstallReleasePlan(
+ requested_tag = requested_tag,
+ llama_tag = release_tag,
+ release_tag = release_tag,
+ attempts = attempts,
+ approved_checksums = synthetic_checksums_for_release(
+ repo,
+ release_tag,
+ release_tag,
+ ),
+ )
+
+
+def resolve_simple_install_release_plans(
+ llama_tag: str,
+ host: HostInfo,
+ published_repo: str,
+ published_release_tag: str,
+ *,
+ max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,
+) -> tuple[str, list[InstallReleasePlan]]:
+ repo = published_repo or DEFAULT_PUBLISHED_REPO
+ requested_tag = normalized_requested_llama_tag(llama_tag)
+ allow_older_release_fallback = (
+ requested_tag == "latest" and not published_release_tag
+ )
+ release_limit = max(1, max_release_fallbacks)
+ plans: list[InstallReleasePlan] = []
+ last_error: PrebuiltFallback | None = None
+
+ try:
+ releases = iter_release_payloads_by_time(
+ repo, published_release_tag, requested_tag
+ )
+ for release in releases:
+ try:
+ if host.is_linux and repo == "unslothai/llama.cpp":
+ plan = direct_linux_release_plan(release, host, repo, requested_tag)
+ else:
+ plan = direct_upstream_release_plan(
+ release, host, repo, requested_tag
+ )
+ if plan is None:
+ continue
+ except PrebuiltFallback as exc:
+ last_error = exc
+ if not allow_older_release_fallback:
+ raise
+ release_tag = release.get("tag_name") or "unknown"
+ log(
+ "published release skipped for install planning: "
+ f"{repo}@{release_tag} ({exc})"
+ )
+ continue
+
+ plans.append(plan)
+ if not allow_older_release_fallback or len(plans) >= release_limit:
+ break
+ except PrebuiltFallback:
+ raise
+ except Exception as exc:
+ raise PrebuiltFallback(
+ f"failed to inspect published releases in {repo}: {exc}"
+ ) from exc
+
+ if plans:
+ return requested_tag, plans
+ if last_error is not None:
+ raise last_error
+ raise PrebuiltFallback(
+ f"no installable published llama.cpp releases were found in {repo}"
+ )
+
+
def normalized_requested_llama_tag(requested_tag: str | None) -> str:
if isinstance(requested_tag, str):
normalized = requested_tag.strip()
@@ -1435,7 +1887,7 @@ def iter_published_release_bundles(
releases = (
[github_release(repo, published_release_tag)]
if published_release_tag
- else github_releases(repo)
+ else github_releases(repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES)
)
for release in releases:
if not published_release_tag and (
@@ -1669,7 +2121,9 @@ def latest_published_linux_cuda_tag(host: HostInfo, published_repo: str) -> str
def iter_upstream_releases() -> Iterable[dict[str, Any]]:
- for release in github_releases(UPSTREAM_REPO):
+ for release in github_releases(
+ UPSTREAM_REPO, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES
+ ):
if release.get("draft") or release.get("prerelease"):
continue
yield release
@@ -2799,7 +3253,7 @@ def hydrate_source_tree(
work_dir: Path,
*,
source_repo: str = UPSTREAM_REPO,
- expected_sha256: str,
+ expected_sha256: str | None,
source_label: str | None = None,
exact_source: bool = False,
) -> None:
@@ -3231,10 +3685,6 @@ def install_from_archives(
) -> tuple[Path, Path]:
main_archive = work_dir / choice.name
log(f"downloading {choice.name} from {choice.source_label} release")
- if not choice.expected_sha256:
- raise PrebuiltFallback(
- f"approved checksum was missing for selected asset {choice.name}"
- )
download_file_verified(
choice.url,
main_archive,
@@ -3962,7 +4412,7 @@ def require_approved_source_hash(
def preferred_source_archive(
checksums: ApprovedReleaseChecksums, llama_tag: str
-) -> tuple[str, str, ApprovedArtifactHash, bool]:
+) -> tuple[str, str, ApprovedArtifactHash | None, bool]:
exact_source = exact_source_archive_hash(checksums)
exact_repo = repo_slug_from_source(checksums.source_repo) or repo_slug_from_source(
checksums.source_repo_url
@@ -3974,7 +4424,7 @@ def preferred_source_archive(
exact_source,
True,
)
- legacy = require_approved_source_hash(checksums, llama_tag)
+ legacy = checksums.artifacts.get(source_archive_logical_name(llama_tag))
return (
UPSTREAM_REPO,
llama_tag,
@@ -3990,6 +4440,8 @@ def selected_source_archive_metadata(
_source_repo, _source_ref, source_archive, _exact_source = preferred_source_archive(
checksums, llama_tag
)
+ if source_archive is None:
+ return source_archive_logical_name(llama_tag), None
return source_archive.asset_name, source_archive.sha256
@@ -4153,8 +4605,6 @@ def expected_install_fingerprint(
choice: AssetChoice,
approved_checksums: ApprovedReleaseChecksums,
) -> str | None:
- if not choice.expected_sha256:
- return None
source_asset_name, source_sha256 = selected_source_archive_metadata(
approved_checksums,
llama_tag,
@@ -4352,7 +4802,7 @@ def validate_prebuilt_choice(
install_dir,
work_dir,
source_repo = source_repo,
- expected_sha256 = source_archive.sha256,
+ expected_sha256 = source_archive.sha256 if source_archive is not None else None,
source_label = (
f"llama.cpp source tree for {source_repo}@{source_ref}"
if exact_source
@@ -4477,7 +4927,12 @@ def validate_prebuilt_attempts(
def install_prebuilt(
- install_dir: Path, llama_tag: str, published_repo: str, published_release_tag: str
+ install_dir: Path,
+ llama_tag: str,
+ published_repo: str,
+ published_release_tag: str,
+ *,
+ simple_policy: bool = False,
) -> None:
host = detect_host()
choice: AssetChoice | None = None
@@ -4491,12 +4946,20 @@ def install_prebuilt(
log(
f"no existing llama.cpp install detected at {install_dir}; performing fresh prebuilt install"
)
- requested_tag, release_plans = resolve_install_release_plans(
- llama_tag,
- host,
- published_repo,
- published_release_tag,
- )
+ if simple_policy:
+ requested_tag, release_plans = resolve_simple_install_release_plans(
+ llama_tag,
+ host,
+ published_repo,
+ published_release_tag,
+ )
+ else:
+ requested_tag, release_plans = resolve_install_release_plans(
+ llama_tag,
+ host,
+ published_repo,
+ published_release_tag,
+ )
if release_plans and existing_install_matches_plan(
install_dir, host, release_plans[0]
):
@@ -4600,6 +5063,11 @@ def parse_args() -> argparse.Namespace:
"until a usable published llama.cpp release bundle is found."
),
)
+ parser.add_argument(
+ "--simple-policy",
+ action = "store_true",
+ help = "Use the simplified platform-specific prebuilt selection policy.",
+ )
resolve_group = parser.add_mutually_exclusive_group()
resolve_group.add_argument(
"--resolve-llama-tag",
@@ -4719,6 +5187,7 @@ def main() -> int:
llama_tag = args.llama_tag,
published_repo = args.published_repo,
published_release_tag = args.published_release_tag or "",
+ simple_policy = args.simple_policy,
)
return EXIT_SUCCESS
diff --git a/studio/setup.ps1 b/studio/setup.ps1
index d478319098..60dfce0661 100644
--- a/studio/setup.ps1
+++ b/studio/setup.ps1
@@ -34,6 +34,7 @@ $PackageDir = Split-Path -Parent $ScriptDir
$DefaultLlamaPrForce = ""
$DefaultLlamaSource = "https://github.com/ggml-org/llama.cpp"
$DefaultLlamaTag = "latest"
+$DefaultLlamaForceCompileRef = "master"
# Verbose can be enabled either by CLI flag or by UNSLOTH_VERBOSE=1.
$script:UnslothVerbose = ($env:UNSLOTH_VERBOSE -eq '1')
@@ -81,6 +82,31 @@ function New-UnslothTemporaryFile {
return Get-Item -LiteralPath $tempPath
}
+function Get-InstalledLlamaPrebuiltRelease {
+ param([string]$InstallDir)
+
+ $metadataPath = Join-Path $InstallDir "UNSLOTH_PREBUILT_INFO.json"
+ if (-not (Test-Path $metadataPath)) {
+ return $null
+ }
+
+ try {
+ $payload = Get-Content $metadataPath -Raw | ConvertFrom-Json
+ } catch {
+ return $null
+ }
+
+ if (-not $payload.published_repo -or -not $payload.release_tag) {
+ return $null
+ }
+
+ $message = "installed release: $($payload.published_repo)@$($payload.release_tag)"
+ if ($payload.tag -and $payload.tag -ne $payload.release_tag) {
+ $message += " (tag $($payload.tag))"
+ }
+ return $message
+}
+
# Find nvcc on PATH, CUDA_PATH, or standard toolkit dirs.
# Returns the path to nvcc.exe, or $null if not found.
function Find-Nvcc {
@@ -133,7 +159,9 @@ function Find-Nvcc {
# 3. Scan standard toolkit directory
if (Test-Path $toolkitBase) {
- $latest = Get-ChildItem -Directory $toolkitBase | Sort-Object Name | Select-Object -Last 1
+ $latest = Get-ChildItem -Directory $toolkitBase | Where-Object {
+ $_.Name -match '^v(\d+)\.(\d+)'
+ } | Sort-Object { [version]($_.Name -replace '^v','') } -Descending | Select-Object -First 1
if ($latest -and (Test-Path (Join-Path $latest.FullName 'bin\nvcc.exe'))) {
return (Join-Path $latest.FullName 'bin\nvcc.exe')
}
@@ -1618,6 +1646,7 @@ if ($LlamaSource.EndsWith('.git')) { $LlamaSource = $LlamaSource.Substring(0, $L
$ResolvedSourceUrl = $LlamaSource
$ResolvedSourceRef = $RequestedLlamaTag
$ResolvedSourceRefKind = "tag"
+$ResolvedLlamaTag = $RequestedLlamaTag
if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") {
$NeedLlamaSourceBuild = $true
@@ -1693,92 +1722,27 @@ if ($LlamaPr) {
$ResolvedSourceRefKind = "pull"
$NeedLlamaSourceBuild = $true
$SkipPrebuiltInstall = $true
-} elseif ($SkipPrebuiltInstall) {
- # Custom source or other override already forced source build; skip the
- # prebuilt release resolution. When building from a custom fork, the fork
- # may not carry upstream bNNNN tags.
- if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") {
- $ResolvedLlamaTag = $RequestedLlamaTag
- } elseif ($LlamaSource -eq "https://github.com/ggml-org/llama.cpp") {
- $resolveTagArgs = @("--resolve-llama-tag", $RequestedLlamaTag, "--published-repo", $HelperReleaseRepo, "--output-format", "json")
- if ($env:UNSLOTH_LLAMA_RELEASE_TAG) { $resolveTagArgs += @("--published-release-tag", $env:UNSLOTH_LLAMA_RELEASE_TAG) }
- $fallbackResult = Invoke-LlamaHelper -Arguments $resolveTagArgs
- $fallbackOutput = $fallbackResult.Output
- $fallbackExit = $fallbackResult.ExitCode
- $ResolvedLlamaTag = if ($fallbackExit -eq 0 -and $fallbackOutput) {
- try {
- (($fallbackOutput | Out-String) | ConvertFrom-Json).llama_tag
- } catch {
- $RequestedLlamaTag
- }
- } else {
- $RequestedLlamaTag
- }
- } else {
- $ResolvedLlamaTag = $RequestedLlamaTag
- }
-} else {
- $resolveInstallArgs = @("--resolve-install-tag", $RequestedLlamaTag, "--published-repo", $HelperReleaseRepo, "--output-format", "json")
- if ($env:UNSLOTH_LLAMA_RELEASE_TAG) { $resolveInstallArgs += @("--published-release-tag", $env:UNSLOTH_LLAMA_RELEASE_TAG) }
- $resolveErrorLog = New-UnslothTemporaryFile
- $resolveResult = Invoke-LlamaHelper -Arguments $resolveInstallArgs -StderrPath $resolveErrorLog
- $resolveOutput = $resolveResult.Output
- $resolveExit = $resolveResult.ExitCode
- $ResolvedLlamaTag = if ($resolveOutput) {
- try {
- (($resolveOutput | Out-String) | ConvertFrom-Json).llama_tag
- } catch {
- ""
- }
- } else { "" }
- if ($resolveExit -ne 0 -or [string]::IsNullOrWhiteSpace($ResolvedLlamaTag)) {
- Write-Host ""
- substep "Failed to resolve a published llama.cpp release via $HelperReleaseRepo" "Yellow"
- Write-LlamaFailureLog -Output (Get-Content -Raw $resolveErrorLog)
- # Resolve the llama.cpp tag for source-build fallback. Pass --published-repo
- # so the resolver prefers the latest usable Unsloth-published upstream tag
- # before falling back to the bleeding-edge ggml-org/llama.cpp tag.
- $resolveFallbackArgs = @("--resolve-llama-tag", $RequestedLlamaTag, "--published-repo", $HelperReleaseRepo, "--output-format", "json")
- if ($env:UNSLOTH_LLAMA_RELEASE_TAG) { $resolveFallbackArgs += @("--published-release-tag", $env:UNSLOTH_LLAMA_RELEASE_TAG) }
- $fallbackResult = Invoke-LlamaHelper -Arguments $resolveFallbackArgs
- $fallbackOutput = $fallbackResult.Output
- $fallbackExit = $fallbackResult.ExitCode
- $ResolvedLlamaTag = if ($fallbackExit -eq 0 -and $fallbackOutput) {
- try {
- (($fallbackOutput | Out-String) | ConvertFrom-Json).llama_tag
- } catch {
- $RequestedLlamaTag
- }
- } else {
- $RequestedLlamaTag
- }
- $NeedLlamaSourceBuild = $true
- $SkipPrebuiltInstall = $true
- }
- Remove-Item $resolveErrorLog -Force -ErrorAction SilentlyContinue
}
-Write-Host ""
-substep "Resolved llama.cpp release tag: $ResolvedLlamaTag"
-
if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") {
Write-Host ""
substep "UNSLOTH_LLAMA_FORCE_COMPILE=1 -- skipping prebuilt llama.cpp install" "Yellow"
$NeedLlamaSourceBuild = $true
+} elseif ($SkipPrebuiltInstall) {
+ Write-Host ""
+ substep "Skipping prebuilt install -- falling back to source build" "Yellow"
} else {
Write-Host ""
substep "installing prebuilt llama.cpp bundle (preferred path)..."
if (Test-Path $LlamaCppDir) {
substep "Existing llama.cpp install detected -- validating staged prebuilt update before replacement"
}
- if ($SkipPrebuiltInstall) {
- substep "Skipping prebuilt install because prebuilt tag resolution failed -- falling back to source build" "Yellow"
- } else {
- $prebuiltArgs = @(
+ $prebuiltArgs = @(
"$PSScriptRoot\install_llama_prebuilt.py",
"--install-dir", $LlamaCppDir,
"--llama-tag", $RequestedLlamaTag,
- "--published-repo", $HelperReleaseRepo
+ "--published-repo", $HelperReleaseRepo,
+ "--simple-policy"
)
if ($env:UNSLOTH_LLAMA_RELEASE_TAG) {
$prebuiltArgs += @("--published-release-tag", $env:UNSLOTH_LLAMA_RELEASE_TAG)
@@ -1817,6 +1781,10 @@ if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") {
} else {
step "llama.cpp" "prebuilt installed and validated"
}
+ $installedRelease = Get-InstalledLlamaPrebuiltRelease -InstallDir $LlamaCppDir
+ if ($installedRelease) {
+ substep $installedRelease
+ }
} elseif ($prebuiltExit -eq 3) {
step "llama.cpp" "install blocked by active llama.cpp process" "Yellow"
Write-LlamaFailureLog -Output $prebuiltOutput
@@ -1834,7 +1802,6 @@ if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") {
substep "Prebuilt llama.cpp path unavailable or failed validation -- falling back to source build" "Yellow"
$NeedLlamaSourceBuild = $true
}
- }
}
# ==========================================================================
@@ -1981,24 +1948,43 @@ if (-not $NeedLlamaSourceBuild) {
}
if (-not $LlamaPr) {
- if ($LlamaSource -eq "https://github.com/ggml-org/llama.cpp") {
- $resolveSourceArgs = @("--resolve-source-build", $RequestedLlamaTag, "--published-repo", $HelperReleaseRepo, "--output-format", "json")
- if ($env:UNSLOTH_LLAMA_RELEASE_TAG) { $resolveSourceArgs += @("--published-release-tag", $env:UNSLOTH_LLAMA_RELEASE_TAG) }
- $sourcePlanResult = Invoke-LlamaHelper -Arguments $resolveSourceArgs
- $sourcePlanOutput = $sourcePlanResult.Output
- $sourcePlanExit = $sourcePlanResult.ExitCode
- if ($sourcePlanExit -eq 0 -and $sourcePlanOutput) {
- try {
- $sourcePlan = ($sourcePlanOutput | Out-String) | ConvertFrom-Json
- $ResolvedSourceUrl = $sourcePlan.source_url
- $ResolvedSourceRefKind = $sourcePlan.source_ref_kind
- $ResolvedSourceRef = $sourcePlan.source_ref
- } catch {
+ $ResolvedSourceUrl = $LlamaSource
+ if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") {
+ if ($RequestedLlamaTag -eq "latest") {
+ $ResolvedSourceRef = if ($env:UNSLOTH_LLAMA_FORCE_COMPILE_REF) {
+ $env:UNSLOTH_LLAMA_FORCE_COMPILE_REF
+ } else {
+ $DefaultLlamaForceCompileRef
}
+ $ResolvedSourceRefKind = "branch"
+ } else {
+ $ResolvedSourceRef = $RequestedLlamaTag
+ $ResolvedSourceRefKind = "tag"
}
+ } elseif ($RequestedLlamaTag -eq "latest") {
+ $resolveTagArgs = @("--resolve-llama-tag", "latest", "--published-repo", "ggml-org/llama.cpp", "--output-format", "json")
+ $resolveTagResult = Invoke-LlamaHelper -Arguments $resolveTagArgs
+ $resolveTagOutput = $resolveTagResult.Output
+ $resolveTagExit = $resolveTagResult.ExitCode
+ if ($resolveTagExit -eq 0 -and $resolveTagOutput) {
+ try {
+ $ResolvedSourceRef = (($resolveTagOutput | Out-String) | ConvertFrom-Json).llama_tag
+ } catch {
+ $ResolvedSourceRef = ""
+ }
+ } else {
+ $ResolvedSourceRef = ""
+ }
+ if ([string]::IsNullOrWhiteSpace($ResolvedSourceRef)) {
+ $ResolvedSourceRef = "latest"
+ }
+ $ResolvedSourceRefKind = "tag"
+ } else {
+ $ResolvedSourceRef = $RequestedLlamaTag
+ $ResolvedSourceRefKind = "tag"
}
if ([string]::IsNullOrWhiteSpace($ResolvedSourceUrl)) { $ResolvedSourceUrl = $LlamaSource }
- if ([string]::IsNullOrWhiteSpace($ResolvedSourceRef)) { $ResolvedSourceRef = $ResolvedLlamaTag }
+ if ([string]::IsNullOrWhiteSpace($ResolvedSourceRef)) { $ResolvedSourceRef = $RequestedLlamaTag }
}
# -- Step A: Clone or pull llama.cpp --
diff --git a/studio/setup.sh b/studio/setup.sh
index d9ce73661f..e3ff2da35c 100755
--- a/studio/setup.sh
+++ b/studio/setup.sh
@@ -25,6 +25,7 @@ RULE=$(printf '\342\224\200%.0s' {1..52})
_DEFAULT_LLAMA_PR_FORCE=""
_DEFAULT_LLAMA_SOURCE="https://github.com/ggml-org/llama.cpp"
_DEFAULT_LLAMA_TAG="latest"
+_DEFAULT_LLAMA_FORCE_COMPILE_REF="master"
# ββ Colors (same palette as startup_banner / install_python_stack) ββ
if [ -n "${NO_COLOR:-}" ]; then
@@ -121,6 +122,45 @@ print_llama_error_log() {
tail -n 120 "$log_file" | sed 's/^/ | /' >&2
}
+installed_llama_prebuilt_release() {
+ local install_dir=${1:-}
+ local metadata_path="$install_dir/UNSLOTH_PREBUILT_INFO.json"
+ [ -f "$metadata_path" ] || return 0
+ python - "$metadata_path" <<'PY' 2>/dev/null || true
+import json
+import sys
+from pathlib import Path
+
+try:
+ payload = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
+except Exception:
+ raise SystemExit(0)
+
+if not isinstance(payload, dict):
+ raise SystemExit(0)
+
+repo = str(payload.get("published_repo") or "").strip()
+release_tag = str(payload.get("release_tag") or "").strip()
+llama_tag = str(payload.get("tag") or "").strip()
+if not repo or not release_tag:
+ raise SystemExit(0)
+
+message = f"installed release: {repo}@{release_tag}"
+if llama_tag and llama_tag != release_tag:
+ message += f" (tag {llama_tag})"
+print(message)
+PY
+}
+
+print_installed_llama_prebuilt_release() {
+ local install_dir=${1:-}
+ local installed_release
+ installed_release="$(installed_llama_prebuilt_release "$install_dir")"
+ if [ -n "$installed_release" ]; then
+ substep "$installed_release"
+ fi
+}
+
# ββ Banner ββ
echo ""
printf " ${C_TITLE}%s${C_RST}\n" "π¦₯ Unsloth Studio Setup"
@@ -487,30 +527,27 @@ _NEED_LLAMA_SOURCE_BUILD=false
_LLAMA_CPP_DEGRADED=false
_LLAMA_FORCE_COMPILE="${UNSLOTH_LLAMA_FORCE_COMPILE:-0}"
_REQUESTED_LLAMA_TAG="${UNSLOTH_LLAMA_TAG:-${_DEFAULT_LLAMA_TAG}}"
-# Force all installs to use mainline llama.cpp from ggml-org.
-_HELPER_RELEASE_REPO="ggml-org/llama.cpp"
+_HOST_SYSTEM="$(uname -s 2>/dev/null || true)"
+if [ "$_HOST_SYSTEM" = "Darwin" ]; then
+ _HELPER_RELEASE_REPO="ggml-org/llama.cpp"
+else
+ _HELPER_RELEASE_REPO="unslothai/llama.cpp"
+fi
_LLAMA_PR="${UNSLOTH_LLAMA_PR:-}"
-
+_SKIP_PREBUILT_INSTALL=false
_LLAMA_PR_FORCE="${UNSLOTH_LLAMA_PR_FORCE:-${_DEFAULT_LLAMA_PR_FORCE}}"
-# Force mainline source -- no env var override for now.
_LLAMA_SOURCE="${_DEFAULT_LLAMA_SOURCE}"
_LLAMA_SOURCE="${_LLAMA_SOURCE%.git}" # normalize: strip trailing .git
_RESOLVED_SOURCE_URL="$_LLAMA_SOURCE"
_RESOLVED_SOURCE_REF="$_REQUESTED_LLAMA_TAG"
_RESOLVED_SOURCE_REF_KIND="tag"
+_RESOLVED_LLAMA_TAG="$_REQUESTED_LLAMA_TAG"
if [ "$_LLAMA_FORCE_COMPILE" = "1" ]; then
_NEED_LLAMA_SOURCE_BUILD=true
_SKIP_PREBUILT_INSTALL=true
fi
-# Non-default source URL forces source build (fork has different code than prebuilt).
-if [ "$_LLAMA_SOURCE" != "https://github.com/ggml-org/llama.cpp" ]; then
- step "llama.cpp" "custom source: $_LLAMA_SOURCE -- forcing source build" "$C_WARN"
- _NEED_LLAMA_SOURCE_BUILD=true
- _SKIP_PREBUILT_INSTALL=true
-fi
-
# Baked-in PR_FORCE promotes to _LLAMA_PR when user hasn't set one.
if [ -z "$_LLAMA_PR" ] && [ -n "$_LLAMA_PR_FORCE" ] && \
[[ "$_LLAMA_PR_FORCE" =~ ^[0-9]+$ ]] && [ "$_LLAMA_PR_FORCE" -gt 0 ]; then
@@ -530,149 +567,68 @@ if [ -n "$_LLAMA_PR" ]; then
_RESOLVED_SOURCE_REF_KIND="pull"
_NEED_LLAMA_SOURCE_BUILD=true
_SKIP_PREBUILT_INSTALL=true
-elif [ "${_SKIP_PREBUILT_INSTALL:-false}" = true ]; then
- # Custom source or other override already forced source build; skip
- # the prebuilt release resolution entirely. When building from a custom
- # fork, the fork may not carry upstream bNNNN tags, so resolve the tag
- # only when the source is the default ggml-org repo.
- if [ "$_LLAMA_FORCE_COMPILE" = "1" ]; then
- _RESOLVED_LLAMA_TAG="$_REQUESTED_LLAMA_TAG"
- elif [ "$_LLAMA_SOURCE" = "https://github.com/ggml-org/llama.cpp" ]; then
- _RESOLVE_TAG_ARGS=(--resolve-llama-tag "$_REQUESTED_LLAMA_TAG" --published-repo "$_HELPER_RELEASE_REPO")
- _RESOLVE_TAG_ARGS+=(--output-format json)
- if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then
- _RESOLVE_TAG_ARGS+=(--published-release-tag "$UNSLOTH_LLAMA_RELEASE_TAG")
- fi
- set +e
- _RESOLVE_TAG_JSON="$(python "$SCRIPT_DIR/install_llama_prebuilt.py" "${_RESOLVE_TAG_ARGS[@]}" 2>/dev/null)"
- _RESOLVE_UPSTREAM_STATUS=$?
- set -e
- if [ "$_RESOLVE_UPSTREAM_STATUS" -eq 0 ] && [ -n "${_RESOLVE_TAG_JSON:-}" ]; then
- _RESOLVED_LLAMA_TAG="$(
- printf '%s' "$_RESOLVE_TAG_JSON" | python -c 'import json,sys; print(json.load(sys.stdin).get("llama_tag",""))' 2>/dev/null || true
- )"
- else
- _RESOLVED_LLAMA_TAG=""
- fi
- if [ -z "$_RESOLVED_LLAMA_TAG" ]; then
- _RESOLVED_LLAMA_TAG="$_REQUESTED_LLAMA_TAG"
- fi
- else
- _RESOLVED_LLAMA_TAG="$_REQUESTED_LLAMA_TAG"
- fi
-else
- _RESOLVE_INSTALL_ARGS=(--resolve-install-tag "$_REQUESTED_LLAMA_TAG" --published-repo "$_HELPER_RELEASE_REPO")
- _RESOLVE_INSTALL_ARGS+=(--output-format json)
- if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then
- _RESOLVE_INSTALL_ARGS+=(--published-release-tag "$UNSLOTH_LLAMA_RELEASE_TAG")
- fi
- _RESOLVE_LLAMA_LOG="$(mktemp)"
- set +e
- _RESOLVE_INSTALL_JSON="$(
- python "$SCRIPT_DIR/install_llama_prebuilt.py" \
- "${_RESOLVE_INSTALL_ARGS[@]}" 2>"$_RESOLVE_LLAMA_LOG"
- )"
- _RESOLVE_LLAMA_STATUS=$?
- set -e
- if [ "$_RESOLVE_LLAMA_STATUS" -eq 0 ]; then
- _RESOLVED_LLAMA_TAG="$(
- printf '%s' "${_RESOLVE_INSTALL_JSON:-}" | python -c 'import json,sys; print(json.load(sys.stdin).get("llama_tag",""))' 2>/dev/null || true
- )"
- else
- _RESOLVED_LLAMA_TAG=""
- fi
- if [ -z "$_RESOLVED_LLAMA_TAG" ]; then
- step "llama.cpp" "failed to resolve a published llama.cpp release via $_HELPER_RELEASE_REPO" "$C_WARN"
- print_llama_error_log "$_RESOLVE_LLAMA_LOG"
- set +e
- # Resolve the llama.cpp tag for source-build fallback. Pass --published-repo
- # so the resolver prefers the latest usable Unsloth-published upstream tag
- # before falling back to the bleeding-edge ggml-org/llama.cpp tag.
- _RESOLVE_FALLBACK_ARGS=(--resolve-llama-tag "$_REQUESTED_LLAMA_TAG" --published-repo "$_HELPER_RELEASE_REPO")
- _RESOLVE_FALLBACK_ARGS+=(--output-format json)
- if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then
- _RESOLVE_FALLBACK_ARGS+=(--published-release-tag "$UNSLOTH_LLAMA_RELEASE_TAG")
- fi
- _RESOLVE_FALLBACK_JSON="$(python "$SCRIPT_DIR/install_llama_prebuilt.py" "${_RESOLVE_FALLBACK_ARGS[@]}" 2>/dev/null)"
- _RESOLVE_UPSTREAM_STATUS=$?
- set -e
- if [ "$_RESOLVE_UPSTREAM_STATUS" -eq 0 ] && [ -n "${_RESOLVE_FALLBACK_JSON:-}" ]; then
- _RESOLVED_LLAMA_TAG="$(
- printf '%s' "$_RESOLVE_FALLBACK_JSON" | python -c 'import json,sys; print(json.load(sys.stdin).get("llama_tag",""))' 2>/dev/null || true
- )"
- else
- _RESOLVED_LLAMA_TAG=""
- fi
- if [ -z "$_RESOLVED_LLAMA_TAG" ]; then
- _RESOLVED_LLAMA_TAG="$_REQUESTED_LLAMA_TAG"
- fi
- _NEED_LLAMA_SOURCE_BUILD=true
- _SKIP_PREBUILT_INSTALL=true
- fi
- rm -f "$_RESOLVE_LLAMA_LOG"
fi
-substep "resolved llama.cpp tag: $_RESOLVED_LLAMA_TAG"
verbose_substep "requested llama.cpp tag: $_REQUESTED_LLAMA_TAG (repo: $_HELPER_RELEASE_REPO)"
if [ "$_LLAMA_FORCE_COMPILE" = "1" ]; then
step "llama.cpp" "UNSLOTH_LLAMA_FORCE_COMPILE=1 -- skipping prebuilt" "$C_WARN"
_NEED_LLAMA_SOURCE_BUILD=true
+elif [ "${_SKIP_PREBUILT_INSTALL:-false}" = true ]; then
+ substep "prebuilt install skipped -- falling back to source build"
else
substep "installing prebuilt llama.cpp..."
if [ -d "$LLAMA_CPP_DIR" ]; then
substep "existing install detected -- validating update"
fi
- if [ "${_SKIP_PREBUILT_INSTALL:-false}" = true ]; then
- substep "prebuilt tag resolution failed -- falling back to source build"
+ _PREBUILT_CMD=(
+ python "$SCRIPT_DIR/install_llama_prebuilt.py"
+ --install-dir "$LLAMA_CPP_DIR"
+ --llama-tag "$_REQUESTED_LLAMA_TAG"
+ --published-repo "$_HELPER_RELEASE_REPO"
+ --simple-policy
+ )
+ if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then
+ _PREBUILT_CMD+=(--published-release-tag "$UNSLOTH_LLAMA_RELEASE_TAG")
+ fi
+ _PREBUILT_LOG="$(mktemp)"
+ set +e
+ if _is_verbose; then
+ "${_PREBUILT_CMD[@]}" 2>&1 | tee "$_PREBUILT_LOG"
+ _PREBUILT_STATUS=${PIPESTATUS[0]}
else
- _PREBUILT_CMD=(
- python "$SCRIPT_DIR/install_llama_prebuilt.py"
- --install-dir "$LLAMA_CPP_DIR"
- --llama-tag "$_REQUESTED_LLAMA_TAG"
- --published-repo "$_HELPER_RELEASE_REPO"
- )
- if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then
- _PREBUILT_CMD+=(--published-release-tag "$UNSLOTH_LLAMA_RELEASE_TAG")
- fi
- _PREBUILT_LOG="$(mktemp)"
- set +e
- if _is_verbose; then
- "${_PREBUILT_CMD[@]}" 2>&1 | tee "$_PREBUILT_LOG"
- _PREBUILT_STATUS=${PIPESTATUS[0]}
- else
- "${_PREBUILT_CMD[@]}" >"$_PREBUILT_LOG" 2>&1
- _PREBUILT_STATUS=$?
- fi
- set -e
+ "${_PREBUILT_CMD[@]}" >"$_PREBUILT_LOG" 2>&1
+ _PREBUILT_STATUS=$?
+ fi
+ set -e
- if [ "$_PREBUILT_STATUS" -eq 0 ]; then
- if grep -Fq "already matches" "$_PREBUILT_LOG"; then
- step "llama.cpp" "prebuilt up to date and validated"
- else
- step "llama.cpp" "prebuilt installed and validated"
- fi
- verbose_substep "llama.cpp install dir: $LLAMA_CPP_DIR"
- rm -f "$_PREBUILT_LOG"
- elif [ "$_PREBUILT_STATUS" -eq 3 ]; then
- step "llama.cpp" "install blocked by active llama.cpp process" "$C_WARN"
- print_llama_error_log "$_PREBUILT_LOG"
- rm -f "$_PREBUILT_LOG"
- if [ -d "$LLAMA_CPP_DIR" ]; then
- substep "existing install was restored"
- fi
- substep "close Studio or other llama.cpp users and retry"
- exit 3
+ if [ "$_PREBUILT_STATUS" -eq 0 ]; then
+ if grep -Fq "already matches" "$_PREBUILT_LOG"; then
+ step "llama.cpp" "prebuilt up to date and validated"
else
- step "llama.cpp" "prebuilt install failed (continuing)" "$C_WARN"
- print_llama_error_log "$_PREBUILT_LOG"
- rm -f "$_PREBUILT_LOG"
- if [ -d "$LLAMA_CPP_DIR" ]; then
- substep "prebuilt update failed; existing install restored"
- fi
- substep "falling back to source build"
- _NEED_LLAMA_SOURCE_BUILD=true
+ step "llama.cpp" "prebuilt installed and validated"
fi
+ print_installed_llama_prebuilt_release "$LLAMA_CPP_DIR"
+ verbose_substep "llama.cpp install dir: $LLAMA_CPP_DIR"
+ rm -f "$_PREBUILT_LOG"
+ elif [ "$_PREBUILT_STATUS" -eq 3 ]; then
+ step "llama.cpp" "install blocked by active llama.cpp process" "$C_WARN"
+ print_llama_error_log "$_PREBUILT_LOG"
+ rm -f "$_PREBUILT_LOG"
+ if [ -d "$LLAMA_CPP_DIR" ]; then
+ substep "existing install was restored"
+ fi
+ substep "close Studio or other llama.cpp users and retry"
+ exit 3
+ else
+ step "llama.cpp" "prebuilt install failed (continuing)" "$C_WARN"
+ print_llama_error_log "$_PREBUILT_LOG"
+ rm -f "$_PREBUILT_LOG"
+ if [ -d "$LLAMA_CPP_DIR" ]; then
+ substep "prebuilt update failed; existing install restored"
+ fi
+ substep "falling back to source build"
+ _NEED_LLAMA_SOURCE_BUILD=true
fi
fi
@@ -746,33 +702,41 @@ else
[ -f "$LLAMA_SERVER_BIN" ] || _LLAMA_CPP_DEGRADED=true
else
if [ -z "$_LLAMA_PR" ]; then
- if [ "$_LLAMA_SOURCE" = "https://github.com/ggml-org/llama.cpp" ]; then
- _RESOLVE_SOURCE_ARGS=(--resolve-source-build "$_REQUESTED_LLAMA_TAG" --published-repo "$_HELPER_RELEASE_REPO")
- _RESOLVE_SOURCE_ARGS+=(--output-format json)
- if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then
- _RESOLVE_SOURCE_ARGS+=(--published-release-tag "$UNSLOTH_LLAMA_RELEASE_TAG")
+ _RESOLVED_SOURCE_URL="$_LLAMA_SOURCE"
+ if [ "$_LLAMA_FORCE_COMPILE" = "1" ]; then
+ if [ "$_REQUESTED_LLAMA_TAG" = "latest" ]; then
+ _RESOLVED_SOURCE_REF="${UNSLOTH_LLAMA_FORCE_COMPILE_REF:-${_DEFAULT_LLAMA_FORCE_COMPILE_REF}}"
+ _RESOLVED_SOURCE_REF_KIND="branch"
+ else
+ _RESOLVED_SOURCE_REF="$_REQUESTED_LLAMA_TAG"
+ _RESOLVED_SOURCE_REF_KIND="tag"
fi
+ elif [ "$_REQUESTED_LLAMA_TAG" = "latest" ]; then
+ _RESOLVE_TAG_ARGS=(--resolve-llama-tag latest --published-repo "ggml-org/llama.cpp" --output-format json)
set +e
- _SOURCE_BUILD_PLAN="$(python "$SCRIPT_DIR/install_llama_prebuilt.py" "${_RESOLVE_SOURCE_ARGS[@]}" 2>/dev/null)"
- _RESOLVE_SOURCE_STATUS=$?
+ _RESOLVE_TAG_JSON="$(python "$SCRIPT_DIR/install_llama_prebuilt.py" "${_RESOLVE_TAG_ARGS[@]}" 2>/dev/null)"
+ _RESOLVE_TAG_STATUS=$?
set -e
- if [ "$_RESOLVE_SOURCE_STATUS" -eq 0 ] && [ -n "$_SOURCE_BUILD_PLAN" ]; then
- _RESOLVED_SOURCE_URL="$(
- printf '%s' "$_SOURCE_BUILD_PLAN" | python -c 'import json,sys; print(json.load(sys.stdin).get("source_url",""))' 2>/dev/null || true
- )"
- _RESOLVED_SOURCE_REF_KIND="$(
- printf '%s' "$_SOURCE_BUILD_PLAN" | python -c 'import json,sys; print(json.load(sys.stdin).get("source_ref_kind",""))' 2>/dev/null || true
- )"
+ if [ "$_RESOLVE_TAG_STATUS" -eq 0 ] && [ -n "${_RESOLVE_TAG_JSON:-}" ]; then
_RESOLVED_SOURCE_REF="$(
- printf '%s' "$_SOURCE_BUILD_PLAN" | python -c 'import json,sys; print(json.load(sys.stdin).get("source_ref",""))' 2>/dev/null || true
+ printf '%s' "$_RESOLVE_TAG_JSON" | python -c 'import json,sys; print(json.load(sys.stdin).get("llama_tag",""))' 2>/dev/null || true
)"
+ else
+ _RESOLVED_SOURCE_REF=""
fi
+ if [ -z "$_RESOLVED_SOURCE_REF" ]; then
+ _RESOLVED_SOURCE_REF="latest"
+ fi
+ _RESOLVED_SOURCE_REF_KIND="tag"
+ else
+ _RESOLVED_SOURCE_REF="$_REQUESTED_LLAMA_TAG"
+ _RESOLVED_SOURCE_REF_KIND="tag"
fi
if [ -z "$_RESOLVED_SOURCE_URL" ]; then
_RESOLVED_SOURCE_URL="$_LLAMA_SOURCE"
fi
if [ -z "$_RESOLVED_SOURCE_REF" ]; then
- _RESOLVED_SOURCE_REF="$_RESOLVED_LLAMA_TAG"
+ _RESOLVED_SOURCE_REF="$_REQUESTED_LLAMA_TAG"
fi
fi
verbose_substep "source build repo: $_RESOLVED_SOURCE_URL"
diff --git a/tests/studio/install/test_pr4562_bugfixes.py b/tests/studio/install/test_pr4562_bugfixes.py
index 7fa654d845..4a309036d6 100644
--- a/tests/studio/install/test_pr4562_bugfixes.py
+++ b/tests/studio/install/test_pr4562_bugfixes.py
@@ -14,10 +14,12 @@ Run: pytest tests/studio/install/test_pr4562_bugfixes.py -v
"""
import importlib.util
+import json
import os
import subprocess
import sys
import textwrap
+import urllib.parse
from pathlib import Path
import pytest
@@ -347,6 +349,47 @@ class TestResolveRequestedLlamaTag:
}
+class TestFetchJsonRetries:
+ def test_fetch_json_retries_invalid_github_api_json(
+ self, monkeypatch: pytest.MonkeyPatch
+ ):
+ calls = {"count": 0}
+
+ def fake_download_bytes(url, **kwargs):
+ calls["count"] += 1
+ if calls["count"] == 1:
+ return b'{"incomplete":"payload'
+ return json.dumps([{"tag_name": "b8635"}]).encode("utf-8")
+
+ monkeypatch.setattr(MOD, "download_bytes", fake_download_bytes)
+ monkeypatch.setattr(MOD, "sleep_backoff", lambda _attempt: None)
+
+ payload = MOD.fetch_json(
+ "https://api.github.com/repos/ggml-org/llama.cpp/releases?per_page=100&page=1"
+ )
+
+ assert isinstance(payload, list)
+ assert payload[0]["tag_name"] == "b8635"
+ assert calls["count"] == 2
+
+ def test_github_releases_honors_max_pages(self, monkeypatch: pytest.MonkeyPatch):
+ seen_pages: list[int] = []
+
+ def fake_fetch_json(url: str):
+ parsed = urllib.parse.urlparse(url)
+ params = urllib.parse.parse_qs(parsed.query)
+ page = int(params["page"][0])
+ seen_pages.append(page)
+ return [{"tag_name": f"b{page:04d}"} for _ in range(100)]
+
+ monkeypatch.setattr(MOD, "fetch_json", fake_fetch_json)
+
+ releases = MOD.github_releases("ggml-org/llama.cpp", max_pages = 2)
+
+ assert seen_pages == [1, 2]
+ assert len(releases) == 200
+
+
# =========================================================================
# TEST GROUP C: setup.sh logic (bash subprocess tests)
# =========================================================================
@@ -643,25 +686,35 @@ class TestSourceCodePatterns:
'_RESOLVED_SOURCE_REF" != "latest"' in content
), "Should guard against literal 'latest' tag"
- def test_setup_sh_source_build_uses_helper_resolution(self):
- """Shell source fallback should consult the helper for repo/ref planning."""
+ def test_setup_sh_source_build_uses_helper_latest_tag_only(self):
+ """Shell source fallback should only use helper latest-tag resolution."""
content = SETUP_SH.read_text()
- assert "--resolve-source-build" in content
+ assert "--resolve-source-build" not in content
+ assert "--resolve-install-tag" not in content
+ assert (
+ '--resolve-llama-tag latest --published-repo "ggml-org/llama.cpp"'
+ in content
+ )
assert "--output-format json" in content
assert "_RESOLVED_SOURCE_URL" in content
assert "_RESOLVED_SOURCE_REF_KIND" in content
assert "_RESOLVED_SOURCE_REF" in content
- def test_setup_sh_latest_resolution_uses_helper_only(self):
- """Shell fallback should rely on helper output, not raw GitHub API tag_name."""
+ def test_setup_sh_prebuilt_install_uses_simple_policy_only(self):
+ """Shell prebuilt path should use the simplified helper install entrypoint."""
content = SETUP_SH.read_text()
- assert "--resolve-install-tag" in content
- assert "--resolve-llama-tag" in content
- assert 'tail -n 1 "$_RESOLVE_LLAMA_LOG"' not in content
- assert "json.load" in content
+ assert "--simple-policy" in content
+ assert "--resolve-install-tag" not in content
assert "_HELPER_RELEASE_REPO}/releases/latest" not in content
assert "ggml-org/llama.cpp/releases/latest" not in content
+ def test_setup_sh_reports_installed_prebuilt_release(self):
+ """Shell wrapper should report the installed prebuilt release from metadata."""
+ content = SETUP_SH.read_text()
+ assert "UNSLOTH_PREBUILT_INFO.json" in content
+ assert "installed release:" in content
+ assert 'print_installed_llama_prebuilt_release "$LLAMA_CPP_DIR"' in content
+
def test_setup_sh_macos_arm64_uses_metal_flags(self):
"""Apple Silicon source builds should explicitly enable Metal like upstream."""
content = SETUP_SH.read_text()
@@ -759,20 +812,34 @@ class TestSourceCodePatterns:
f"Found 'git pull' in llama.cpp build section at line {i+1}"
)
- def test_setup_ps1_latest_resolution_uses_helper_only(self):
- """PS1 fallback should rely on helper output, not raw GitHub API tag_name."""
+ def test_setup_ps1_prebuilt_install_uses_simple_policy_only(self):
+ """PS1 prebuilt path should use the simplified helper install entrypoint."""
content = SETUP_PS1.read_text()
- assert "--resolve-install-tag" in content
- assert "--resolve-llama-tag" in content
- assert '--output-format", "json"' in content
- assert "ConvertFrom-Json" in content
+ assert '"--simple-policy"' in content
+ assert "--resolve-install-tag" not in content
assert "$HelperReleaseRepo/releases/latest" not in content
assert "ggml-org/llama.cpp/releases/latest" not in content
- def test_setup_ps1_source_build_uses_helper_resolution(self):
- """PS1 source fallback should consult the helper for repo/ref planning."""
+ def test_setup_ps1_reports_installed_prebuilt_release(self):
+ """PS1 wrapper should report the installed prebuilt release from metadata."""
content = SETUP_PS1.read_text()
- assert "--resolve-source-build" in content
+ assert "Get-InstalledLlamaPrebuiltRelease" in content
+ assert "UNSLOTH_PREBUILT_INFO.json" in content
+ assert "installed release:" in content
+ assert (
+ "$installedRelease = Get-InstalledLlamaPrebuiltRelease -InstallDir $LlamaCppDir"
+ in content
+ )
+
+ def test_setup_ps1_source_build_uses_helper_latest_tag_only(self):
+ """PS1 source fallback should only use helper latest-tag resolution."""
+ content = SETUP_PS1.read_text()
+ assert "--resolve-source-build" not in content
+ assert "--resolve-install-tag" not in content
+ assert (
+ '"--resolve-llama-tag", "latest", "--published-repo", "ggml-org/llama.cpp"'
+ in content
+ )
assert '--output-format", "json"' in content
assert "$ResolvedSourceUrl" in content
assert "$ResolvedSourceRefKind" in content
@@ -794,18 +861,25 @@ class TestSourceCodePatterns:
"""Helper resolution should suppress terminating NativeCommandError on PS 5.1."""
content = SETUP_PS1.read_text()
helper_idx = content.index("function Invoke-LlamaHelper")
- block = content[helper_idx : helper_idx + 1200]
+ block = content[helper_idx : helper_idx + 2200]
assert "$previousErrorActionPreference = $ErrorActionPreference" in block
assert '$ErrorActionPreference = "Continue"' in block
assert "$ErrorActionPreference = $previousErrorActionPreference" in block
def test_setup_ps1_uses_local_tempfile_helper(self):
- """PS1 should not depend on New-TemporaryFile being available."""
+ """PS1 should not depend on New-TemporaryFile being available anywhere."""
content = SETUP_PS1.read_text()
assert "function New-UnslothTemporaryFile" in content
- assert "$resolveErrorLog = New-UnslothTemporaryFile" in content
assert "$resolveErrorLog = New-TemporaryFile" not in content
+ def test_setup_ps1_find_nvcc_uses_version_sort_for_latest_toolkit(self):
+ """The unconstrained nvcc fallback should not sort toolkit dirs lexicographically."""
+ content = SETUP_PS1.read_text()
+ assert "Sort-Object Name | Select-Object -Last 1" not in content
+ assert (
+ "Sort-Object { [version]($_.Name -replace '^v','') } -Descending" in content
+ )
+
def test_binary_env_linux_has_binary_parent(self):
"""The Linux branch of binary_env should include binary_path.parent."""
content = MODULE_PATH.read_text()
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 28526056ba..dcb7334417 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-__version__ = "2026.4.1"
+__version__ = "2026.4.2"
__all__ = [
"SUPPORTS_BFLOAT16",
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index a811d3fb75..a6cb3eb529 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -114,6 +114,17 @@ FORCE_FLOAT32 = [
global DISABLE_COMPILE_MODEL_NAMES
# Must be alphabetically sorted for each entry
+
+
+def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:
+ """Remove Unsloth 4bit suffixes without lowercasing (HF cache dirs are case-sensitive)."""
+ s = model_name
+ for suffix in ("-unsloth-bnb-4bit", "-bnb-4bit"):
+ if len(s) >= len(suffix) and s.lower().endswith(suffix.lower()):
+ s = s[: -len(suffix)]
+ return s
+
+
DISABLE_COMPILE_MODEL_NAMES = [
"aya_vision",
"modernbert",
@@ -404,8 +415,7 @@ class FastLanguageModel(FastLlamaModel):
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
- model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
- model_name = model_name.lower().removesuffix("-bnb-4bit")
+ model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
# Change -BF16 to all False for 4bit, 8bit etc
if model_name.lower().endswith("-bf16"):
load_in_4bit = False
@@ -551,8 +561,7 @@ class FastLanguageModel(FastLlamaModel):
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
- model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
- model_name = model_name.lower().removesuffix("-bnb-4bit")
+ model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
# Change -BF16 to all False for 4bit, 8bit etc
if model_name.lower().endswith("-bf16"):
load_in_4bit = False
@@ -1019,8 +1028,7 @@ class FastModel(FastBaseModel):
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
- model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
- model_name = model_name.lower().removesuffix("-bnb-4bit")
+ model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
# Change -BF16 to all False for 4bit, 8bit etc
if model_name.lower().endswith("-bf16"):
load_in_4bit = False
@@ -1320,8 +1328,7 @@ class FastModel(FastBaseModel):
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
- model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
- model_name = model_name.lower().removesuffix("-bnb-4bit")
+ model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
# Change -BF16 to all False for 4bit, 8bit etc
if model_name.lower().endswith("-bf16"):
load_in_4bit = False
@@ -1533,14 +1540,72 @@ class FastModel(FastBaseModel):
if is_peft:
# From https://github.com/huggingface/peft/issues/184
# Now add PEFT adapters
- model = PeftModel.from_pretrained(
- model,
- old_model_name,
- token = token,
- revision = revision,
- is_trainable = True,
- trust_remote_code = trust_remote_code,
- )
+
+ # Gemma4 ClippableLinear wraps nn.Linear -- PEFT can't inject LoRA
+ # on it directly. Monkey-patch PEFT to target the inner .linear
+ # child instead (same patch as vision.py training path).
+ # See https://github.com/huggingface/peft/issues/3129
+ _clippable_linear_cls = None
+ try:
+ from transformers.models.gemma4.modeling_gemma4 import (
+ Gemma4ClippableLinear as _clippable_linear_cls,
+ )
+ except ImportError:
+ pass
+
+ if _clippable_linear_cls is not None:
+ from peft.tuners.lora.model import LoraModel as _LoraModel
+
+ _original_car = _LoraModel._create_and_replace
+
+ def _patched_car(
+ self,
+ peft_config,
+ adapter_name,
+ target,
+ target_name,
+ parent,
+ current_key = None,
+ **kwargs,
+ ):
+ if isinstance(target, _clippable_linear_cls):
+ return _original_car(
+ self,
+ peft_config,
+ adapter_name,
+ target.linear,
+ "linear",
+ target,
+ current_key = current_key,
+ **kwargs,
+ )
+ return _original_car(
+ self,
+ peft_config,
+ adapter_name,
+ target,
+ target_name,
+ parent,
+ current_key = current_key,
+ **kwargs,
+ )
+
+ _LoraModel._create_and_replace = _patched_car
+
+ try:
+ model = PeftModel.from_pretrained(
+ model,
+ old_model_name,
+ token = token,
+ revision = revision,
+ is_trainable = True,
+ trust_remote_code = trust_remote_code,
+ )
+ finally:
+ # Always restore original PEFT method, even if loading fails
+ if _clippable_linear_cls is not None:
+ _LoraModel._create_and_replace = _original_car
+
# Patch it as well!
model = FastBaseModel.post_patch_model(
model, use_gradient_checkpointing, trust_remote_code = trust_remote_code
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index cf5af983a6..99da5f799e 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -162,7 +162,7 @@ def __get_model_name(
# Support returning original full -bnb-4bit name if specified specifically
# since we'll map it to the dynamic version instead
if lower_model_name.endswith("-bnb-4bit"):
- return lower_model_name
+ return model_name
new_model_name = FLOAT_TO_INT_MAPPER[lower_model_name]
# logger.warning_once(