From ac562bac668347b70017dce7540fad1dd96989bc Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Fri, 3 Apr 2026 02:34:20 -0500 Subject: [PATCH] Fix/llama.cppbuilding (#4804) * Simplify llama.cpp install logic * print release tag * Retry failed json decode * don't pull all ggml releases * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove test file changes from main PR Test changes for test_pr4562_bugfixes.py will be submitted in a separate PR to keep this PR focused on the install path simplification. * Fix setup.sh executable bit and direct tag lookup for pinned releases - Restore setup.sh file mode to 100755 (was accidentally changed to 100644) - Add direct GitHub API tag lookup in iter_release_payloads_by_time for non-latest requested tags (e.g. b7879) instead of relying on paginated release scans that may miss older releases beyond the 5-page limit - Update stale DEFAULT_PUBLISHED_REPO comment to match new value * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix force-compile default ref and remove dead code in setup.ps1 - Change FORCE_COMPILE_DEFAULT_REF from "main" to "master" in all three files (install_llama_prebuilt.py, setup.sh, setup.ps1) since ggml-org/llama.cpp uses "master" as its default branch, not "main". Using "main" would cause git clone --branch to fail when UNSLOTH_LLAMA_FORCE_COMPILE=1 with UNSLOTH_LLAMA_TAG=latest. - Remove dead if ($SkipPrebuiltInstall) block inside the else branch of setup.ps1 that could never be reached (the outer elseif already handles $SkipPrebuiltInstall=true). - Maintain setup.sh executable bit (100755). * Improve iter_release_payloads_by_time error handling for direct tag lookup When a pinned release tag is not found (HTTP 404), fall through to the paginated release scan instead of silently returning empty results. Non-404 errors (network failures, rate limits) are propagated to the caller so users get actionable error messages. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- studio/install_llama_prebuilt.py | 571 ++++++++++++++++++++++++++++--- studio/setup.ps1 | 160 ++++----- studio/setup.sh | 272 +++++++-------- 3 files changed, 711 insertions(+), 292 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 5e8fe314b4..8d06c7d0e1 100755 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -65,9 +65,10 @@ def env_int(name: str, default: int, *, minimum: int | None = None) -> int: # errors. Only use "master" temporarily when the latest release is missing # support for a new model architecture. DEFAULT_LLAMA_TAG = os.environ.get("UNSLOTH_LLAMA_TAG", "latest") -# Force all installs to use mainline llama.cpp from ggml-org. -# Previously: DEFAULT_PUBLISHED_REPO = os.environ.get("UNSLOTH_LLAMA_RELEASE_REPO", "unslothai/llama.cpp") -DEFAULT_PUBLISHED_REPO = "ggml-org/llama.cpp" +# Default published repo for prebuilt release resolution. Linux uses +# Unsloth prebuilts; setup.sh/setup.ps1 pass --published-repo explicitly +# for macOS/Windows to override with ggml-org/llama.cpp when needed. +DEFAULT_PUBLISHED_REPO = "unslothai/llama.cpp" DEFAULT_PUBLISHED_TAG = os.environ.get("UNSLOTH_LLAMA_RELEASE_TAG") DEFAULT_PUBLISHED_MANIFEST_ASSET = os.environ.get( "UNSLOTH_LLAMA_RELEASE_MANIFEST_ASSET", "llama-prebuilt-manifest.json" @@ -89,6 +90,12 @@ GITHUB_AUTH_HOSTS = {"api.github.com", "github.com"} RETRYABLE_HTTP_STATUS = {408, 429, 500, 502, 503, 504} HTTP_FETCH_ATTEMPTS = 4 HTTP_FETCH_BASE_DELAY_SECONDS = 0.75 +JSON_FETCH_ATTEMPTS = 3 +DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES = env_int( + "UNSLOTH_LLAMA_GITHUB_RELEASE_SCAN_MAX_PAGES", + 5, + minimum = 1, +) SERVER_PORT_BIND_ATTEMPTS = 3 SERVER_BIND_RETRY_WINDOW_SECONDS = 5.0 TTY_PROGRESS_START_DELAY_SECONDS = 0.5 @@ -97,6 +104,58 @@ DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS = env_int( 2, minimum = 1, ) +FORCE_COMPILE_DEFAULT_REF = os.environ.get("UNSLOTH_LLAMA_FORCE_COMPILE_REF", "master") + +DIRECT_LINUX_BUNDLE_PROFILES: dict[str, dict[str, Any]] = { + "cuda12-older": { + "runtime_line": "cuda12", + "coverage_class": "older", + "supported_sms": ["70", "75", "80", "86", "89"], + "min_sm": 70, + "max_sm": 89, + "rank": 10, + }, + "cuda12-newer": { + "runtime_line": "cuda12", + "coverage_class": "newer", + "supported_sms": ["86", "89", "90", "100", "120"], + "min_sm": 86, + "max_sm": 120, + "rank": 20, + }, + "cuda12-portable": { + "runtime_line": "cuda12", + "coverage_class": "portable", + "supported_sms": ["70", "75", "80", "86", "89", "90", "100", "120"], + "min_sm": 70, + "max_sm": 120, + "rank": 30, + }, + "cuda13-older": { + "runtime_line": "cuda13", + "coverage_class": "older", + "supported_sms": ["75", "80", "86", "89"], + "min_sm": 75, + "max_sm": 89, + "rank": 40, + }, + "cuda13-newer": { + "runtime_line": "cuda13", + "coverage_class": "newer", + "supported_sms": ["86", "89", "90", "100", "120"], + "min_sm": 86, + "max_sm": 120, + "rank": 50, + }, + "cuda13-portable": { + "runtime_line": "cuda13", + "coverage_class": "portable", + "supported_sms": ["75", "80", "86", "89", "90", "100", "120"], + "min_sm": 75, + "max_sm": 120, + "rank": 60, + }, +} @dataclass @@ -753,32 +812,48 @@ def download_bytes( def fetch_json(url: str) -> Any: - try: - data = download_bytes( - url, - timeout = 30, - headers = github_api_headers(url) - if is_github_api_url(url) - else auth_headers(url), - ) - except urllib.error.HTTPError as exc: - if exc.code == 403 and is_github_api_url(url): - hint = "" - if not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")): - hint = "; set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits" - raise RuntimeError(f"GitHub API returned 403 for {url}{hint}") from exc - raise - if not data: - raise RuntimeError(f"downloaded empty JSON payload from {url}") - try: - payload = json.loads(data.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError(f"downloaded invalid JSON from {url}: {exc}") from exc - if not isinstance(payload, dict) and not isinstance(payload, list): - raise RuntimeError( - f"downloaded unexpected JSON type from {url}: {type(payload).__name__}" - ) - return payload + attempts = JSON_FETCH_ATTEMPTS if is_github_api_url(url) else 1 + last_decode_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + data = download_bytes( + url, + timeout = 30, + headers = github_api_headers(url) + if is_github_api_url(url) + else auth_headers(url), + ) + except urllib.error.HTTPError as exc: + if exc.code == 403 and is_github_api_url(url): + hint = "" + if not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")): + hint = ( + "; set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits" + ) + raise RuntimeError(f"GitHub API returned 403 for {url}{hint}") from exc + raise + if not data: + last_decode_exc = RuntimeError(f"downloaded empty JSON payload from {url}") + else: + try: + payload = json.loads(data.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + last_decode_exc = RuntimeError( + f"downloaded invalid JSON from {url}: {exc}" + ) + else: + if not isinstance(payload, dict) and not isinstance(payload, list): + raise RuntimeError( + f"downloaded unexpected JSON type from {url}: {type(payload).__name__}" + ) + return payload + if attempt >= attempts: + assert last_decode_exc is not None + raise last_decode_exc + log(f"json fetch failed ({attempt}/{attempts}) for {url}; retrying") + sleep_backoff(attempt) + assert last_decode_exc is not None + raise last_decode_exc def download_file(url: str, destination: Path) -> None: @@ -838,12 +913,16 @@ def download_file_verified( url: str, destination: Path, *, - expected_sha256: str, + expected_sha256: str | None, label: str, ) -> None: normalized_expected = normalize_sha256_digest(expected_sha256) if not normalized_expected: - raise PrebuiltFallback(f"{label} did not have a valid approved sha256") + download_file(url, destination) + log( + f"downloaded {label} without a published sha256; relying on install validation" + ) + return for attempt in range(1, 3): download_file(url, destination) @@ -898,7 +977,12 @@ def github_release(repo: str, tag: str) -> dict[str, Any]: return payload -def github_releases(repo: str, *, per_page: int = 100) -> list[dict[str, Any]]: +def github_releases( + repo: str, + *, + per_page: int = 100, + max_pages: int = 0, +) -> list[dict[str, Any]]: releases: list[dict[str, Any]] = [] page = 1 while True: @@ -912,6 +996,8 @@ def github_releases(repo: str, *, per_page: int = 100) -> list[dict[str, Any]]: if len(payload) < per_page: break page += 1 + if max_pages > 0 and page > max_pages: + break return releases @@ -925,6 +1011,372 @@ def latest_upstream_release_tag() -> str: return tag +def is_release_tag_like(value: str | None) -> bool: + return isinstance(value, str) and bool(re.fullmatch(r"b\d+", value.strip())) + + +def release_time_sort_key(release: dict[str, Any]) -> tuple[str, int]: + published_at = release.get("published_at") + created_at = release.get("created_at") + release_id = release.get("id") + timestamp = ( + published_at + if isinstance(published_at, str) and published_at + else created_at + if isinstance(created_at, str) and created_at + else "" + ) + try: + normalized_id = int(release_id) + except (TypeError, ValueError): + normalized_id = 0 + return (timestamp, normalized_id) + + +def iter_release_payloads_by_time( + repo: str, + published_release_tag: str = "", + requested_tag: str = "", +) -> Iterable[dict[str, Any]]: + if published_release_tag: + yield github_release(repo, published_release_tag) + return + + if ( + requested_tag + and requested_tag != "latest" + and is_release_tag_like(requested_tag) + ): + try: + yield github_release(repo, requested_tag) + return + except urllib.error.HTTPError as exc: + if exc.code == 404: + log( + f"release tag {requested_tag} not found in {repo}; scanning recent releases" + ) + else: + raise + except Exception: + raise + + releases = [ + release + for release in github_releases( + repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES + ) + if isinstance(release, dict) + and not release.get("draft") + and not release.get("prerelease") + ] + releases.sort(key = release_time_sort_key, reverse = True) + for release in releases: + yield release + + +def direct_release_matches_request( + *, release_tag: str, llama_tag: str, requested_tag: str +) -> bool: + if requested_tag == "latest": + return True + for candidate in (release_tag, llama_tag): + if refs_match(candidate, requested_tag): + return True + return False + + +def synthetic_checksums_for_release( + repo: str, release_tag: str, upstream_tag: str +) -> ApprovedReleaseChecksums: + return ApprovedReleaseChecksums( + repo = repo, + release_tag = release_tag, + upstream_tag = upstream_tag, + artifacts = {}, + ) + + +def parse_direct_linux_release_bundle( + repo: str, release: dict[str, Any] +) -> PublishedReleaseBundle | None: + release_tag = release.get("tag_name") + if not isinstance(release_tag, str) or not release_tag: + return None + + assets = release_asset_map(release) + artifacts: list[PublishedLlamaArtifact] = [] + inferred_labels: list[str] = [] + + linux_asset_re = re.compile( + r"^app-(?P