Studio: warn when llama.cpp prebuilt is at least 3 days behind (#5529)
* Studio: warn when llama.cpp prebuilt is at least 3 days behind Layered on #5528. Generalises the MTP-specific staleness warning to every llama.cpp prebuilt update, not just the ones that add MTP. If the installed prebuilt is at least 3 days old AND its tag differs from the latest published tag on the helper release repo (default unslothai/llama.cpp), Studio nudges the user to run "unsloth studio update". How it works Reads the install marker UNSLOTH_PREBUILT_INFO.json that install_llama_prebuilt.py already writes to install_dir. The marker carries the installed tag, the helper repo, and an installed_at_utc timestamp. Studio compares those against the latest published tag from the GitHub releases API for the helper repo. GitHub fetch is cached at two levels: - Process-level memo for /status hot path. - Disk-level cache (24h TTL) at ~/.unsloth/studio/cache/llama_cpp_freshness/ so cold-start Studio launches do not always hit the API. On a transient fetch failure (offline, rate-limited) we keep the last-good disk value alive rather than poisoning the cache with None. The check fails open: if anything is missing (marker, timestamp, GitHub response), stale stays False so users never see a misleading banner. Surfaced in two places 1. Startup banner (logs + stderr) in main.py:lifespan(), alongside the MTP capability probe added in #5528. Single line, e.g.: WARNING: llama.cpp prebuilt is 5 days behind: installed b9190, latest b9300. Run "unsloth studio update" to refresh. 2. /api/inference/status now returns: llama_cpp_prebuilt_stale: bool llama_cpp_installed_tag: str | None llama_cpp_latest_tag: str | None so the frontend can render a banner / popup with the actual tag delta the user is missing. 3-day threshold Mirrors the typical Unsloth llama.cpp release cadence. Anything shorter would nag users who restart Studio at the wrong moment; longer leaves real bugs sitting on the user's machine. Configurable via the threshold_days kwarg if a future call site wants a different window. Tests 17 new cases in tests/test_llama_cpp_freshness.py cover marker discovery in both cmake and root install layouts, missing / invalid marker, GitHub fetch caching across process restarts (disk cache hit after the in-memory cache is reset), the stale / not-stale decision matrix (tag mismatch + age threshold), fail-open behaviour when GitHub is unreachable, custom threshold, singular/plural day in the warning string, and unparseable installed_at_utc. The broader 205-test inference regression suite still passes. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
fc04809bfe
commit
c690b28e99
5 changed files with 626 additions and 8 deletions
|
|
@ -198,27 +198,41 @@ async def lifespan(app: FastAPI):
|
|||
# Detect hardware first — sets DEVICE global used everywhere
|
||||
detect_hardware()
|
||||
|
||||
# llama.cpp capability probe; warns if the prebuilt lacks MTP support.
|
||||
# llama.cpp probes: capability (MTP support) + freshness (release age).
|
||||
# Both cached; freshness has a 24h disk TTL.
|
||||
try:
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
from utils.llama_cpp_freshness import (
|
||||
check_prebuilt_freshness,
|
||||
format_stale_warning,
|
||||
)
|
||||
|
||||
_caps = LlamaCppBackend.probe_server_capabilities()
|
||||
_bin = LlamaCppBackend._find_llama_server_binary()
|
||||
_caps = LlamaCppBackend.probe_server_capabilities(_bin)
|
||||
app.state.llama_cpp_capabilities = _caps
|
||||
if _caps.get("found") and not _caps.get("supports_mtp"):
|
||||
import structlog as _structlog
|
||||
_freshness = check_prebuilt_freshness(_bin)
|
||||
app.state.llama_cpp_freshness = _freshness
|
||||
|
||||
import structlog as _structlog
|
||||
|
||||
_log = _structlog.get_logger(__name__)
|
||||
if _caps.get("found") and not _caps.get("supports_mtp"):
|
||||
_msg = (
|
||||
"llama.cpp prebuilt lacks MTP support "
|
||||
"(--spec-type mtp/draft-mtp). Run `unsloth studio update`. "
|
||||
"MTP GGUFs will load without speculative decoding."
|
||||
)
|
||||
_structlog.get_logger(__name__).warning(_msg)
|
||||
_log.warning(_msg)
|
||||
print(f"WARNING: {_msg}", flush = True)
|
||||
if _freshness.get("stale"):
|
||||
_msg = format_stale_warning(_freshness)
|
||||
_log.warning(_msg)
|
||||
print(f"WARNING: {_msg}", flush = True)
|
||||
except Exception as _probe_exc:
|
||||
import structlog as _structlog
|
||||
|
||||
_structlog.get_logger(__name__).debug(
|
||||
"llama.cpp capability probe failed: %s", _probe_exc
|
||||
"llama.cpp startup probes failed: %s", _probe_exc
|
||||
)
|
||||
|
||||
from storage.studio_db import cleanup_orphaned_runs
|
||||
|
|
|
|||
|
|
@ -349,6 +349,21 @@ class InferenceStatusResponse(BaseModel):
|
|||
"False -> recommend `unsloth studio update`."
|
||||
),
|
||||
)
|
||||
llama_cpp_prebuilt_stale: bool = Field(
|
||||
False,
|
||||
description = (
|
||||
"Installed llama.cpp prebuilt is >=3 days behind the latest "
|
||||
"release. True -> show `unsloth studio update` banner."
|
||||
),
|
||||
)
|
||||
llama_cpp_installed_tag: Optional[str] = Field(
|
||||
None,
|
||||
description = "Installed llama.cpp tag, or None if unknown.",
|
||||
)
|
||||
llama_cpp_latest_tag: Optional[str] = Field(
|
||||
None,
|
||||
description = "Latest published llama.cpp tag, or None if GitHub unreachable.",
|
||||
)
|
||||
|
||||
|
||||
# =====================================================================
|
||||
|
|
|
|||
|
|
@ -1282,12 +1282,23 @@ async def get_status(
|
|||
try:
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
|
||||
# MTP capability probe (cached). Drives the UI update banner.
|
||||
# MTP probe + freshness check (both cached). Drive the UI banner.
|
||||
try:
|
||||
_caps = type(llama_backend).probe_server_capabilities()
|
||||
_bin = type(llama_backend)._find_llama_server_binary()
|
||||
_caps = type(llama_backend).probe_server_capabilities(_bin)
|
||||
_supports_mtp = bool(_caps.get("supports_mtp", False))
|
||||
except Exception:
|
||||
_bin = None
|
||||
_supports_mtp = True # fail open
|
||||
try:
|
||||
from utils.llama_cpp_freshness import check_prebuilt_freshness
|
||||
|
||||
_freshness = check_prebuilt_freshness(_bin)
|
||||
except Exception:
|
||||
_freshness = {}
|
||||
_stale = bool(_freshness.get("stale"))
|
||||
_installed_tag = _freshness.get("installed_tag")
|
||||
_latest_tag = _freshness.get("latest_tag")
|
||||
|
||||
# If a GGUF model is loaded via llama-server, report that
|
||||
if llama_backend.is_loaded:
|
||||
|
|
@ -1332,6 +1343,9 @@ async def get_status(
|
|||
chat_template_override = llama_backend.chat_template_override,
|
||||
speculative_type = llama_backend.speculative_type,
|
||||
llama_cpp_supports_mtp = _supports_mtp,
|
||||
llama_cpp_prebuilt_stale = _stale,
|
||||
llama_cpp_installed_tag = _installed_tag,
|
||||
llama_cpp_latest_tag = _latest_tag,
|
||||
)
|
||||
|
||||
# Otherwise, report Unsloth backend status
|
||||
|
|
@ -1393,6 +1407,9 @@ async def get_status(
|
|||
supports_tools = False,
|
||||
chat_template = chat_template,
|
||||
llama_cpp_supports_mtp = _supports_mtp,
|
||||
llama_cpp_prebuilt_stale = _stale,
|
||||
llama_cpp_installed_tag = _installed_tag,
|
||||
llama_cpp_latest_tag = _latest_tag,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
328
studio/backend/tests/test_llama_cpp_freshness.py
Normal file
328
studio/backend/tests/test_llama_cpp_freshness.py
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Tests for the llama.cpp prebuilt freshness check.
|
||||
|
||||
Pins the marker parser, the disk+memory cache, the stale decision
|
||||
matrix, and fail-open behaviour on missing data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import types as _types
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
|
||||
if _BACKEND_DIR not in sys.path:
|
||||
sys.path.insert(0, _BACKEND_DIR)
|
||||
|
||||
_loggers_stub = _types.ModuleType("loggers")
|
||||
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
|
||||
sys.modules.setdefault("loggers", _loggers_stub)
|
||||
|
||||
_structlog_stub = _types.ModuleType("structlog")
|
||||
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
|
||||
sys.modules.setdefault("structlog", _structlog_stub)
|
||||
|
||||
import pytest
|
||||
|
||||
from utils import llama_cpp_freshness as fr
|
||||
|
||||
|
||||
# Helpers.
|
||||
|
||||
|
||||
def _write_marker(install_dir: Path, **overrides) -> Path:
|
||||
payload = {
|
||||
"requested_tag": "latest",
|
||||
"tag": "b9190",
|
||||
"release_tag": "b9190",
|
||||
"published_repo": "unslothai/llama.cpp",
|
||||
"asset": "app-b9190-linux-x64-cuda13-newer.tar.gz",
|
||||
"asset_sha256": None,
|
||||
"source": "published",
|
||||
"installed_at_utc": (datetime.now(tz = timezone.utc) - timedelta(days = 1))
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
}
|
||||
payload.update(overrides)
|
||||
install_dir.mkdir(parents = True, exist_ok = True)
|
||||
(install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(payload))
|
||||
return install_dir / "UNSLOTH_PREBUILT_INFO.json"
|
||||
|
||||
|
||||
def _fake_binary(install_dir: Path, *, layout: str = "cmake") -> Path:
|
||||
"""Stub llama-server under one of the supported install layouts."""
|
||||
if layout == "cmake":
|
||||
bin_dir = install_dir / "build" / "bin"
|
||||
bin_name = "llama-server"
|
||||
elif layout == "root":
|
||||
bin_dir = install_dir
|
||||
bin_name = "llama-server"
|
||||
elif layout == "windows":
|
||||
bin_dir = install_dir / "build" / "bin" / "Release"
|
||||
bin_name = "llama-server.exe"
|
||||
else:
|
||||
raise ValueError(f"unknown layout {layout}")
|
||||
bin_dir.mkdir(parents = True, exist_ok = True)
|
||||
bin_path = bin_dir / bin_name
|
||||
bin_path.write_text("stub\n")
|
||||
return bin_path
|
||||
|
||||
|
||||
@pytest.fixture(autouse = True)
|
||||
def _reset(monkeypatch, tmp_path):
|
||||
# Isolate disk cache per-test; never touch the user's real cache.
|
||||
monkeypatch.setattr(fr, "_cache_dir", lambda: tmp_path / ".freshness")
|
||||
fr.reset_caches()
|
||||
yield
|
||||
fr.reset_caches()
|
||||
|
||||
|
||||
# read_install_marker.
|
||||
|
||||
|
||||
def test_read_install_marker_finds_cmake_layout(tmp_path):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(install_dir, tag = "b9190")
|
||||
bin_path = _fake_binary(install_dir, layout = "cmake")
|
||||
marker = fr.read_install_marker(str(bin_path))
|
||||
assert marker is not None
|
||||
assert marker["tag"] == "b9190"
|
||||
assert marker["published_repo"] == "unslothai/llama.cpp"
|
||||
|
||||
|
||||
def test_read_install_marker_finds_root_layout(tmp_path):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(install_dir, tag = "b9999")
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
marker = fr.read_install_marker(str(bin_path))
|
||||
assert marker is not None
|
||||
assert marker["tag"] == "b9999"
|
||||
|
||||
|
||||
def test_read_install_marker_finds_windows_cmake_layout(tmp_path):
|
||||
# Windows cmake puts the .exe under build/bin/Release/, so the
|
||||
# marker is four levels above the binary.
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(install_dir, tag = "b8888")
|
||||
bin_path = _fake_binary(install_dir, layout = "windows")
|
||||
marker = fr.read_install_marker(str(bin_path))
|
||||
assert marker is not None
|
||||
assert marker["tag"] == "b8888"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("repo", ["unslothai/llama.cpp", "ggml-org/llama.cpp"])
|
||||
def test_read_install_marker_carries_published_repo_dynamically(tmp_path, repo):
|
||||
# The freshness check queries whichever release repo the marker
|
||||
# records, so CUDA Linux (unslothai), CPU Linux x86_64 / macOS
|
||||
# (ggml-org), and ROCm source-build (unslothai upstream label)
|
||||
# all surface the right "latest" tag.
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(install_dir, tag = "b9000", published_repo = repo)
|
||||
bin_path = _fake_binary(install_dir, layout = "cmake")
|
||||
marker = fr.read_install_marker(str(bin_path))
|
||||
assert marker is not None
|
||||
assert marker["published_repo"] == repo
|
||||
|
||||
|
||||
def test_read_install_marker_missing_returns_none(tmp_path):
|
||||
bin_path = _fake_binary(tmp_path / "no_marker", layout = "root")
|
||||
assert fr.read_install_marker(str(bin_path)) is None
|
||||
|
||||
|
||||
def test_read_install_marker_handles_invalid_json(tmp_path):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
install_dir.mkdir(parents = True)
|
||||
(install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text("not json")
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
assert fr.read_install_marker(str(bin_path)) is None
|
||||
|
||||
|
||||
def test_read_install_marker_handles_none_path():
|
||||
assert fr.read_install_marker(None) is None
|
||||
|
||||
|
||||
# latest_published_release (with monkeypatched fetcher).
|
||||
|
||||
|
||||
def test_latest_published_release_uses_disk_cache(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def _fake_fetch(repo, timeout = 5.0):
|
||||
calls.append(repo)
|
||||
return "b9999"
|
||||
|
||||
monkeypatch.setattr(fr, "_fetch_latest_release_tag", _fake_fetch)
|
||||
first = fr.latest_published_release("unslothai/llama.cpp")
|
||||
second = fr.latest_published_release("unslothai/llama.cpp")
|
||||
assert first == "b9999"
|
||||
assert second == "b9999"
|
||||
# Memo + disk cache -> only one fetch.
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_latest_published_release_returns_none_on_network_failure(monkeypatch):
|
||||
monkeypatch.setattr(fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None)
|
||||
assert fr.latest_published_release("unslothai/llama.cpp") is None
|
||||
|
||||
|
||||
def test_latest_published_release_keeps_old_cache_on_transient_failure(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
# Disk entry older than TTL + network fail -> return cached value.
|
||||
cache_dir = tmp_path / ".freshness"
|
||||
cache_dir.mkdir()
|
||||
cache_file = cache_dir / "unslothai__llama.cpp.json"
|
||||
yesterday = time.time() - 25 * 60 * 60 # > 24h
|
||||
cache_file.write_text(json.dumps({"fetched_at": yesterday, "latest_tag": "b9000"}))
|
||||
monkeypatch.setattr(fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None)
|
||||
assert fr.latest_published_release("unslothai/llama.cpp") == "b9000"
|
||||
|
||||
|
||||
# check_prebuilt_freshness end-to-end.
|
||||
|
||||
|
||||
def test_check_prebuilt_freshness_reports_stale_when_old_and_behind(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(
|
||||
install_dir,
|
||||
tag = "b9190",
|
||||
installed_at_utc = (datetime.now(tz = timezone.utc) - timedelta(days = 5))
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
)
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
monkeypatch.setattr(
|
||||
fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9300"
|
||||
)
|
||||
info = fr.check_prebuilt_freshness(str(bin_path))
|
||||
assert info["has_marker"] is True
|
||||
assert info["stale"] is True
|
||||
assert info["installed_tag"] == "b9190"
|
||||
assert info["latest_tag"] == "b9300"
|
||||
assert info["age_days"] == 5
|
||||
assert info["published_repo"] == "unslothai/llama.cpp"
|
||||
|
||||
|
||||
def test_check_prebuilt_freshness_not_stale_when_tag_matches(monkeypatch, tmp_path):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(
|
||||
install_dir,
|
||||
tag = "b9300",
|
||||
installed_at_utc = (datetime.now(tz = timezone.utc) - timedelta(days = 30))
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
)
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
monkeypatch.setattr(
|
||||
fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9300"
|
||||
)
|
||||
info = fr.check_prebuilt_freshness(str(bin_path))
|
||||
assert info["stale"] is False
|
||||
assert info["installed_tag"] == "b9300"
|
||||
assert info["latest_tag"] == "b9300"
|
||||
|
||||
|
||||
def test_check_prebuilt_freshness_not_stale_within_threshold(monkeypatch, tmp_path):
|
||||
# Behind by tag but within the 3-day grace window.
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(
|
||||
install_dir,
|
||||
tag = "b9190",
|
||||
installed_at_utc = (datetime.now(tz = timezone.utc) - timedelta(days = 1))
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
)
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
monkeypatch.setattr(
|
||||
fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9300"
|
||||
)
|
||||
info = fr.check_prebuilt_freshness(str(bin_path))
|
||||
assert info["stale"] is False
|
||||
assert info["age_days"] == 1
|
||||
|
||||
|
||||
def test_check_prebuilt_freshness_fails_open_without_marker(tmp_path):
|
||||
bin_path = _fake_binary(tmp_path / "custom_build", layout = "root")
|
||||
info = fr.check_prebuilt_freshness(str(bin_path))
|
||||
assert info["has_marker"] is False
|
||||
assert info["stale"] is False
|
||||
|
||||
|
||||
def test_check_prebuilt_freshness_fails_open_when_github_unreachable(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(
|
||||
install_dir,
|
||||
tag = "b9190",
|
||||
installed_at_utc = (datetime.now(tz = timezone.utc) - timedelta(days = 10))
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
)
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
monkeypatch.setattr(fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None)
|
||||
info = fr.check_prebuilt_freshness(str(bin_path))
|
||||
assert info["has_marker"] is True
|
||||
assert info["stale"] is False
|
||||
assert info["latest_tag"] is None
|
||||
|
||||
|
||||
def test_check_prebuilt_freshness_handles_unparseable_install_timestamp(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(install_dir, tag = "b9190", installed_at_utc = "not-a-date")
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
monkeypatch.setattr(
|
||||
fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9300"
|
||||
)
|
||||
info = fr.check_prebuilt_freshness(str(bin_path))
|
||||
assert info["stale"] is False
|
||||
assert info["age_days"] is None
|
||||
|
||||
|
||||
def test_check_prebuilt_freshness_respects_custom_threshold(monkeypatch, tmp_path):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
_write_marker(
|
||||
install_dir,
|
||||
tag = "b9190",
|
||||
installed_at_utc = (datetime.now(tz = timezone.utc) - timedelta(days = 2))
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
)
|
||||
bin_path = _fake_binary(install_dir, layout = "root")
|
||||
monkeypatch.setattr(
|
||||
fr, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9300"
|
||||
)
|
||||
info = fr.check_prebuilt_freshness(str(bin_path), threshold_days = 1)
|
||||
assert info["stale"] is True
|
||||
|
||||
|
||||
# format_stale_warning.
|
||||
|
||||
|
||||
def test_format_stale_warning_contains_actionable_command():
|
||||
msg = fr.format_stale_warning(
|
||||
{"installed_tag": "b9190", "latest_tag": "b9300", "age_days": 5}
|
||||
)
|
||||
assert "b9190" in msg
|
||||
assert "b9300" in msg
|
||||
assert "5 days" in msg
|
||||
assert "unsloth studio update" in msg
|
||||
|
||||
|
||||
def test_format_stale_warning_singular_day():
|
||||
msg = fr.format_stale_warning(
|
||||
{"installed_tag": "b9190", "latest_tag": "b9300", "age_days": 1}
|
||||
)
|
||||
assert "1 day" in msg
|
||||
assert "1 days" not in msg
|
||||
244
studio/backend/utils/llama_cpp_freshness.py
Normal file
244
studio/backend/utils/llama_cpp_freshness.py
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""llama.cpp prebuilt freshness check.
|
||||
|
||||
Reads UNSLOTH_PREBUILT_INFO.json (written by install_llama_prebuilt.py)
|
||||
and compares the installed release tag against the latest on GitHub.
|
||||
Surfaced via main.py:lifespan() and /api/inference/status. Fails open
|
||||
on any missing data so we never show a misleading banner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import structlog
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# 3 days matches Unsloth's typical llama.cpp release cadence.
|
||||
STALENESS_THRESHOLD_DAYS = 3
|
||||
|
||||
# 24h TTL keeps the GitHub call off the hot path and within rate limits.
|
||||
_RELEASE_CACHE_TTL_SECONDS = 24 * 60 * 60
|
||||
|
||||
_INSTALL_MARKER_NAME = "UNSLOTH_PREBUILT_INFO.json"
|
||||
|
||||
_marker_cache: dict[str, Optional[dict]] = {}
|
||||
_release_memo: dict[str, tuple[float, Optional[str]]] = {}
|
||||
|
||||
|
||||
def _cache_dir() -> Path:
|
||||
"""Lazy import so tests can stub storage_roots."""
|
||||
try:
|
||||
from utils.paths.storage_roots import cache_root
|
||||
|
||||
return cache_root() / "llama_cpp_freshness"
|
||||
except Exception:
|
||||
return Path.home() / ".unsloth" / "studio" / "cache" / "llama_cpp_freshness"
|
||||
|
||||
|
||||
def read_install_marker(binary_path: Optional[str]) -> Optional[dict]:
|
||||
"""Walk up from binary_path to find UNSLOTH_PREBUILT_INFO.json.
|
||||
None means no marker (source build / custom path) or invalid JSON."""
|
||||
if not binary_path:
|
||||
return None
|
||||
cached = _marker_cache.get(binary_path)
|
||||
if cached is not None or binary_path in _marker_cache:
|
||||
return cached
|
||||
p = Path(binary_path)
|
||||
marker: Optional[dict] = None
|
||||
# Cover all _find_llama_server_binary layouts:
|
||||
# <install>/llama-server (1 up)
|
||||
# <install>/build/bin/llama-server (3 up, Linux/macOS cmake)
|
||||
# <install>/build/bin/Release/llama-server.exe (4 up, Windows cmake)
|
||||
for parent in p.parents[:5]:
|
||||
candidate = parent / _INSTALL_MARKER_NAME
|
||||
if candidate.is_file():
|
||||
try:
|
||||
marker = json.loads(candidate.read_text(encoding = "utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.debug(
|
||||
"failed to parse install marker",
|
||||
path = str(candidate),
|
||||
error = str(exc),
|
||||
)
|
||||
marker = None
|
||||
break
|
||||
_marker_cache[binary_path] = marker
|
||||
return marker
|
||||
|
||||
|
||||
def _cache_path_for(repo: str) -> Path:
|
||||
safe = repo.replace("/", "__")
|
||||
return _cache_dir() / f"{safe}.json"
|
||||
|
||||
|
||||
def _load_disk_cache(repo: str) -> Optional[tuple[float, Optional[str]]]:
|
||||
path = _cache_path_for(repo)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding = "utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
ts = payload.get("fetched_at")
|
||||
tag = payload.get("latest_tag")
|
||||
if not isinstance(ts, (int, float)):
|
||||
return None
|
||||
return float(ts), tag if isinstance(tag, str) else None
|
||||
|
||||
|
||||
def _save_disk_cache(repo: str, latest_tag: Optional[str]) -> None:
|
||||
path = _cache_path_for(repo)
|
||||
try:
|
||||
path.parent.mkdir(parents = True, exist_ok = True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(
|
||||
json.dumps({"fetched_at": time.time(), "latest_tag": latest_tag}),
|
||||
encoding = "utf-8",
|
||||
)
|
||||
tmp.replace(path)
|
||||
except OSError as exc:
|
||||
logger.debug("freshness cache write failed", repo = repo, error = str(exc))
|
||||
|
||||
|
||||
def _fetch_latest_release_tag(repo: str, timeout: float = 5.0) -> Optional[str]:
|
||||
"""GitHub API call. None on any failure (offline, rate-limited, etc)."""
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
url = f"https://api.github.com/repos/{repo}/releases/latest"
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"User-Agent": "unsloth-studio-freshness-check",
|
||||
}
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
req = urllib.request.Request(url, headers = headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
except (
|
||||
urllib.error.URLError,
|
||||
urllib.error.HTTPError,
|
||||
OSError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
logger.debug("freshness fetch failed", repo = repo, error = str(exc))
|
||||
return None
|
||||
tag = data.get("tag_name")
|
||||
return tag if isinstance(tag, str) and tag else None
|
||||
|
||||
|
||||
def latest_published_release(
|
||||
repo: str, *, force_refresh: bool = False
|
||||
) -> Optional[str]:
|
||||
"""Latest release tag for `repo`. Memo + disk-cached (24h TTL).
|
||||
None when offline and never previously cached."""
|
||||
if not repo:
|
||||
return None
|
||||
now = time.time()
|
||||
if not force_refresh:
|
||||
memo = _release_memo.get(repo)
|
||||
if memo and now - memo[0] < _RELEASE_CACHE_TTL_SECONDS:
|
||||
return memo[1]
|
||||
disk = _load_disk_cache(repo)
|
||||
if disk and now - disk[0] < _RELEASE_CACHE_TTL_SECONDS:
|
||||
_release_memo[repo] = disk
|
||||
return disk[1]
|
||||
latest = _fetch_latest_release_tag(repo)
|
||||
if latest is None:
|
||||
# Keep last-good disk value rather than poisoning with None.
|
||||
disk = _load_disk_cache(repo)
|
||||
if disk:
|
||||
_release_memo[repo] = disk
|
||||
return disk[1]
|
||||
return None
|
||||
_release_memo[repo] = (now, latest)
|
||||
_save_disk_cache(repo, latest)
|
||||
return latest
|
||||
|
||||
|
||||
def _parse_installed_at(value: object) -> Optional[datetime]:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
s = value.replace("Z", "+00:00") if value.endswith("Z") else value
|
||||
try:
|
||||
dt = datetime.fromisoformat(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo = timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def check_prebuilt_freshness(
|
||||
binary_path: Optional[str],
|
||||
*,
|
||||
threshold_days: int = STALENESS_THRESHOLD_DAYS,
|
||||
now: Optional[datetime] = None,
|
||||
) -> dict:
|
||||
"""Returns {has_marker, stale, installed_tag, latest_tag,
|
||||
installed_at_utc, age_days, published_repo, threshold_days}.
|
||||
stale = True iff installed != latest AND age >= threshold.
|
||||
Fails open on missing data (stale stays False)."""
|
||||
out: dict = {
|
||||
"has_marker": False,
|
||||
"stale": False,
|
||||
"installed_tag": None,
|
||||
"latest_tag": None,
|
||||
"installed_at_utc": None,
|
||||
"age_days": None,
|
||||
"published_repo": None,
|
||||
"threshold_days": int(threshold_days),
|
||||
}
|
||||
marker = read_install_marker(binary_path)
|
||||
if not marker:
|
||||
return out
|
||||
out["has_marker"] = True
|
||||
out["installed_tag"] = marker.get("tag") or marker.get("release_tag")
|
||||
out["installed_at_utc"] = marker.get("installed_at_utc")
|
||||
out["published_repo"] = marker.get("published_repo")
|
||||
|
||||
repo = out["published_repo"]
|
||||
if not repo or not out["installed_tag"]:
|
||||
return out
|
||||
latest = latest_published_release(repo)
|
||||
out["latest_tag"] = latest
|
||||
if not latest or latest == out["installed_tag"]:
|
||||
return out
|
||||
|
||||
installed_at = _parse_installed_at(out["installed_at_utc"])
|
||||
if installed_at is None:
|
||||
return out
|
||||
now = now or datetime.now(tz = timezone.utc)
|
||||
age_seconds = (now - installed_at).total_seconds()
|
||||
out["age_days"] = max(0, int(age_seconds // 86400))
|
||||
if age_seconds >= threshold_days * 86400:
|
||||
out["stale"] = True
|
||||
return out
|
||||
|
||||
|
||||
def format_stale_warning(info: dict) -> str:
|
||||
"""Human-readable one-liner for stale prebuilt info."""
|
||||
age = info.get("age_days")
|
||||
installed = info.get("installed_tag") or "unknown"
|
||||
latest = info.get("latest_tag") or "unknown"
|
||||
age_str = f"{age} day{'s' if age != 1 else ''}" if age is not None else "some time"
|
||||
return (
|
||||
f"llama.cpp prebuilt is {age_str} behind: installed "
|
||||
f"{installed}, latest {latest}. Run `unsloth studio update` "
|
||||
f"to refresh."
|
||||
)
|
||||
|
||||
|
||||
def reset_caches() -> None:
|
||||
"""Test-only: drop all in-memory caches."""
|
||||
_marker_cache.clear()
|
||||
_release_memo.clear()
|
||||
Loading…
Add table
Add a link
Reference in a new issue