80c15643cf
A loaded GGUF model stayed resident until a manual unload or process exit, with no way to set an idle timeout. Add an idle TTL (seconds): when set, a background task started by the app lifespan unloads the model after it has been idle that long. 0 disables eviction, preserving the historical behavior. - utils/model_ttl_settings.py: the setting (app_settings key), the UNSLOTH_MODEL_IDLE_TTL env default, and validation (0 to one week). - LlamaCppBackend tracks last-activity: refreshed on load, on every generation request entry, and on every streamed chunk, so a long generation (e.g. a minutes-long reasoning stream) is never evicted mid-flight; idle_seconds is None when nothing is loaded. - main.py runs the eviction loop in the lifespan and cancels it on shutdown. - GET/PUT /api/settings/model-ttl to read/set the TTL at runtime; the response also reports the loaded model's current idle and time-to-eviction. The request-path activity ping goes through a small _note_idle_activity() guard so a backend that does not implement the hook degrades gracefully instead of turning a served request into a 500. Eviction is made race-safe. unload_model sets _cancel_event, which an in-flight load of a different model watches, so a naive evict could abort that load. The evictor now goes through LlamaCppBackend.evict_if_idle, which re-checks idle and in-flight under _serial_load_lock (the lock load_model holds for its whole duration) and unloads atomically, so it can never run concurrently with a load or unload. An in-flight request counter (request_in_flight, applied to the chat and tool generators) additionally blocks eviction while a request is active, covering the window before the first streamed token when per-chunk activity has not started refreshing yet.
71 lines
2.5 KiB
Python
71 lines
2.5 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Idle TTL for a loaded inference model.
|
|
|
|
A loaded GGUF model otherwise stays resident until a manual unload or process
|
|
exit. When the idle TTL is set (> 0), a background task unloads the model after
|
|
it has been idle (no generation activity) for that many seconds. 0 disables
|
|
eviction, preserving the historical behavior. The value can be set at startup
|
|
via the ``UNSLOTH_MODEL_IDLE_TTL`` env var or at runtime via
|
|
``PUT /api/settings/model-ttl``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Any
|
|
|
|
MODEL_IDLE_TTL_SETTING_KEY = "model_idle_ttl_seconds"
|
|
DEFAULT_MODEL_IDLE_TTL_SECONDS = 0 # 0 = disabled (never auto-evict)
|
|
MIN_MODEL_IDLE_TTL_SECONDS = 0
|
|
MAX_MODEL_IDLE_TTL_SECONDS = 7 * 24 * 60 * 60 # one week ceiling
|
|
MODEL_IDLE_EVICTION_POLL_SECONDS = 30.0
|
|
_ENV_VAR = "UNSLOTH_MODEL_IDLE_TTL"
|
|
|
|
|
|
def _coerce_ttl_seconds(value: Any) -> int | None:
|
|
if isinstance(value, bool):
|
|
return None
|
|
try:
|
|
parsed = int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if parsed < MIN_MODEL_IDLE_TTL_SECONDS or parsed > MAX_MODEL_IDLE_TTL_SECONDS:
|
|
return None
|
|
return parsed
|
|
|
|
|
|
def default_model_idle_ttl_seconds() -> int:
|
|
"""The startup default: the env override when valid, else disabled (0)."""
|
|
env_value = _coerce_ttl_seconds(os.environ.get(_ENV_VAR))
|
|
return env_value if env_value is not None else DEFAULT_MODEL_IDLE_TTL_SECONDS
|
|
|
|
|
|
def validate_model_idle_ttl_seconds(value: Any) -> int:
|
|
parsed = _coerce_ttl_seconds(value)
|
|
if parsed is None:
|
|
raise ValueError(
|
|
"Model idle TTL must be a whole number of seconds from "
|
|
f"{MIN_MODEL_IDLE_TTL_SECONDS} to {MAX_MODEL_IDLE_TTL_SECONDS} (0 disables it)."
|
|
)
|
|
return parsed
|
|
|
|
|
|
def get_model_idle_ttl_seconds() -> int:
|
|
"""Effective TTL: the stored setting when valid, else the startup default."""
|
|
try:
|
|
from storage.studio_db import get_app_setting
|
|
stored = get_app_setting(MODEL_IDLE_TTL_SETTING_KEY, None)
|
|
except Exception:
|
|
stored = None
|
|
parsed = _coerce_ttl_seconds(stored)
|
|
return parsed if parsed is not None else default_model_idle_ttl_seconds()
|
|
|
|
|
|
def set_model_idle_ttl_seconds(value: Any) -> int:
|
|
parsed = validate_model_idle_ttl_seconds(value)
|
|
from storage.studio_db import upsert_app_settings
|
|
|
|
upsert_app_settings({MODEL_IDLE_TTL_SETTING_KEY: parsed})
|
|
return parsed
|