ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: add local render_html tool support

Browse source
This commit is contained in:
wasimysaid 2026-05-25 18:04:21 +02:00
commit 07972fd515
5 changed files with 1353 additions and 1297 deletions
Download patch file Download diff file Expand all files Collapse all files

327
studio/backend/core/inference/llama_cpp.py
View file

@ -128,7 +128,7 @@ def _probe_dns_dead(host: str = "huggingface.co", timeout: float = 2.0) -> bool:
except Exception:
result[0] = True
t = threading.Thread(target = _probe, daemon = True)
t = threading.Thread(target=_probe, daemon=True)
t.start()
t.join(timeout)
# Thread still running -> resolver wedged -> treat as dead.
@ -185,10 +185,10 @@ def _load_swa_cache() -> dict:
def _save_swa_cache(cache: dict) -> None:
try:
path = _swa_cache_path()
path.parent.mkdir(parents = True, exist_ok = True)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".json.tmp")
with open(tmp, "w") as f:
json.dump(cache, f, indent = 2, sort_keys = True)
json.dump(cache, f, indent=2, sort_keys=True)
tmp.replace(path)
except OSError:
pass
@ -211,7 +211,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]:
try:
from huggingface_hub import hf_hub_download
cfg_path = hf_hub_download(repo_id, "config.json", repo_type = "model")
cfg_path = hf_hub_download(repo_id, "config.json", repo_type="model")
with open(cfg_path) as f:
cfg = json.load(f)
except Exception:
@ -835,7 +835,7 @@ class LlamaCppBackend:
# Read VmRSS from /proc/<pid>/status. Kilobytes on Linux.
bytes_loaded = 0
try:
with open(f"/proc/{pid}/status", "r", encoding = "utf-8") as f:
with open(f"/proc/{pid}/status", "r", encoding="utf-8") as f:
for line in f:
if line.startswith("VmRSS:"):
kb = int(line.split()[1])
@ -1105,10 +1105,10 @@ class LlamaCppBackend:
try:
result = subprocess.run(
[bin_path, "--help"],
capture_output = True,
text = True,
timeout = 10,
check = False,
capture_output=True,
text=True,
timeout=10,
check=False,
)
help_text = (result.stdout or "") + "\n" + (result.stderr or "")
# Split into per-flag blocks: each --flag line plus its
@ -1257,10 +1257,10 @@ class LlamaCppBackend:
"--query-gpu=index,memory.free",
"--format=csv,noheader,nounits",
],
capture_output = True,
text = True,
timeout = 10,
env = child_env_without_native_path_secret(),
capture_output=True,
text=True,
timeout=10,
env=child_env_without_native_path_secret(),
**_windows_hidden_subprocess_kwargs(),
)
if result.returncode == 0:
@ -1290,7 +1290,7 @@ class LlamaCppBackend:
# Match the docstring's sort-by-id guarantee. nvidia-smi
# almost always returns sorted output, but driver order
# is not formally guaranteed.
gpus.sort(key = lambda g: g[0])
gpus.sort(key=lambda g: g[0])
if gpus:
return gpus
except Exception as e:
@ -1349,7 +1349,7 @@ class LlamaCppBackend:
)
gpus.append((idx, free_bytes // (1024 * 1024)))
# Match the nvidia-smi path's docstring guarantee of sorted-by-id.
return sorted(gpus, key = lambda g: g[0])
return sorted(gpus, key=lambda g: g[0])
except Exception as e:
logger.debug(f"torch GPU probe failed: {e}")
return []
@ -1534,7 +1534,7 @@ class LlamaCppBackend:
usable_fraction = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
# Sort GPUs by free memory descending
ranked = sorted(gpus, key = lambda g: g[1], reverse = True)
ranked = sorted(gpus, key=lambda g: g[1], reverse=True)
# Try fitting on 1 GPU at the usable-VRAM threshold.
if ranked[0][1] * usable_fraction >= model_size_mib:
@ -1552,8 +1552,8 @@ class LlamaCppBackend:
# Model is too large even for all GPUs, let --fit handle it
logger.debug(
"Model does not fit in available GPU memory, falling back to --fit",
model_size_mib = round(model_size_mib, 2),
ranked_gpus = ranked,
model_size_mib=round(model_size_mib, 2),
ranked_gpus=ranked,
)
return None, True
@ -1799,8 +1799,8 @@ class LlamaCppBackend:
if not self._can_estimate_kv():
logger.debug(
"Skipping context fit because KV cache metadata is unavailable",
requested_ctx = requested_ctx,
available_mib = available_mib,
requested_ctx=requested_ctx,
available_mib=available_mib,
)
return requested_ctx
@ -1809,10 +1809,10 @@ class LlamaCppBackend:
return requested_ctx
kv_kwargs = dict(
swa_full = swa_full,
n_parallel = n_parallel,
kv_unified = kv_unified,
ctx_checkpoints = ctx_checkpoints,
swa_full=swa_full,
n_parallel=n_parallel,
kv_unified=kv_unified,
ctx_checkpoints=ctx_checkpoints,
)
# MTP needs a tighter budget; drop from 0.90 to 0.85.
@ -1830,9 +1830,9 @@ class LlamaCppBackend:
if model_footprint >= budget_bytes:
logger.debug(
"Model footprint exceeds GPU budget before KV cache",
requested_ctx = requested_ctx,
available_mib = available_mib,
model_size_gb = round(model_footprint / (1024**3), 2),
requested_ctx=requested_ctx,
available_mib=available_mib,
model_size_gb=round(model_footprint / (1024**3), 2),
)
return requested_ctx
@ -1874,7 +1874,7 @@ class LlamaCppBackend:
try:
from huggingface_hub import get_paths_info, list_repo_files
files = list_repo_files(hf_repo, token = hf_token)
files = list_repo_files(hf_repo, token=hf_token)
gguf_files = [
f for f in files if f.endswith(".gguf") and "mmproj" not in f.lower()
]
@ -1882,7 +1882,7 @@ class LlamaCppBackend:
return None
# Get sizes for all GGUF files
path_infos = list(get_paths_info(hf_repo, gguf_files, token = hf_token))
path_infos = list(get_paths_info(hf_repo, gguf_files, token=hf_token))
size_map = {p.path: (p.size or 0) for p in path_infos}
# Group files by variant: shards share a prefix before -NNNNN-of-NNNNN
@ -1900,7 +1900,7 @@ class LlamaCppBackend:
variant_sizes.append((first, total, shard_files))
# Sort by total size ascending and pick the smallest that fits
variant_sizes.sort(key = lambda x: x[1])
variant_sizes.sort(key=lambda x: x[1])
for first_file, total_size, _ in variant_sizes:
if total_size > 0 and total_size <= free_bytes:
return first_file, total_size
@ -2215,7 +2215,7 @@ class LlamaCppBackend:
flags = detect_reasoning_flags(
self._chat_template,
self._model_identifier,
log_source = "GGUF metadata",
log_source="GGUF metadata",
)
self._supports_reasoning = flags["supports_reasoning"]
self._reasoning_style = flags["reasoning_style"]
@ -2255,7 +2255,7 @@ class LlamaCppBackend:
try:
from huggingface_hub import list_repo_files
files = list_repo_files(hf_repo, token = hf_token)
files = list_repo_files(hf_repo, token=hf_token)
variant_lower = hf_variant.lower()
boundary = re.compile(
r"(?<![a-zA-Z0-9])" + re.escape(variant_lower) + r"(?![a-zA-Z0-9])"
@ -2342,7 +2342,7 @@ class LlamaCppBackend:
try:
from huggingface_hub import get_paths_info, try_to_load_from_cache
path_infos = list(get_paths_info(hf_repo, all_gguf_files, token = hf_token))
path_infos = list(get_paths_info(hf_repo, all_gguf_files, token=hf_token))
total_bytes = sum((p.size or 0) for p in path_infos)
# Subtract bytes already present in the HF cache so we only
@ -2374,7 +2374,7 @@ class LlamaCppBackend:
"HF_HUB_CACHE",
str(Path.home() / ".cache" / "huggingface" / "hub"),
)
Path(cache_dir).mkdir(parents = True, exist_ok = True)
Path(cache_dir).mkdir(parents=True, exist_ok=True)
free_bytes = shutil.disk_usage(cache_dir).free
total_gb = total_download_bytes / (1024**3)
@ -2431,18 +2431,18 @@ class LlamaCppBackend:
raise RuntimeError("Cancelled")
dl_start = time.monotonic()
local_path = hf_hub_download(
repo_id = hf_repo,
filename = gguf_filename,
token = hf_token,
repo_id=hf_repo,
filename=gguf_filename,
token=hf_token,
)
for shard in gguf_extra_shards:
if self._cancel_event.is_set():
raise RuntimeError("Cancelled")
logger.info(f"Resolving GGUF shard: {shard}")
hf_hub_download(
repo_id = hf_repo,
filename = shard,
token = hf_token,
repo_id=hf_repo,
filename=shard,
token=hf_token,
)
except RuntimeError as e:
if "Cancelled" in str(e):
@ -2491,7 +2491,7 @@ class LlamaCppBackend:
try:
from huggingface_hub import list_repo_files
target = _pick_mmproj(list_repo_files(hf_repo, token = hf_token))
target = _pick_mmproj(list_repo_files(hf_repo, token=hf_token))
except Exception as e:
logger.debug(f"Could not list repo files for mmproj: {e}")
@ -2520,9 +2520,9 @@ class LlamaCppBackend:
logger.info(f"Downloading mmproj: {hf_repo}/{target}")
local_path = hf_hub_download(
repo_id = hf_repo,
filename = target,
token = hf_token,
repo_id=hf_repo,
filename=target,
token=hf_token,
)
return local_path
except Exception as e:
@ -2604,16 +2604,16 @@ class LlamaCppBackend:
# (the first one hadn't published _healthy=True yet). If the
# live server already satisfies this request, do nothing.
if self._already_in_target_state(
gguf_path = gguf_path,
model_identifier = model_identifier,
hf_variant = hf_variant,
n_ctx = n_ctx,
cache_type_kv = cache_type_kv,
speculative_type = speculative_type,
spec_draft_n_max = spec_draft_n_max,
chat_template_override = chat_template_override,
extra_args = extra_args,
is_vision = is_vision,
gguf_path=gguf_path,
model_identifier=model_identifier,
hf_variant=hf_variant,
n_ctx=n_ctx,
cache_type_kv=cache_type_kv,
speculative_type=speculative_type,
spec_draft_n_max=spec_draft_n_max,
chat_template_override=chat_template_override,
extra_args=extra_args,
is_vision=is_vision,
):
logger.info(
f"load_model: backend already in target state for "
@ -2676,15 +2676,15 @@ class LlamaCppBackend:
if hf_repo:
with _hf_offline_if_dns_dead():
model_path = self._download_gguf(
hf_repo = hf_repo,
hf_variant = hf_variant,
hf_token = hf_token,
hf_repo=hf_repo,
hf_variant=hf_variant,
hf_token=hf_token,
)
# Auto-download mmproj for vision models
if is_vision and not mmproj_path:
mmproj_path = self._download_mmproj(
hf_repo = hf_repo,
hf_token = hf_token,
hf_repo=hf_repo,
hf_token=hf_token,
)
elif gguf_path:
if not Path(gguf_path).is_file():
@ -2708,7 +2708,7 @@ class LlamaCppBackend:
# not blocked. ``unload_model`` also records the kill, so
# the frontend /unload+/load Apply path engages the wait
# here even though no in-process kill happened.
self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)
self._wait_for_vram_settle(since_kill=self._last_kill_monotonic)
# ── Phase 3: start llama-server (under lock) ──────────────
with self._lock:
@ -2795,7 +2795,7 @@ class LlamaCppBackend:
native_ctx_for_cap = self._context_length or effective_ctx
if native_ctx_for_cap > 0:
ranked_for_cap = sorted(
gpus, key = lambda g: g[1], reverse = True
gpus, key=lambda g: g[1], reverse=True
)
best_cap = 0
for n_gpus in range(1, len(ranked_for_cap) + 1):
@ -2806,11 +2806,11 @@ class LlamaCppBackend:
pool_mib,
model_size,
cache_type_kv,
n_parallel = n_parallel,
mtp_engaged = _mtp_will_engage,
n_parallel=n_parallel,
mtp_engaged=_mtp_will_engage,
)
kv = self._estimate_kv_cache_bytes(
capped, cache_type_kv, n_parallel = n_parallel
capped, cache_type_kv, n_parallel=n_parallel
)
total_mib = (model_size + kv) / (1024 * 1024)
if total_mib <= pool_mib * 0.90:
@ -2837,7 +2837,7 @@ class LlamaCppBackend:
requested_total = (
model_size
+ self._estimate_kv_cache_bytes(
effective_ctx, cache_type_kv, n_parallel = n_parallel
effective_ctx, cache_type_kv, n_parallel=n_parallel
)
)
gpu_indices, use_fit = self._select_gpus(
@ -2848,7 +2848,7 @@ class LlamaCppBackend:
# Auto context: prefer fewer GPUs, cap context
# to fit. Same headroom threshold as
# _select_gpus (#5106).
ranked = sorted(gpus, key = lambda g: g[1], reverse = True)
ranked = sorted(gpus, key=lambda g: g[1], reverse=True)
pin_fraction = self._GPU_PIN_VRAM_FRACTION
for n_gpus in range(1, len(ranked) + 1):
subset = ranked[:n_gpus]
@ -2858,11 +2858,11 @@ class LlamaCppBackend:
pool_mib,
model_size,
cache_type_kv,
n_parallel = n_parallel,
mtp_engaged = _mtp_will_engage,
n_parallel=n_parallel,
mtp_engaged=_mtp_will_engage,
)
kv = self._estimate_kv_cache_bytes(
capped, cache_type_kv, n_parallel = n_parallel
capped, cache_type_kv, n_parallel=n_parallel
)
total_mib = (model_size + kv) / (1024 * 1024)
if total_mib <= pool_mib * pin_fraction:
@ -2883,7 +2883,7 @@ class LlamaCppBackend:
kv = self._estimate_kv_cache_bytes(
effective_ctx,
cache_type_kv,
n_parallel = n_parallel,
n_parallel=n_parallel,
)
total_mib = (model_size + kv) / (1024 * 1024)
if total_mib <= pool_mib * pin_fraction:
@ -2899,7 +2899,7 @@ class LlamaCppBackend:
# keep the ceiling at the native context (already the default).
logger.debug(
"Falling back to file-size-only GPU selection",
model_size_gb = round(model_size / (1024**3), 2),
model_size_gb=round(model_size / (1024**3), 2),
)
gpu_indices, use_fit = self._select_gpus(model_size, gpus)
if use_fit and not explicit_ctx:
@ -2912,7 +2912,7 @@ class LlamaCppBackend:
if effective_ctx < original_ctx:
kv_est = self._estimate_kv_cache_bytes(
effective_ctx, cache_type_kv, n_parallel = n_parallel
effective_ctx, cache_type_kv, n_parallel=n_parallel
)
logger.info(
f"Context auto-reduced: {original_ctx} -> {effective_ctx} "
@ -2921,7 +2921,7 @@ class LlamaCppBackend:
)
kv_cache_bytes = self._estimate_kv_cache_bytes(
effective_ctx, cache_type_kv, n_parallel = n_parallel
effective_ctx, cache_type_kv, n_parallel=n_parallel
)
logger.info(
f"GGUF size: {model_size / (1024**3):.1f} GB, "
@ -2935,8 +2935,8 @@ class LlamaCppBackend:
effective_ctx = n_ctx # fall back to original
launch_mmproj_path = self._resolve_launch_mmproj_path(
model_path = model_path,
mmproj_path = mmproj_path,
model_path=model_path,
mmproj_path=mmproj_path,
)
# Need both a resolved mmproj AND the config vision flag; a stray
# mmproj passing the family-name heuristic must not flip a non-VLM
@ -3030,13 +3030,13 @@ class LlamaCppBackend:
# fallback to -MTP in name. GPU: MTP-only. CPU/Mac: chain
# with ngram-mod. See unsloth.ai/docs/models/qwen3.6#mtp-guide.
spec_flags = self._build_speculative_flags(
speculative_type = speculative_type,
spec_draft_n_max = spec_draft_n_max,
extra_args = extra_args,
model_identifier = model_identifier,
model_path = model_path,
gpus = bool(gpus),
binary = binary,
speculative_type=speculative_type,
spec_draft_n_max=spec_draft_n_max,
extra_args=extra_args,
model_identifier=model_identifier,
model_path=model_path,
gpus=bool(gpus),
binary=binary,
)
cmd.extend(spec_flags)
@ -3048,7 +3048,7 @@ class LlamaCppBackend:
flags = detect_reasoning_flags(
chat_template_override,
self._model_identifier,
log_source = "GGUF chat template override",
log_source="GGUF chat template override",
)
self._supports_reasoning = flags["supports_reasoning"]
self._reasoning_style = flags["reasoning_style"]
@ -3059,10 +3059,10 @@ class LlamaCppBackend:
self._supports_tools = flags["supports_tools"]
self._chat_template_file = tempfile.NamedTemporaryFile(
mode = "w",
suffix = ".jinja",
delete = False,
prefix = "unsloth_chat_template_",
mode="w",
suffix=".jinja",
delete=False,
prefix="unsloth_chat_template_",
)
self._chat_template_file.write(chat_template_override)
self._chat_template_file.close()
@ -3243,15 +3243,15 @@ class LlamaCppBackend:
self._llama_log_fh = None
try:
log_dir = _swa_cache_path().parent / "logs" / "llama-server"
log_dir.mkdir(parents = True, exist_ok = True)
log_dir.mkdir(parents=True, exist_ok=True)
self._llama_log_path = (
log_dir / f"llama-{int(time.time())}-port-{self._port}.log"
)
self._llama_log_fh = open(
self._llama_log_path,
"w",
encoding = "utf-8",
buffering = 1,
encoding="utf-8",
buffering=1,
)
logger.info(f"llama-server stdout/stderr -> {self._llama_log_path}")
except OSError as e:
@ -3260,16 +3260,16 @@ class LlamaCppBackend:
self._llama_log_path = None
self._process = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
env = env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
env=env,
**_windows_hidden_subprocess_kwargs(),
)
# Start background thread to drain stdout and prevent pipe deadlock
self._stdout_thread = threading.Thread(
target = self._drain_stdout, daemon = True, name = "llama-stdout"
target=self._drain_stdout, daemon=True, name="llama-stdout"
)
self._stdout_thread.start()
@ -3308,7 +3308,7 @@ class LlamaCppBackend:
)
# Wait for llama-server to become healthy
if not self._wait_for_health(timeout = 600.0):
if not self._wait_for_health(timeout=600.0):
self._kill_process()
_gguf = gguf_path or ""
_is_ollama = (
@ -3573,7 +3573,7 @@ class LlamaCppBackend:
"fall back to spec-off if no nextn head is present. "
"Engaging anyway (user override)."
)
_emit_mtp(chain_ngram = False)
_emit_mtp(chain_ngram=False)
return flags
if effective_mode == "mtp+ngram":
if _mtp_too_small:
@ -3588,14 +3588,14 @@ class LlamaCppBackend:
"may fall back to ngram-only if no nextn head is "
"present. Engaging anyway (user override)."
)
_emit_mtp(chain_ngram = True)
_emit_mtp(chain_ngram=True)
return flags
# effective_mode == "auto": today's promotion path. llama.cpp
# #22673: MTP is compatible with mmproj, so there's no vision gate.
if is_mtp_model and not _mtp_too_small:
# GPU: MTP-only. CPU/Mac: chain ngram-mod + MTP.
_emit_mtp(chain_ngram = not gpus)
_emit_mtp(chain_ngram=not gpus)
elif is_mtp_model and _mtp_too_small:
# Sub-3B fallback: drop the MTP draft head, keep ngram-mod
# when the binary supports it.
@ -3807,11 +3807,11 @@ class LlamaCppBackend:
return
try:
self._process.terminate()
self._process.wait(timeout = 5)
self._process.wait(timeout=5)
except subprocess.TimeoutExpired:
logger.warning("llama-server did not exit on SIGTERM, sending SIGKILL")
self._process.kill()
self._process.wait(timeout = 5)
self._process.wait(timeout=5)
except Exception as e:
logger.warning(f"Error killing llama-server process: {e}")
finally:
@ -3825,7 +3825,7 @@ class LlamaCppBackend:
# /unload+/load Apply paths record the kill.
self._last_kill_monotonic = time.monotonic()
if self._stdout_thread is not None:
self._stdout_thread.join(timeout = 2)
self._stdout_thread.join(timeout=2)
self._stdout_thread = None
fh = getattr(self, "_llama_log_fh", None)
if fh is not None:
@ -3968,10 +3968,10 @@ class LlamaCppBackend:
return
result = subprocess.run(
["pgrep", "-a", "-f", "llama-server"],
capture_output = True,
text = True,
timeout = 5,
env = child_env_without_native_path_secret(),
capture_output=True,
text=True,
timeout=5,
env=child_env_without_native_path_secret(),
)
if result.returncode != 0:
return
@ -3991,13 +3991,13 @@ class LlamaCppBackend:
# unavailable.
proc_exe = Path(f"/proc/{pid}/exe")
try:
binary = proc_exe.resolve(strict = True)
binary = proc_exe.resolve(strict=True)
except (OSError, ValueError):
cmdline = parts[1]
token = cmdline.split()[0] if cmdline.strip() else ""
if not token:
continue
binary = Path(token).resolve(strict = False)
binary = Path(token).resolve(strict=False)
owned = binary in exact_binaries or any(
binary.is_relative_to(root) for root in resolved_roots
@ -4013,7 +4013,7 @@ class LlamaCppBackend:
except PermissionError:
pass
except Exception:
logger.warning("Error during orphan server cleanup", exc_info = True)
logger.warning("Error during orphan server cleanup", exc_info=True)
def _cleanup(self):
"""atexit handler to ensure llama-server is terminated."""
@ -4033,7 +4033,7 @@ class LlamaCppBackend:
if self._process.poll() is not None:
# Give the drain thread a moment to collect final output
if self._stdout_thread is not None:
self._stdout_thread.join(timeout = 2)
self._stdout_thread.join(timeout=2)
output = "\n".join(self._stdout_lines[-50:])
logger.error(
f"llama-server exited with code {self._process.returncode}. "
@ -4042,7 +4042,7 @@ class LlamaCppBackend:
return False
try:
resp = httpx.get(url, timeout = 2.0)
resp = httpx.get(url, timeout=2.0)
if resp.status_code == 200:
return True
except (
@ -4173,7 +4173,7 @@ class LlamaCppBackend:
def _cancel_watcher():
while not _cancel_closed.is_set():
if cancel_event.wait(timeout = 0.3):
if cancel_event.wait(timeout=0.3):
# Cancel requested. Keep polling until the response object
# exists so we can close it, or until the main thread
# finishes on its own (_cancel_closed is set in finally).
@ -4188,13 +4188,13 @@ class LlamaCppBackend:
f"Error closing response in cancel watcher: {e}"
)
# Response not created yet -- wait briefly and retry
_cancel_closed.wait(timeout = 0.1)
_cancel_closed.wait(timeout=0.1)
return
watcher = None
if cancel_event is not None:
watcher = threading.Thread(
target = _cancel_watcher, daemon = True, name = "prefill-cancel"
target=_cancel_watcher, daemon=True, name="prefill-cancel"
)
watcher.start()
@ -4204,17 +4204,17 @@ class LlamaCppBackend:
# prefill and streaming is handled by the watcher thread
# which closes the response, unblocking any httpx read.
prefill_timeout = httpx.Timeout(
connect = 30,
read = 120.0,
write = 10,
pool = 10,
connect=30,
read=120.0,
write=10,
pool=10,
)
with client.stream(
"POST",
url,
json = payload,
timeout = prefill_timeout,
headers = headers,
json=payload,
timeout=prefill_timeout,
headers=headers,
) as response:
_response_ref[0] = response
if cancel_event is not None and cancel_event.is_set():
@ -4299,19 +4299,19 @@ class LlamaCppBackend:
# _stream_with_retry uses a 120 s read timeout so prefill
# can finish. Cancel during streaming is handled by the
# watcher thread (closes the response on cancel_event).
stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10)
stream_timeout = httpx.Timeout(connect=10, read=0.5, write=10, pool=10)
_auth_headers = (
{"Authorization": f"Bearer {self._api_key}"} if self._api_key else None
)
with httpx.Client(
timeout = stream_timeout, limits = httpx.Limits(max_keepalive_connections = 0)
timeout=stream_timeout, limits=httpx.Limits(max_keepalive_connections=0)
) as client:
with self._stream_with_retry(
client,
url,
payload,
cancel_event,
headers = _auth_headers,
headers=_auth_headers,
) as response:
if response.status_code != 200:
error_body = response.read().decode()
@ -4451,7 +4451,7 @@ class LlamaCppBackend:
def _strip_tool_markup(text: str, *, final: bool = False) -> str:
if not auto_heal_tool_calls:
return text
return strip_tool_call_markup(text, final = final)
return strip_tool_call_markup(text, final=final)
# XML prefixes that signal a tool call in content.
# Empty when auto_heal is disabled so the buffer never
@ -4543,21 +4543,21 @@ class LlamaCppBackend:
_last_emitted = ""
stream_timeout = httpx.Timeout(
connect = 10,
read = 0.5,
write = 10,
pool = 10,
connect=10,
read=0.5,
write=10,
pool=10,
)
with httpx.Client(
timeout = stream_timeout,
limits = httpx.Limits(max_keepalive_connections = 0),
timeout=stream_timeout,
limits=httpx.Limits(max_keepalive_connections=0),
) as client:
with self._stream_with_retry(
client,
url,
payload,
cancel_event,
headers = _auth_headers,
headers=_auth_headers,
) as response:
if response.status_code != 200:
error_body = response.read().decode()
@ -4587,7 +4587,7 @@ class LlamaCppBackend:
"type": "content",
"text": _strip_tool_markup(
cumulative_display,
final = True,
final=True,
),
}
else:
@ -4771,7 +4771,7 @@ class LlamaCppBackend:
"type": "content",
"text": _strip_tool_markup(
cumulative_display,
final = True,
final=True,
),
}
elif reasoning_accum and not has_content_tokens:
@ -4831,13 +4831,25 @@ class LlamaCppBackend:
"content": _stripped,
}
)
available_tool_names = [
tool.get("function", {}).get("name")
for tool in tools
if isinstance(tool, dict)
and isinstance(tool.get("function"), dict)
]
available_tool_names = [
name for name in available_tool_names if name
]
tool_hint = (
" or ".join(available_tool_names) or "an available tool"
)
conversation.append(
{
"role": "user",
"content": (
"STOP. Do NOT write code or explain. "
"You MUST call a tool NOW. "
"Call web_search or python immediately."
f"Call {tool_hint} immediately."
),
}
)
@ -4897,7 +4909,7 @@ class LlamaCppBackend:
tool_calls = _safety_tc
content_text = _strip_tool_markup(
content_accum,
final = True,
final=True,
)
logger.info(
f"Safety net: parsed {len(tool_calls)} tool call(s) "
@ -4931,7 +4943,7 @@ class LlamaCppBackend:
if tool_calls and not has_structured_tc:
content_text = _strip_tool_markup(
content_text,
final = True,
final=True,
)
if tool_calls:
logger.info(
@ -4946,7 +4958,7 @@ class LlamaCppBackend:
if content_accum:
# Strip leaked tool-call XML before yielding
content_accum = _strip_tool_markup(
content_accum, final = True
content_accum, final=True
)
if content_accum:
yield {"type": "content", "text": content_accum}
@ -5009,7 +5021,12 @@ class LlamaCppBackend:
arguments = json.loads(raw_args)
except (json.JSONDecodeError, ValueError):
if auto_heal_tool_calls:
arguments = {"query": raw_args}
heal_key = {
"python": "code",
"terminal": "command",
"render_html": "code",
}.get(tool_name, "query")
arguments = {heal_key: raw_args}
else:
arguments = {"raw": raw_args}
else:
@ -5077,9 +5094,9 @@ class LlamaCppBackend:
result = execute_tool(
tool_name,
arguments,
cancel_event = cancel_event,
timeout = _effective_timeout,
session_id = session_id,
cancel_event=cancel_event,
timeout=_effective_timeout,
session_id=session_id,
)
yield {
@ -5193,19 +5210,19 @@ class LlamaCppBackend:
_stream_done = False
try:
stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10)
stream_timeout = httpx.Timeout(connect=10, read=0.5, write=10, pool=10)
_auth_headers = (
{"Authorization": f"Bearer {self._api_key}"} if self._api_key else None
)
with httpx.Client(
timeout = stream_timeout, limits = httpx.Limits(max_keepalive_connections = 0)
timeout=stream_timeout, limits=httpx.Limits(max_keepalive_connections=0)
) as client:
with self._stream_with_retry(
client,
url,
stream_payload,
cancel_event,
headers = _auth_headers,
headers=_auth_headers,
) as response:
if response.status_code != 200:
error_body = response.read().decode()
@ -5231,7 +5248,7 @@ class LlamaCppBackend:
yield {
"type": "content",
"text": _strip_tool_markup(
cumulative, final = True
cumulative, final=True
),
}
else:
@ -5341,12 +5358,12 @@ class LlamaCppBackend:
_auth_headers = (
{"Authorization": f"Bearer {self._api_key}"} if self._api_key else None
)
with httpx.Client(timeout = 10, headers = _auth_headers) as client:
with httpx.Client(timeout=10, headers=_auth_headers) as client:
def _detok(tid: int) -> str:
# Non-200 means "marker not in vocab" -- keep probing.
# Transport / JSON errors still raise.
r = client.post(f"{self.base_url}/detokenize", json = {"tokens": [tid]})
r = client.post(f"{self.base_url}/detokenize", json={"tokens": [tid]})
if r.status_code != 200:
return ""
return r.json().get("content", "")
@ -5354,7 +5371,7 @@ class LlamaCppBackend:
def _tok(text: str) -> list[int]:
r = client.post(
f"{self.base_url}/tokenize",
json = {"content": text, "add_special": False},
json={"content": text, "add_special": False},
)
if r.status_code != 200:
return []
@ -5419,12 +5436,12 @@ class LlamaCppBackend:
import os
repo_path = snapshot_download(
"unsloth/Spark-TTS-0.5B", local_dir = "Spark-TTS-0.5B"
"unsloth/Spark-TTS-0.5B", local_dir="Spark-TTS-0.5B"
)
model_repo_path = os.path.abspath(repo_path)
LlamaCppBackend._codec_mgr.load_codec(
audio_type, device, model_repo_path = model_repo_path
audio_type, device, model_repo_path=model_repo_path
)
logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}")
@ -5449,7 +5466,7 @@ class LlamaCppBackend:
tpl, stop, need_ids = self._TTS_PROMPTS[audio_type]
payload: dict = {
"prompt": tpl.format(text = text),
"prompt": tpl.format(text=text),
"stream": False,
"n_predict": max_new_tokens,
"temperature": temperature,
@ -5467,9 +5484,9 @@ class LlamaCppBackend:
{"Authorization": f"Bearer {self._api_key}"} if self._api_key else None
)
with httpx.Client(
timeout = httpx.Timeout(300, connect = 10), headers = _auth_headers
timeout=httpx.Timeout(300, connect=10), headers=_auth_headers
) as client:
resp = client.post(f"{self.base_url}/completion", json = payload)
resp = client.post(f"{self.base_url}/completion", json=payload)
if resp.status_code != 200:
raise RuntimeError(
f"llama-server returned {resp.status_code}: {resp.text}"
@ -5486,5 +5503,5 @@ class LlamaCppBackend:
device = "cuda" if torch.cuda.is_available() else "cpu"
return LlamaCppBackend._codec_mgr.decode(
audio_type, device, token_ids = token_ids, text = data.get("content", "")
audio_type, device, token_ids=token_ids, text=data.get("content", "")
)

6
studio/backend/core/inference/safetensors_agentic.py
View file

@ -66,7 +66,11 @@ def _status_for_tool(tool_name: str, arguments: dict) -> str:
return f"Calling: {tool_name}"
_CANONICAL_HEAL_ARG = {"python": "code", "terminal": "command"}
_CANONICAL_HEAL_ARG = {
"python": "code",
"terminal": "command",
"render_html": "code",
}
def _coerce_arguments(raw_args, *, heal: bool, tool_name: str = "") -> dict:

115
studio/backend/core/inference/tools.py
View file

@ -41,7 +41,7 @@ if sys.platform == "linux":
_libc_name = ctypes.util.find_library("c")
if _libc_name:
_libc = ctypes.CDLL(_libc_name, use_errno = True)
_libc = ctypes.CDLL(_libc_name, use_errno=True)
except (OSError, AttributeError):
pass
@ -159,9 +159,9 @@ def _find_blocked_commands(command: str) -> set[str]:
# position after the `;` separator).
try:
if sys.platform == "win32":
tokens = shlex.split(command, posix = False)
tokens = shlex.split(command, posix=False)
else:
lexer = shlex.shlex(command, posix = True, punctuation_chars = ";&|()`")
lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|()`")
lexer.whitespace_split = True
tokens = list(lexer)
except ValueError:
@ -428,7 +428,7 @@ def _get_workdir(session_id: str | None = None) -> str:
workdir = os.path.join(sandbox_root, "_invalid")
else:
workdir = os.path.join(sandbox_root, "_default")
os.makedirs(workdir, exist_ok = True)
os.makedirs(workdir, exist_ok=True)
try:
os.chmod(sandbox_root, 0o700)
except OSError:
@ -502,16 +502,53 @@ TERMINAL_TOOL = {
},
}
ALL_TOOLS = [WEB_SEARCH_TOOL, PYTHON_TOOL, TERMINAL_TOOL]
RENDER_HTML_TOOL = {
"type": "function",
"function": {
"name": "render_html",
"description": (
"Render a self-contained HTML/CSS/JavaScript artifact for the user. "
"Put the entire document in code, including any CSS in <style> tags "
"and JavaScript in <script> tags."
),
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "A complete self-contained HTML document.",
},
"title": {
"type": "string",
"description": "Short display title for the artifact.",
},
},
"required": ["code"],
},
},
}
ALL_TOOLS = [WEB_SEARCH_TOOL, PYTHON_TOOL, TERMINAL_TOOL, RENDER_HTML_TOOL]
_TIMEOUT_UNSET = object()
def _render_html_result(arguments: dict) -> str:
code = arguments.get("code")
if not isinstance(code, str) or not code.strip():
return "Error: render_html requires a non-empty code string."
title = arguments.get("title")
if isinstance(title, str) and title.strip():
safe_title = title.strip()[:120]
return f"Rendered HTML artifact: {safe_title}"
return "Rendered HTML artifact."
def execute_tool(
name: str,
arguments: dict,
cancel_event = None,
cancel_event=None,
timeout: int | None = _TIMEOUT_UNSET,
session_id: str | None = None,
) -> str:
@ -525,11 +562,13 @@ def execute_tool(
f"execute_tool: name={name}, session_id={session_id}, timeout={timeout}"
)
effective_timeout = _EXEC_TIMEOUT if timeout is _TIMEOUT_UNSET else timeout
if name == "render_html":
return _render_html_result(arguments)
if name == "web_search":
return _web_search(
arguments.get("query", ""),
url = arguments.get("url"),
timeout = effective_timeout,
url=arguments.get("url"),
timeout=effective_timeout,
)
if name == "python":
return _python_exec(
@ -588,7 +627,7 @@ class _PinnedHTTPSConnection(http.client.HTTPSConnection):
# TLS handshake with the real hostname for SNI + cert verification.
self.sock = self._context.wrap_socket(
self.sock,
server_hostname = self._sni_hostname,
server_hostname=self._sni_hostname,
)
@ -601,7 +640,7 @@ class _SNIHTTPSHandler(urllib.request.HTTPSHandler):
"""
def __init__(self, hostname: str):
super().__init__(context = _tls_ctx)
super().__init__(context=_tls_ctx)
self._sni_hostname = hostname
def https_open(self, req):
@ -609,7 +648,7 @@ class _SNIHTTPSHandler(urllib.request.HTTPSHandler):
def _sni_connection(self, host, **kwargs):
kwargs["context"] = _tls_ctx
return _PinnedHTTPSConnection(host, sni_hostname = self._sni_hostname, **kwargs)
return _PinnedHTTPSConnection(host, sni_hostname=self._sni_hostname, **kwargs)
def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str]:
@ -623,7 +662,7 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str
import socket
try:
infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM)
infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
except OSError as e:
return False, f"Failed to resolve host: {e}", ""
@ -684,7 +723,7 @@ def _fetch_page_text(
# Bracket IPv6 addresses so the netloc is valid in a URL.
ip_str = f"[{pinned_ip}]" if ":" in pinned_ip else pinned_ip
ip_netloc = f"{ip_str}:{cp.port}" if cp.port else ip_str
pinned_url = urlunparse(cp._replace(netloc = ip_netloc))
pinned_url = urlunparse(cp._replace(netloc=ip_netloc))
opener = urllib.request.build_opener(
_NoRedirect,
@ -693,13 +732,13 @@ def _fetch_page_text(
req = urllib.request.Request(
pinned_url,
headers = {
headers={
"User-Agent": ua,
"Host": current_host,
},
)
try:
resp = opener.open(req, timeout = timeout)
resp = opener.open(req, timeout=timeout)
except _HTTPError as e:
if e.code not in (301, 302, 303, 307, 308):
return (
@ -728,7 +767,7 @@ def _fetch_page_text(
return "Failed to fetch URL: too many redirects."
charset = resp.headers.get_content_charset() or "utf-8"
raw_html = raw_bytes.decode(charset, errors = "replace")
raw_html = raw_bytes.decode(charset, errors="replace")
except _HTTPError as e:
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"
except Exception as e:
@ -759,14 +798,14 @@ def _web_search(
# Direct URL fetch mode
if url and url.strip():
fetch_timeout = 60 if timeout is None else min(timeout, 60)
return _fetch_page_text(url.strip(), timeout = fetch_timeout)
return _fetch_page_text(url.strip(), timeout=fetch_timeout)
if not query or not query.strip():
return "No query provided."
try:
from ddgs import DDGS
results = DDGS(timeout = timeout).text(query, max_results = max_results)
results = DDGS(timeout=timeout).text(query, max_results=max_results)
if not results:
return "No results found."
parts = []
@ -1844,7 +1883,7 @@ def _kill_process_tree(proc) -> None:
pass
def _cancel_watcher(proc, cancel_event, poll_interval = 0.2):
def _cancel_watcher(proc, cancel_event, poll_interval=0.2):
"""Daemon thread that kills a process when cancel_event is set."""
while proc.poll() is None:
if cancel_event is not None and cancel_event.is_set():
@ -1861,7 +1900,7 @@ def _truncate(text: str, limit: int = _MAX_OUTPUT_CHARS) -> str:
def _python_exec(
code: str,
cancel_event = None,
cancel_event=None,
timeout: int = _EXEC_TIMEOUT,
session_id: str | None = None,
) -> str:
@ -1889,18 +1928,18 @@ def _python_exec(
pass
try:
fd, tmp_path = tempfile.mkstemp(
suffix = ".py", prefix = "studio_exec_", dir = workdir
suffix=".py", prefix="studio_exec_", dir=workdir
)
with os.fdopen(fd, "w") as f:
f.write(code)
safe_env = _build_safe_env(workdir)
popen_kwargs = dict(
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
cwd = workdir,
env = safe_env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
cwd=workdir,
env=safe_env,
)
if sys.platform != "win32":
popen_kwargs["preexec_fn"] = _sandbox_preexec
@ -1912,16 +1951,16 @@ def _python_exec(
# Spawn cancel watcher if we have a cancel event
if cancel_event is not None:
watcher = threading.Thread(
target = _cancel_watcher, args = (proc, cancel_event), daemon = True
target=_cancel_watcher, args=(proc, cancel_event), daemon=True
)
watcher.start()
try:
output, _ = proc.communicate(timeout = timeout)
output, _ = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
_kill_process_tree(proc)
try:
proc.communicate(timeout = 5)
proc.communicate(timeout=5)
except subprocess.TimeoutExpired:
pass
return _truncate(f"Execution timed out after {timeout} seconds.")
@ -1968,7 +2007,7 @@ def _python_exec(
def _bash_exec(
command: str,
cancel_event = None,
cancel_event=None,
timeout: int = _EXEC_TIMEOUT,
session_id: str | None = None,
) -> str:
@ -1985,11 +2024,11 @@ def _bash_exec(
workdir = _get_workdir(session_id)
safe_env = _build_safe_env(workdir)
popen_kwargs = dict(
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
cwd = workdir,
env = safe_env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
cwd=workdir,
env=safe_env,
)
if sys.platform != "win32":
popen_kwargs["preexec_fn"] = _sandbox_preexec
@ -2000,16 +2039,16 @@ def _bash_exec(
if cancel_event is not None:
watcher = threading.Thread(
target = _cancel_watcher, args = (proc, cancel_event), daemon = True
target=_cancel_watcher, args=(proc, cancel_event), daemon=True
)
watcher.start()
try:
output, _ = proc.communicate(timeout = timeout)
output, _ = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
_kill_process_tree(proc)
try:
proc.communicate(timeout = 5)
proc.communicate(timeout=5)
except subprocess.TimeoutExpired:
pass
return _truncate(f"Execution timed out after {timeout} seconds.")

403
studio/backend/models/inference.py
View file

@ -24,31 +24,31 @@ from pydantic import (
class LoadRequest(BaseModel):
"""Request to load a model for inference"""
model_path: str = Field(..., description = "Model identifier or local path")
model_path: str = Field(..., description="Model identifier or local path")
native_path_lease: Optional[str] = Field(
None, description = "Frontend-visible signed native path grant"
None, description="Frontend-visible signed native path grant"
)
hf_token: Optional[str] = Field(
None, description = "HuggingFace token for gated models"
None, description="HuggingFace token for gated models"
)
max_seq_length: int = Field(
0,
ge = 0,
le = 1048576,
description = "Maximum sequence length (0 = model default for GGUF)",
ge=0,
le=1048576,
description="Maximum sequence length (0 = model default for GGUF)",
)
load_in_4bit: bool = Field(True, description = "Load model in 4-bit quantization")
is_lora: bool = Field(False, description = "Whether this is a LoRA adapter")
load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization")
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
gguf_variant: Optional[str] = Field(
None, description = "GGUF quantization variant (e.g. 'Q4_K_M')"
None, description="GGUF quantization variant (e.g. 'Q4_K_M')"
)
trust_remote_code: bool = Field(
False,
description = "Allow loading models with custom code (e.g. NVIDIA Nemotron). Only enable for repos you trust.",
description="Allow loading models with custom code (e.g. NVIDIA Nemotron). Only enable for repos you trust.",
)
chat_template_override: Optional[str] = Field(
None,
description = "Custom Jinja2 chat template to use instead of the model's default",
description="Custom Jinja2 chat template to use instead of the model's default",
)
@field_validator("chat_template_override")
@ -62,15 +62,15 @@ class LoadRequest(BaseModel):
cache_type_kv: Optional[str] = Field(
None,
description = "KV cache data type for both K and V (e.g. 'f16', 'bf16', 'q8_0', 'q4_1', 'q5_1')",
description="KV cache data type for both K and V (e.g. 'f16', 'bf16', 'q8_0', 'q4_1', 'q5_1')",
)
gpu_ids: Optional[List[int]] = Field(
None,
description = "Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries. Not supported for GGUF models.",
description="Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries. Not supported for GGUF models.",
)
speculative_type: Optional[str] = Field(
None,
description = (
description=(
"Speculative decoding mode for GGUF models. Canonical values: "
"'auto' (platform-aware: MTP on MTP GGUFs, ngram-mod fallback "
"for sub-3B), 'mtp' (force draft-mtp only on both GPU and CPU), "
@ -83,9 +83,9 @@ class LoadRequest(BaseModel):
)
spec_draft_n_max: Optional[int] = Field(
None,
ge = 1,
le = 16,
description = (
ge=1,
le=16,
description=(
"Max draft tokens per step for MTP speculative decoding "
"(--spec-draft-n-max). Defaults to 2 on GPU and 3 on CPU/Mac "
"when unset (upstream-bench sweet spot for dense Qwen3.6 MTP "
@ -95,7 +95,7 @@ class LoadRequest(BaseModel):
)
llama_extra_args: Optional[List[str]] = Field(
None,
description = (
description=(
"Extra arguments forwarded verbatim to llama-server for GGUF models. "
"One token per list entry, e.g. ['--top-k', '20', '--seed', '42']. "
"Studio-managed flags (model identity, port, context length, GPU placement, "
@ -108,7 +108,7 @@ class LoadRequest(BaseModel):
class UnloadRequest(BaseModel):
"""Request to unload a model"""
model_path: str = Field(..., description = "Model identifier to unload")
model_path: str = Field(..., description="Model identifier to unload")
class ValidateModelRequest(BaseModel):
@ -119,15 +119,15 @@ class ValidateModelRequest(BaseModel):
This does NOT actually load weights into GPU memory.
"""
model_path: str = Field(..., description = "Model identifier or local path")
model_path: str = Field(..., description="Model identifier or local path")
native_path_lease: Optional[str] = Field(
None, description = "Frontend-visible signed native path grant"
None, description="Frontend-visible signed native path grant"
)
hf_token: Optional[str] = Field(
None, description = "HuggingFace token for gated models"
None, description="HuggingFace token for gated models"
)
gguf_variant: Optional[str] = Field(
None, description = "GGUF quantization variant (e.g. 'Q4_K_M')"
None, description="GGUF quantization variant (e.g. 'Q4_K_M')"
)
@ -139,107 +139,107 @@ class ValidateModelResponse(BaseModel):
introspection (GGUF / LoRA / vision flags) is available.
"""
valid: bool = Field(..., description = "Whether the model identifier looks valid")
message: str = Field(..., description = "Human-readable validation message")
identifier: Optional[str] = Field(None, description = "Resolved model identifier")
valid: bool = Field(..., description="Whether the model identifier looks valid")
message: str = Field(..., description="Human-readable validation message")
identifier: Optional[str] = Field(None, description="Resolved model identifier")
display_name: Optional[str] = Field(
None, description = "Display name derived from identifier"
None, description="Display name derived from identifier"
)
is_gguf: bool = Field(False, description = "Whether this is a GGUF model (llama.cpp)")
is_lora: bool = Field(False, description = "Whether this is a LoRA adapter")
is_vision: bool = Field(False, description = "Whether this is a vision-capable model")
is_gguf: bool = Field(False, description="Whether this is a GGUF model (llama.cpp)")
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
is_vision: bool = Field(False, description="Whether this is a vision-capable model")
requires_trust_remote_code: bool = Field(
False,
description = "Whether the model defaults require trust_remote_code to be enabled for loading.",
description="Whether the model defaults require trust_remote_code to be enabled for loading.",
)
class GenerateRequest(BaseModel):
"""Request for text generation (legacy /generate/stream endpoint)"""
messages: List[dict] = Field(..., description = "Chat messages in OpenAI format")
system_prompt: str = Field("", description = "System prompt")
temperature: float = Field(0.6, ge = 0.0, le = 2.0, description = "Sampling temperature")
top_p: float = Field(0.95, ge = 0.0, le = 1.0, description = "Top-p sampling")
top_k: int = Field(20, ge = -1, le = 100, description = "Top-k sampling")
messages: List[dict] = Field(..., description="Chat messages in OpenAI format")
system_prompt: str = Field("", description="System prompt")
temperature: float = Field(0.6, ge=0.0, le=2.0, description="Sampling temperature")
top_p: float = Field(0.95, ge=0.0, le=1.0, description="Top-p sampling")
top_k: int = Field(20, ge=-1, le=100, description="Top-k sampling")
max_new_tokens: int = Field(
2048, ge = 1, le = 4096, description = "Maximum tokens to generate"
2048, ge=1, le=4096, description="Maximum tokens to generate"
)
repetition_penalty: float = Field(
1.0, ge = 1.0, le = 2.0, description = "Repetition penalty"
1.0, ge=1.0, le=2.0, description="Repetition penalty"
)
presence_penalty: float = Field(0.0, ge = 0.0, le = 2.0, description = "Presence penalty")
presence_penalty: float = Field(0.0, ge=0.0, le=2.0, description="Presence penalty")
image_base64: Optional[str] = Field(
None, description = "Base64 encoded image for vision models"
None, description="Base64 encoded image for vision models"
)
class LoadResponse(BaseModel):
"""Response after loading a model"""
status: str = Field(..., description = "Load status")
model: str = Field(..., description = "Model identifier")
display_name: str = Field(..., description = "Display name of the model")
is_vision: bool = Field(False, description = "Whether model is a vision model")
is_lora: bool = Field(False, description = "Whether model is a LoRA adapter")
status: str = Field(..., description="Load status")
model: str = Field(..., description="Model identifier")
display_name: str = Field(..., description="Display name of the model")
is_vision: bool = Field(False, description="Whether model is a vision model")
is_lora: bool = Field(False, description="Whether model is a LoRA adapter")
is_gguf: bool = Field(
False, description = "Whether model is a GGUF model (llama.cpp)"
False, description="Whether model is a GGUF model (llama.cpp)"
)
is_audio: bool = Field(False, description = "Whether model is a TTS audio model")
is_audio: bool = Field(False, description="Whether model is a TTS audio model")
audio_type: Optional[str] = Field(
None, description = "Audio codec type: snac, csm, bicodec, dac"
None, description="Audio codec type: snac, csm, bicodec, dac"
)
has_audio_input: bool = Field(
False, description = "Whether model accepts audio input (ASR)"
False, description="Whether model accepts audio input (ASR)"
)
inference: dict = Field(
..., description = "Inference parameters (temperature, top_p, top_k, min_p)"
..., description="Inference parameters (temperature, top_p, top_k, min_p)"
)
requires_trust_remote_code: bool = Field(
False,
description = "Whether the model defaults require trust_remote_code to be enabled for loading.",
description="Whether the model defaults require trust_remote_code to be enabled for loading.",
)
context_length: Optional[int] = Field(
None, description = "Model's native context length (from GGUF metadata)"
None, description="Model's native context length (from GGUF metadata)"
)
max_context_length: Optional[int] = Field(
None, description = "Maximum context length currently available on this hardware"
None, description="Maximum context length currently available on this hardware"
)
native_context_length: Optional[int] = Field(
None,
description = "Model's native context length from GGUF metadata (not capped by VRAM)",
description="Model's native context length from GGUF metadata (not capped by VRAM)",
)
supports_reasoning: bool = Field(
False,
description = "Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)",
description="Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)",
)
reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field(
"enable_thinking",
description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)",
description="Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)",
)
reasoning_always_on: bool = Field(
False,
description = "Whether reasoning is always on (hardcoded <think> tags, not toggleable)",
description="Whether reasoning is always on (hardcoded <think> tags, not toggleable)",
)
supports_preserve_thinking: bool = Field(
False,
description = "Whether the template understands the optional preserve_thinking kwarg (Qwen3.6-style)",
description="Whether the template understands the optional preserve_thinking kwarg (Qwen3.6-style)",
)
supports_tools: bool = Field(
False,
description = "Whether model supports tool calling (web search, etc.)",
description="Whether model supports tool calling (web search, etc.)",
)
cache_type_kv: Optional[str] = Field(
None,
description = "KV cache data type for K and V (e.g. 'f16', 'bf16', 'q8_0')",
description="KV cache data type for K and V (e.g. 'f16', 'bf16', 'q8_0')",
)
chat_template: Optional[str] = Field(
None,
description = "Jinja2 chat template string (from GGUF metadata or tokenizer)",
description="Jinja2 chat template string (from GGUF metadata or tokenizer)",
)
speculative_type: Optional[str] = Field(
None,
description = (
description=(
"Canonical UI-facing requested speculative decoding mode "
"('auto' / 'mtp' / 'ngram' / 'mtp+ngram' / 'off' / "
"'ngram-simple'), round-tripped from the original LoadRequest "
@ -248,7 +248,7 @@ class LoadResponse(BaseModel):
)
spec_draft_n_max: Optional[int] = Field(
None,
description = (
description=(
"Active --spec-draft-n-max for MTP speculative decoding, or "
"None when the platform default is in effect."
),
@ -258,8 +258,8 @@ class LoadResponse(BaseModel):
class UnloadResponse(BaseModel):
"""Response after unloading a model"""
status: str = Field(..., description = "Unload status")
model: str = Field(..., description = "Model identifier that was unloaded")
status: str = Field(..., description="Unload status")
model: str = Field(..., description="Model identifier that was unloaded")
class LoadProgressResponse(BaseModel):
@ -273,7 +273,7 @@ class LoadProgressResponse(BaseModel):
phase: Optional[str] = Field(
None,
description = (
description=(
"Load phase: 'mmap' (weights paging into RAM via mmap), "
"'ready' (llama-server reported healthy), or null when no "
"load is in flight."
@ -281,17 +281,17 @@ class LoadProgressResponse(BaseModel):
)
bytes_loaded: int = Field(
0,
description = (
description=(
"Bytes of the model already resident in the llama-server "
"process (VmRSS on Linux)."
),
)
bytes_total: int = Field(
0,
description = "Total bytes across all GGUF shards for the active model.",
description="Total bytes across all GGUF shards for the active model.",
)
fraction: float = Field(
0.0, description = "bytes_loaded / bytes_total, clamped to 0..1."
0.0, description="bytes_loaded / bytes_total, clamped to 0..1."
)
@ -299,81 +299,81 @@ class InferenceStatusResponse(BaseModel):
"""Current inference backend status"""
active_model: Optional[str] = Field(
None, description = "Currently active model identifier"
None, description="Currently active model identifier"
)
is_vision: bool = Field(
False, description = "Whether the active model is a vision model"
False, description="Whether the active model is a vision model"
)
is_gguf: bool = Field(
False, description = "Whether the active model is a GGUF model (llama.cpp)"
False, description="Whether the active model is a GGUF model (llama.cpp)"
)
gguf_variant: Optional[str] = Field(
None, description = "GGUF quantization variant (e.g. Q4_K_M)"
None, description="GGUF quantization variant (e.g. Q4_K_M)"
)
is_audio: bool = Field(
False, description = "Whether the active model is a TTS audio model"
False, description="Whether the active model is a TTS audio model"
)
audio_type: Optional[str] = Field(
None, description = "Audio codec type: snac, csm, bicodec, dac"
None, description="Audio codec type: snac, csm, bicodec, dac"
)
has_audio_input: bool = Field(
False, description = "Whether model accepts audio input (ASR)"
False, description="Whether model accepts audio input (ASR)"
)
loading: List[str] = Field(
default_factory = list, description = "Models currently being loaded"
default_factory=list, description="Models currently being loaded"
)
loaded: List[str] = Field(
default_factory = list, description = "Models currently loaded"
default_factory=list, description="Models currently loaded"
)
inference: Optional[Dict[str, Any]] = Field(
None, description = "Recommended inference parameters for the active model"
None, description="Recommended inference parameters for the active model"
)
requires_trust_remote_code: bool = Field(
False,
description = "Whether the active model requires trust_remote_code to be enabled for loading.",
description="Whether the active model requires trust_remote_code to be enabled for loading.",
)
supports_reasoning: bool = Field(
False, description = "Whether the active model supports reasoning/thinking mode"
False, description="Whether the active model supports reasoning/thinking mode"
)
reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field(
"enable_thinking",
description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)",
description="Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)",
)
reasoning_always_on: bool = Field(
False, description = "Whether reasoning is always on (not toggleable)"
False, description="Whether reasoning is always on (not toggleable)"
)
supports_preserve_thinking: bool = Field(
False,
description = "Whether the active model's template understands the optional preserve_thinking kwarg",
description="Whether the active model's template understands the optional preserve_thinking kwarg",
)
supports_tools: bool = Field(
False, description = "Whether the active model supports tool calling"
False, description="Whether the active model supports tool calling"
)
context_length: Optional[int] = Field(
None, description = "Context length of the active model"
None, description="Context length of the active model"
)
max_context_length: Optional[int] = Field(
None,
description = "Maximum context length currently available for the active model",
description="Maximum context length currently available for the active model",
)
native_context_length: Optional[int] = Field(
None,
description = "Model's native context length from GGUF metadata (not capped by VRAM)",
description="Model's native context length from GGUF metadata (not capped by VRAM)",
)
cache_type_kv: Optional[str] = Field(
None,
description = "KV cache quantization dtype (e.g. 'q8_0'), or None for default",
description="KV cache quantization dtype (e.g. 'q8_0'), or None for default",
)
chat_template: Optional[str] = Field(
None, description = "Model's default chat template (Jinja2 source), if any"
None, description="Model's default chat template (Jinja2 source), if any"
)
chat_template_override: Optional[str] = Field(
None,
description = "Active chat template override applied at load time, or None if model is using its default",
description="Active chat template override applied at load time, or None if model is using its default",
)
speculative_type: Optional[str] = Field(
None,
description = (
description=(
"Canonical UI-facing requested speculative decoding mode "
"('auto' / 'mtp' / 'ngram' / 'mtp+ngram' / 'off' / "
"'ngram-simple'), round-tripped from the original LoadRequest. "
@ -382,32 +382,32 @@ class InferenceStatusResponse(BaseModel):
)
spec_draft_n_max: Optional[int] = Field(
None,
description = (
description=(
"Active --spec-draft-n-max for MTP speculative decoding, or "
"None when the platform default is in effect."
),
)
llama_cpp_supports_mtp: bool = Field(
True,
description = (
description=(
"Whether llama.cpp supports MTP (--spec-type mtp/draft-mtp). "
"False -> recommend `unsloth studio update`."
),
)
llama_cpp_prebuilt_stale: bool = Field(
False,
description = (
description=(
"Installed llama.cpp prebuilt is >=3 days behind the latest "
"release. True -> show `unsloth studio update` banner."
),
)
llama_cpp_installed_tag: Optional[str] = Field(
None,
description = "Installed llama.cpp tag, or None if unknown.",
description="Installed llama.cpp tag, or None if unknown.",
)
llama_cpp_latest_tag: Optional[str] = Field(
None,
description = "Latest published llama.cpp tag, or None if GitHub unreachable.",
description="Latest published llama.cpp tag, or None if GitHub unreachable.",
)
@ -429,7 +429,7 @@ class TextContentPart(BaseModel):
class ImageUrl(BaseModel):
"""Image URL object — supports data URIs and remote URLs."""
url: str = Field(..., description = "data:image/png;base64,... or https://...")
url: str = Field(..., description="data:image/png;base64,... or https://...")
detail: Optional[Literal["auto", "low", "high"]] = "auto"
@ -455,19 +455,19 @@ class InputDocumentContentPart(BaseModel):
type: Literal["input_document"]
file_data: Optional[str] = Field(
None,
description = "data:<media_type>;base64,<DATA> URI for inline payloads. Either file_data or file_url must be set; otherwise the part is dropped.",
description="data:<media_type>;base64,<DATA> URI for inline payloads. Either file_data or file_url must be set; otherwise the part is dropped.",
)
file_url: Optional[str] = Field(
None,
description = "Remote URL pointing to the document (https://...).",
description="Remote URL pointing to the document (https://...).",
)
filename: Optional[str] = Field(
None,
description = "Display filename, forwarded to providers as `title`/`filename`.",
description="Display filename, forwarded to providers as `title`/`filename`.",
)
media_type: Optional[str] = Field(
None,
description = 'Override the media type sniffed from the data URI (e.g. "application/pdf").',
description='Override the media type sniffed from the data URI (e.g. "application/pdf").',
)
@ -489,7 +489,7 @@ class CompactionContentPart(BaseModel):
type: Literal["compaction"]
content: str = Field(
...,
description = "Anthropic-produced summary of the compacted-away conversation prefix.",
description="Anthropic-produced summary of the compacted-away conversation prefix.",
)
@ -524,25 +524,25 @@ class ChatMessage(BaseModel):
"""
role: Literal["system", "user", "assistant", "tool"] = Field(
..., description = "Message role"
..., description="Message role"
)
content: Optional[Union[str, list[ContentPart]]] = Field(
None, description = "Message content (string or multimodal parts)"
None, description="Message content (string or multimodal parts)"
)
tool_call_id: Optional[str] = Field(
None,
description = "OpenAI tool-result messages: id of the tool call this result belongs to.",
description="OpenAI tool-result messages: id of the tool call this result belongs to.",
)
tool_calls: Optional[list[dict]] = Field(
None,
description = "OpenAI assistant messages: structured tool calls the model decided to make.",
description="OpenAI assistant messages: structured tool calls the model decided to make.",
)
name: Optional[str] = Field(
None,
description = "OpenAI tool-result messages: name of the tool whose result this is.",
description="OpenAI tool-result messages: name of the tool whose result this is.",
)
@model_validator(mode = "after")
@model_validator(mode="after")
def _validate_role_shape(self) -> "ChatMessage":
if self.tool_calls is not None and self.role != "assistant":
raise ValueError('"tool_calls" is only valid on role="assistant" messages.')
@ -580,29 +580,29 @@ class ChatCompletionRequest(BaseModel):
model: str = Field(
"default",
description = "Model identifier (informational; the active model is used)",
description="Model identifier (informational; the active model is used)",
)
messages: list[ChatMessage] = Field(..., description = "Conversation messages")
messages: list[ChatMessage] = Field(..., description="Conversation messages")
stream: bool = Field(
False,
description = (
description=(
"Whether to stream the response via SSE. Default matches OpenAI's "
"spec (`false`); opt into streaming by sending `stream: true`."
),
)
temperature: float = Field(0.6, ge = 0.0, le = 2.0)
top_p: float = Field(0.95, ge = 0.0, le = 1.0)
temperature: float = Field(0.6, ge=0.0, le=2.0)
top_p: float = Field(0.95, ge=0.0, le=1.0)
max_tokens: Optional[int] = Field(
None, ge = 1, description = "Maximum tokens to generate (None = until EOS)"
None, ge=1, description="Maximum tokens to generate (None = until EOS)"
)
presence_penalty: float = Field(0.0, ge = 0.0, le = 2.0, description = "Presence penalty")
presence_penalty: float = Field(0.0, ge=0.0, le=2.0, description="Presence penalty")
stop: Optional[Union[str, list[str]]] = Field(
None,
description = "OpenAI stop sequences: a single string or list of strings at which generation halts.",
description="OpenAI stop sequences: a single string or list of strings at which generation halts.",
)
tools: Optional[list[dict]] = Field(
None,
description = (
description=(
"OpenAI function-tool definitions. When provided without `enable_tools=true`, "
"Studio forwards the tools to the backend so the model returns structured "
"tool_calls for the client to execute (standard OpenAI function calling)."
@ -610,29 +610,29 @@ class ChatCompletionRequest(BaseModel):
)
tool_choice: Optional[Union[str, dict]] = Field(
None,
description = (
description=(
"OpenAI tool choice: 'auto' | 'required' | 'none' | "
"{'type': 'function', 'function': {'name': ...}}"
),
)
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
top_k: int = Field(20, ge = -1, le = 100, description = "[x-unsloth] Top-k sampling")
top_k: int = Field(20, ge=-1, le=100, description="[x-unsloth] Top-k sampling")
min_p: float = Field(
0.01, ge = 0.0, le = 1.0, description = "[x-unsloth] Min-p sampling threshold"
0.01, ge=0.0, le=1.0, description="[x-unsloth] Min-p sampling threshold"
)
repetition_penalty: float = Field(
1.0, ge = 1.0, le = 2.0, description = "[x-unsloth] Repetition penalty"
1.0, ge=1.0, le=2.0, description="[x-unsloth] Repetition penalty"
)
image_base64: Optional[str] = Field(
None, description = "[x-unsloth] Base64-encoded image for vision models"
None, description="[x-unsloth] Base64-encoded image for vision models"
)
audio_base64: Optional[str] = Field(
None, description = "[x-unsloth] Base64-encoded WAV for audio-input models (ASR)"
None, description="[x-unsloth] Base64-encoded WAV for audio-input models (ASR)"
)
use_adapter: Optional[Union[bool, str]] = Field(
None,
description = (
description=(
"[x-unsloth] Adapter control for compare mode. "
"null = no change (default), "
"false = disable adapters (base model), "
@ -642,79 +642,80 @@ class ChatCompletionRequest(BaseModel):
)
enable_thinking: Optional[bool] = Field(
None,
description = "[x-unsloth] Enable/disable thinking/reasoning mode for supported models",
description="[x-unsloth] Enable/disable thinking/reasoning mode for supported models",
)
reasoning_effort: Optional[
Literal["none", "minimal", "low", "medium", "high", "max", "xhigh"]
] = Field(
None,
description = "[x-unsloth] Reasoning effort level ('none'|'minimal'|'low'|'medium'|'high'|'max'|'xhigh'). OpenAI `/v1/responses` accepts model-dependent subsets; Anthropic adaptive thinking uses `max` as the top tier on Claude 4.6 Opus/Sonnet (inbound `xhigh` is mapped to `max`) and `xhigh` on Claude 4.7 Opus; local Harmony/gpt-oss templates support low|medium|high.",
description="[x-unsloth] Reasoning effort level ('none'|'minimal'|'low'|'medium'|'high'|'max'|'xhigh'). OpenAI `/v1/responses` accepts model-dependent subsets; Anthropic adaptive thinking uses `max` as the top tier on Claude 4.6 Opus/Sonnet (inbound `xhigh` is mapped to `max`) and `xhigh` on Claude 4.7 Opus; local Harmony/gpt-oss templates support low|medium|high.",
)
preserve_thinking: Optional[bool] = Field(
None,
description = "[x-unsloth] When true, keep historical <think> blocks from past assistant turns in the prompt (Qwen3.6 templates). Independent of enable_thinking / reasoning_effort.",
description="[x-unsloth] When true, keep historical <think> blocks from past assistant turns in the prompt (Qwen3.6 templates). Independent of enable_thinking / reasoning_effort.",
)
enable_tools: Optional[bool] = Field(
None,
description = "[x-unsloth] Enable tool calling for supported models",
description="[x-unsloth] Enable tool calling for supported models",
)
enabled_tools: Optional[list[str]] = Field(
None,
description = (
"[x-unsloth] List of enabled tool names. Local GGUF models accept "
"['web_search', 'python', 'terminal']. External providers accept "
"['web_search', 'web_fetch', 'code_execution'] for Anthropic and "
"['web_search', 'code_execution'] for OpenAI Responses. If None, "
"all local tools are enabled and no server-side tools are forwarded."
description=(
"[x-unsloth] List of enabled tool names. Local GGUF/safetensors models "
"accept ['web_search', 'python', 'terminal', 'render_html']. External "
"providers accept ['web_search', 'web_fetch', 'code_execution'] for "
"Anthropic and ['web_search', 'code_execution', 'image_generation'] for "
"OpenAI Responses. If None, all local tools are enabled and no "
"server-side tools are forwarded."
),
)
auto_heal_tool_calls: Optional[bool] = Field(
True,
description = "[x-unsloth] Auto-detect and fix malformed tool calls from model output.",
description="[x-unsloth] Auto-detect and fix malformed tool calls from model output.",
)
max_tool_calls_per_message: Optional[int] = Field(
25,
ge = 0,
description = "[x-unsloth] Maximum number of tool call iterations per message (0 = disabled, 9999 = unlimited).",
ge=0,
description="[x-unsloth] Maximum number of tool call iterations per message (0 = disabled, 9999 = unlimited).",
)
tool_call_timeout: Optional[int] = Field(
300,
ge = 1,
description = "[x-unsloth] Timeout in seconds for each tool call execution (9999 = no limit).",
ge=1,
description="[x-unsloth] Timeout in seconds for each tool call execution (9999 = no limit).",
)
session_id: Optional[str] = Field(
None,
description = "[x-unsloth] Session/thread ID for scoping tool execution sandbox.",
description="[x-unsloth] Session/thread ID for scoping tool execution sandbox.",
)
cancel_id: Optional[str] = Field(
None,
description = "[x-unsloth] Per-request cancellation token. Frontend sends a fresh UUID per run so /inference/cancel matches one specific generation.",
description="[x-unsloth] Per-request cancellation token. Frontend sends a fresh UUID per run so /inference/cancel matches one specific generation.",
)
# ── External provider routing (x-unsloth extensions) ──────────
provider_id: Optional[str] = Field(
None,
description = "[x-unsloth] Saved provider config ID. If set with encrypted_api_key, routes to external LLM.",
description="[x-unsloth] Saved provider config ID. If set with encrypted_api_key, routes to external LLM.",
)
provider_type: Optional[str] = Field(
None,
description = "[x-unsloth] Provider type (e.g. 'openai', 'mistral'). Used if provider_id is not set.",
description="[x-unsloth] Provider type (e.g. 'openai', 'mistral'). Used if provider_id is not set.",
)
external_model: Optional[str] = Field(
None,
description = "[x-unsloth] Model ID at the external provider.",
description="[x-unsloth] Model ID at the external provider.",
)
encrypted_api_key: Optional[str] = Field(
None,
description = "[x-unsloth] RSA-encrypted, base64-encoded API key for the external provider.",
description="[x-unsloth] RSA-encrypted, base64-encoded API key for the external provider.",
)
provider_base_url: Optional[str] = Field(
None,
description = "[x-unsloth] Override base URL for the external provider.",
description="[x-unsloth] Override base URL for the external provider.",
)
enable_prompt_caching: Optional[bool] = Field(
None,
description = (
description=(
"[x-unsloth] Opt in to provider-side prompt caching. On Anthropic, "
"attaches cache_control={type:ephemeral} to the system block so the "
"static prefix is reused across turns. On OpenAI cloud, caching is "
@ -725,7 +726,7 @@ class ChatCompletionRequest(BaseModel):
)
prompt_cache_ttl: Optional[str] = Field(
None,
description = (
description=(
"[x-unsloth] Anthropic cache_control TTL. Defaults to the 5-minute "
"ephemeral pool when omitted. Pass `1h` to write into the 1-hour "
"pool instead -- 1h writes are billed at 2x base input vs 1.25x "
@ -738,9 +739,9 @@ class ChatCompletionRequest(BaseModel):
)
compaction_threshold: Optional[int] = Field(
None,
ge = 1,
le = 2_000_000,
description = (
ge=1,
le=2_000_000,
description=(
"[x-unsloth] Server-side context compaction trigger, in tokens. "
"Per-provider routing:\n"
" - Anthropic (Opus 4.6+, Sonnet 4.6, Mythos preview): attaches "
@ -762,7 +763,7 @@ class ChatCompletionRequest(BaseModel):
)
openai_code_exec_container_id: Optional[str] = Field(
None,
description = (
description=(
"[x-unsloth] OpenAI shell-tool container id from the prior response "
"in the same chat thread. When set and `code_execution` is in "
"`enabled_tools`, the next /v1/responses call uses "
@ -774,7 +775,7 @@ class ChatCompletionRequest(BaseModel):
)
anthropic_code_exec_container_id: Optional[str] = Field(
None,
description = (
description=(
"[x-unsloth] Anthropic code_execution container id from the prior "
"response in the same chat thread. When set and `code_execution` "
"is in `enabled_tools`, the next /v1/messages call carries a "
@ -787,7 +788,7 @@ class ChatCompletionRequest(BaseModel):
),
)
@model_validator(mode = "after")
@model_validator(mode="after")
def _resolve_missing_tool_call_ids(self) -> "ChatCompletionRequest":
"""Fill missing tool_call_id by walking back to the preceding assistant.
@ -872,26 +873,26 @@ class OpenAIContainerRequest(BaseModel):
encrypted_api_key: str = Field(
...,
description = "[x-unsloth] RSA-encrypted, base64-encoded OpenAI API key.",
description="[x-unsloth] RSA-encrypted, base64-encoded OpenAI API key.",
)
provider_base_url: Optional[str] = Field(
None,
description = "[x-unsloth] OpenAI base URL. Only api.openai.com is supported; non-cloud bases are rejected with 400.",
description="[x-unsloth] OpenAI base URL. Only api.openai.com is supported; non-cloud bases are rejected with 400.",
)
class CreateOpenAIContainerBody(OpenAIContainerRequest):
name: str = Field(
...,
min_length = 1,
max_length = 256,
description = "Human-readable container name. Surfaces in the picker UI.",
min_length=1,
max_length=256,
description="Human-readable container name. Surfaces in the picker UI.",
)
ttl_minutes: int = Field(
20,
ge = 1,
le = 20,
description = (
ge=1,
le=20,
description=(
"Idle-timeout TTL the new container will inherit (anchor="
"last_active_at). OpenAI hard-caps this at 20 minutes and "
"rejects larger values with integer_above_max_value."
@ -902,7 +903,7 @@ class CreateOpenAIContainerBody(OpenAIContainerRequest):
class DeleteOpenAIContainerBody(OpenAIContainerRequest):
container_id: str = Field(
...,
description = "OpenAI container id (cntr_...) to delete.",
description="OpenAI container id (cntr_...) to delete.",
)
@ -942,9 +943,9 @@ class ChunkChoice(BaseModel):
class ChatCompletionChunk(BaseModel):
"""A single SSE chunk in OpenAI streaming format."""
id: str = Field(default_factory = lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}")
id: str = Field(default_factory=lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}")
object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
created: int = Field(default_factory = lambda: int(time.time()))
created: int = Field(default_factory=lambda: int(time.time()))
model: str = "default"
choices: list[ChunkChoice]
usage: Optional[CompletionUsage] = None
@ -980,12 +981,12 @@ class CompletionUsage(BaseModel):
class ChatCompletion(BaseModel):
"""Non-streaming chat completion response."""
id: str = Field(default_factory = lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}")
id: str = Field(default_factory=lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}")
object: Literal["chat.completion"] = "chat.completion"
created: int = Field(default_factory = lambda: int(time.time()))
created: int = Field(default_factory=lambda: int(time.time()))
model: str = "default"
choices: list[CompletionChoice]
usage: CompletionUsage = Field(default_factory = CompletionUsage)
usage: CompletionUsage = Field(default_factory=CompletionUsage)
# =====================================================================
@ -1007,7 +1008,7 @@ class ResponsesInputImagePart(BaseModel):
"""Image content part in a Responses API message (type=input_image)."""
type: Literal["input_image"]
image_url: str = Field(..., description = "data:image/png;base64,... or https://...")
image_url: str = Field(..., description="data:image/png;base64,... or https://...")
detail: Optional[Literal["auto", "low", "high"]] = "auto"
@ -1073,15 +1074,15 @@ class ResponsesFunctionCallInputItem(BaseModel):
type: Literal["function_call"]
id: Optional[str] = Field(
None, description = "Item id assigned by the server (e.g. fc_...)"
None, description="Item id assigned by the server (e.g. fc_...)"
)
call_id: str = Field(
...,
description = "Correlation id matching a function_call_output on the next turn.",
description="Correlation id matching a function_call_output on the next turn.",
)
name: str
arguments: str = Field(
..., description = "JSON string of the arguments the model produced."
..., description="JSON string of the arguments the model produced."
)
status: Optional[Literal["in_progress", "completed", "incomplete"]] = None
@ -1097,7 +1098,7 @@ class ResponsesFunctionCallOutputInputItem(BaseModel):
id: Optional[str] = None
call_id: str
output: Union[str, list] = Field(
..., description = "String or content-array result of the tool call."
..., description="String or content-array result of the tool call."
)
status: Optional[Literal["in_progress", "completed", "incomplete"]] = None
@ -1173,18 +1174,18 @@ class ResponsesFunctionTool(BaseModel):
class ResponsesRequest(BaseModel):
"""OpenAI Responses API request."""
model: str = Field("default", description = "Model identifier")
model: str = Field("default", description="Model identifier")
input: Union[str, list[ResponsesInputItem]] = Field(
default = [],
description = "Input text or list of messages / function_call / function_call_output items",
default=[],
description="Input text or list of messages / function_call / function_call_output items",
)
instructions: Optional[str] = Field(
None, description = "System / developer instructions"
None, description="System / developer instructions"
)
temperature: Optional[float] = Field(None, ge = 0.0, le = 2.0)
top_p: Optional[float] = Field(None, ge = 0.0, le = 1.0)
max_output_tokens: Optional[int] = Field(None, ge = 1)
stream: bool = Field(False, description = "Whether to stream the response via SSE")
temperature: Optional[float] = Field(None, ge=0.0, le=2.0)
top_p: Optional[float] = Field(None, ge=0.0, le=1.0)
max_output_tokens: Optional[int] = Field(None, ge=1)
stream: bool = Field(False, description="Whether to stream the response via SSE")
# OpenAI function-calling fields — forwarded to llama-server via the
# Chat Completions pass-through (see routes/inference.py). Typed as a
@ -1193,7 +1194,7 @@ class ResponsesRequest(BaseModel):
# picks out only ``type=="function"`` entries for forwarding.
tools: Optional[list[dict]] = Field(
None,
description = (
description=(
"Responses-shape function tool definitions. Entries with "
'`type="function"` are translated to the Chat Completions nested '
"shape before being forwarded to llama-server; other tool types "
@ -1203,7 +1204,7 @@ class ResponsesRequest(BaseModel):
)
tool_choice: Optional[Any] = Field(
None,
description = (
description=(
"'auto' | 'required' | 'none' | {'type': 'function', 'name': ...} — "
"the Responses-shape forcing object is translated to the Chat "
"Completions nested shape internally."
@ -1230,17 +1231,17 @@ class ResponsesOutputTextContent(BaseModel):
type: Literal["output_text"] = "output_text"
text: str
annotations: list = Field(default_factory = list)
annotations: list = Field(default_factory=list)
class ResponsesOutputMessage(BaseModel):
"""An output message in the Responses API response."""
type: Literal["message"] = "message"
id: str = Field(default_factory = lambda: f"msg_{uuid.uuid4().hex[:12]}")
id: str = Field(default_factory=lambda: f"msg_{uuid.uuid4().hex[:12]}")
status: Literal["completed", "in_progress"] = "completed"
role: Literal["assistant"] = "assistant"
content: list[ResponsesOutputTextContent] = Field(default_factory = list)
content: list[ResponsesOutputTextContent] = Field(default_factory=list)
class ResponsesOutputFunctionCall(BaseModel):
@ -1253,11 +1254,11 @@ class ResponsesOutputFunctionCall(BaseModel):
"""
type: Literal["function_call"] = "function_call"
id: str = Field(default_factory = lambda: f"fc_{uuid.uuid4().hex[:12]}")
id: str = Field(default_factory=lambda: f"fc_{uuid.uuid4().hex[:12]}")
call_id: str
name: str
arguments: str = Field(
..., description = "JSON string of the arguments the model produced."
..., description="JSON string of the arguments the model produced."
)
status: Literal["completed", "in_progress", "incomplete"] = "completed"
@ -1276,24 +1277,24 @@ class ResponsesUsage(BaseModel):
class ResponsesResponse(BaseModel):
"""Top-level Responses API response object."""
id: str = Field(default_factory = lambda: f"resp_{uuid.uuid4().hex[:12]}")
id: str = Field(default_factory=lambda: f"resp_{uuid.uuid4().hex[:12]}")
object: Literal["response"] = "response"
created_at: int = Field(default_factory = lambda: int(time.time()))
created_at: int = Field(default_factory=lambda: int(time.time()))
status: Literal["completed", "in_progress", "failed"] = "completed"
model: str = "default"
output: list[ResponsesOutputItem] = Field(default_factory = list)
usage: ResponsesUsage = Field(default_factory = ResponsesUsage)
output: list[ResponsesOutputItem] = Field(default_factory=list)
usage: ResponsesUsage = Field(default_factory=ResponsesUsage)
error: Optional[Any] = None
incomplete_details: Optional[Any] = None
instructions: Optional[str] = None
metadata: dict = Field(default_factory = dict)
metadata: dict = Field(default_factory=dict)
temperature: Optional[float] = None
top_p: Optional[float] = None
max_output_tokens: Optional[int] = None
previous_response_id: Optional[str] = None
text: Optional[Any] = None
tool_choice: Optional[Any] = None
tools: list = Field(default_factory = list)
tools: list = Field(default_factory=list)
truncation: Optional[Any] = None
@ -1372,13 +1373,13 @@ class AnthropicMessagesRequest(BaseModel):
metadata: Optional[dict] = None
# [x-unsloth] extensions — mirror the OpenAI endpoint convenience fields
min_p: Optional[float] = Field(
None, ge = 0.0, le = 1.0, description = "[x-unsloth] Min-p sampling threshold"
None, ge=0.0, le=1.0, description="[x-unsloth] Min-p sampling threshold"
)
repetition_penalty: Optional[float] = Field(
None, ge = 1.0, le = 2.0, description = "[x-unsloth] Repetition penalty"
None, ge=1.0, le=2.0, description="[x-unsloth] Repetition penalty"
)
presence_penalty: Optional[float] = Field(
None, ge = 0.0, le = 2.0, description = "[x-unsloth] Presence penalty"
None, ge=0.0, le=2.0, description="[x-unsloth] Presence penalty"
)
enable_tools: Optional[bool] = None
enabled_tools: Optional[list[str]] = None
@ -1413,11 +1414,11 @@ AnthropicResponseBlock = Union[
class AnthropicMessagesResponse(BaseModel):
id: str = Field(default_factory = lambda: f"msg_{uuid.uuid4().hex[:24]}")
id: str = Field(default_factory=lambda: f"msg_{uuid.uuid4().hex[:24]}")
type: Literal["message"] = "message"
role: Literal["assistant"] = "assistant"
content: list[AnthropicResponseBlock] = Field(default_factory = list)
content: list[AnthropicResponseBlock] = Field(default_factory=list)
model: str = "default"
stop_reason: Optional[str] = None
stop_sequence: Optional[str] = None
usage: AnthropicUsage = Field(default_factory = AnthropicUsage)
usage: AnthropicUsage = Field(default_factory=AnthropicUsage)

1799
studio/backend/routes/inference.py
View file

File diff suppressed because it is too large Load diff