diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index e4fb0afa80..5e54751fd1 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -128,7 +128,7 @@ def _probe_dns_dead(host: str = "huggingface.co", timeout: float = 2.0) -> bool: except Exception: result[0] = True - t = threading.Thread(target=_probe, daemon=True) + t = threading.Thread(target = _probe, daemon = True) t.start() t.join(timeout) # Thread still running -> resolver wedged -> treat as dead. @@ -185,10 +185,10 @@ def _load_swa_cache() -> dict: def _save_swa_cache(cache: dict) -> None: try: path = _swa_cache_path() - path.parent.mkdir(parents=True, exist_ok=True) + path.parent.mkdir(parents = True, exist_ok = True) tmp = path.with_suffix(".json.tmp") with open(tmp, "w") as f: - json.dump(cache, f, indent=2, sort_keys=True) + json.dump(cache, f, indent = 2, sort_keys = True) tmp.replace(path) except OSError: pass @@ -211,7 +211,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: try: from huggingface_hub import hf_hub_download - cfg_path = hf_hub_download(repo_id, "config.json", repo_type="model") + cfg_path = hf_hub_download(repo_id, "config.json", repo_type = "model") with open(cfg_path) as f: cfg = json.load(f) except Exception: @@ -835,7 +835,7 @@ class LlamaCppBackend: # Read VmRSS from /proc//status. Kilobytes on Linux. bytes_loaded = 0 try: - with open(f"/proc/{pid}/status", "r", encoding="utf-8") as f: + with open(f"/proc/{pid}/status", "r", encoding = "utf-8") as f: for line in f: if line.startswith("VmRSS:"): kb = int(line.split()[1]) @@ -1105,10 +1105,10 @@ class LlamaCppBackend: try: result = subprocess.run( [bin_path, "--help"], - capture_output=True, - text=True, - timeout=10, - check=False, + capture_output = True, + text = True, + timeout = 10, + check = False, ) help_text = (result.stdout or "") + "\n" + (result.stderr or "") # Split into per-flag blocks: each --flag line plus its @@ -1257,10 +1257,10 @@ class LlamaCppBackend: "--query-gpu=index,memory.free", "--format=csv,noheader,nounits", ], - capture_output=True, - text=True, - timeout=10, - env=child_env_without_native_path_secret(), + capture_output = True, + text = True, + timeout = 10, + env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: @@ -1290,7 +1290,7 @@ class LlamaCppBackend: # Match the docstring's sort-by-id guarantee. nvidia-smi # almost always returns sorted output, but driver order # is not formally guaranteed. - gpus.sort(key=lambda g: g[0]) + gpus.sort(key = lambda g: g[0]) if gpus: return gpus except Exception as e: @@ -1349,7 +1349,7 @@ class LlamaCppBackend: ) gpus.append((idx, free_bytes // (1024 * 1024))) # Match the nvidia-smi path's docstring guarantee of sorted-by-id. - return sorted(gpus, key=lambda g: g[0]) + return sorted(gpus, key = lambda g: g[0]) except Exception as e: logger.debug(f"torch GPU probe failed: {e}") return [] @@ -1534,7 +1534,7 @@ class LlamaCppBackend: usable_fraction = LlamaCppBackend._GPU_PIN_VRAM_FRACTION # Sort GPUs by free memory descending - ranked = sorted(gpus, key=lambda g: g[1], reverse=True) + ranked = sorted(gpus, key = lambda g: g[1], reverse = True) # Try fitting on 1 GPU at the usable-VRAM threshold. if ranked[0][1] * usable_fraction >= model_size_mib: @@ -1552,8 +1552,8 @@ class LlamaCppBackend: # Model is too large even for all GPUs, let --fit handle it logger.debug( "Model does not fit in available GPU memory, falling back to --fit", - model_size_mib=round(model_size_mib, 2), - ranked_gpus=ranked, + model_size_mib = round(model_size_mib, 2), + ranked_gpus = ranked, ) return None, True @@ -1799,8 +1799,8 @@ class LlamaCppBackend: if not self._can_estimate_kv(): logger.debug( "Skipping context fit because KV cache metadata is unavailable", - requested_ctx=requested_ctx, - available_mib=available_mib, + requested_ctx = requested_ctx, + available_mib = available_mib, ) return requested_ctx @@ -1809,10 +1809,10 @@ class LlamaCppBackend: return requested_ctx kv_kwargs = dict( - swa_full=swa_full, - n_parallel=n_parallel, - kv_unified=kv_unified, - ctx_checkpoints=ctx_checkpoints, + swa_full = swa_full, + n_parallel = n_parallel, + kv_unified = kv_unified, + ctx_checkpoints = ctx_checkpoints, ) # MTP needs a tighter budget; drop from 0.90 to 0.85. @@ -1830,9 +1830,9 @@ class LlamaCppBackend: if model_footprint >= budget_bytes: logger.debug( "Model footprint exceeds GPU budget before KV cache", - requested_ctx=requested_ctx, - available_mib=available_mib, - model_size_gb=round(model_footprint / (1024**3), 2), + requested_ctx = requested_ctx, + available_mib = available_mib, + model_size_gb = round(model_footprint / (1024**3), 2), ) return requested_ctx @@ -1874,7 +1874,7 @@ class LlamaCppBackend: try: from huggingface_hub import get_paths_info, list_repo_files - files = list_repo_files(hf_repo, token=hf_token) + files = list_repo_files(hf_repo, token = hf_token) gguf_files = [ f for f in files if f.endswith(".gguf") and "mmproj" not in f.lower() ] @@ -1882,7 +1882,7 @@ class LlamaCppBackend: return None # Get sizes for all GGUF files - path_infos = list(get_paths_info(hf_repo, gguf_files, token=hf_token)) + path_infos = list(get_paths_info(hf_repo, gguf_files, token = hf_token)) size_map = {p.path: (p.size or 0) for p in path_infos} # Group files by variant: shards share a prefix before -NNNNN-of-NNNNN @@ -1900,7 +1900,7 @@ class LlamaCppBackend: variant_sizes.append((first, total, shard_files)) # Sort by total size ascending and pick the smallest that fits - variant_sizes.sort(key=lambda x: x[1]) + variant_sizes.sort(key = lambda x: x[1]) for first_file, total_size, _ in variant_sizes: if total_size > 0 and total_size <= free_bytes: return first_file, total_size @@ -2215,7 +2215,7 @@ class LlamaCppBackend: flags = detect_reasoning_flags( self._chat_template, self._model_identifier, - log_source="GGUF metadata", + log_source = "GGUF metadata", ) self._supports_reasoning = flags["supports_reasoning"] self._reasoning_style = flags["reasoning_style"] @@ -2255,7 +2255,7 @@ class LlamaCppBackend: try: from huggingface_hub import list_repo_files - files = list_repo_files(hf_repo, token=hf_token) + files = list_repo_files(hf_repo, token = hf_token) variant_lower = hf_variant.lower() boundary = re.compile( r"(? 0: ranked_for_cap = sorted( - gpus, key=lambda g: g[1], reverse=True + gpus, key = lambda g: g[1], reverse = True ) best_cap = 0 for n_gpus in range(1, len(ranked_for_cap) + 1): @@ -2806,11 +2806,11 @@ class LlamaCppBackend: pool_mib, model_size, cache_type_kv, - n_parallel=n_parallel, - mtp_engaged=_mtp_will_engage, + n_parallel = n_parallel, + mtp_engaged = _mtp_will_engage, ) kv = self._estimate_kv_cache_bytes( - capped, cache_type_kv, n_parallel=n_parallel + capped, cache_type_kv, n_parallel = n_parallel ) total_mib = (model_size + kv) / (1024 * 1024) if total_mib <= pool_mib * 0.90: @@ -2837,7 +2837,7 @@ class LlamaCppBackend: requested_total = ( model_size + self._estimate_kv_cache_bytes( - effective_ctx, cache_type_kv, n_parallel=n_parallel + effective_ctx, cache_type_kv, n_parallel = n_parallel ) ) gpu_indices, use_fit = self._select_gpus( @@ -2848,7 +2848,7 @@ class LlamaCppBackend: # Auto context: prefer fewer GPUs, cap context # to fit. Same headroom threshold as # _select_gpus (#5106). - ranked = sorted(gpus, key=lambda g: g[1], reverse=True) + ranked = sorted(gpus, key = lambda g: g[1], reverse = True) pin_fraction = self._GPU_PIN_VRAM_FRACTION for n_gpus in range(1, len(ranked) + 1): subset = ranked[:n_gpus] @@ -2858,11 +2858,11 @@ class LlamaCppBackend: pool_mib, model_size, cache_type_kv, - n_parallel=n_parallel, - mtp_engaged=_mtp_will_engage, + n_parallel = n_parallel, + mtp_engaged = _mtp_will_engage, ) kv = self._estimate_kv_cache_bytes( - capped, cache_type_kv, n_parallel=n_parallel + capped, cache_type_kv, n_parallel = n_parallel ) total_mib = (model_size + kv) / (1024 * 1024) if total_mib <= pool_mib * pin_fraction: @@ -2883,7 +2883,7 @@ class LlamaCppBackend: kv = self._estimate_kv_cache_bytes( effective_ctx, cache_type_kv, - n_parallel=n_parallel, + n_parallel = n_parallel, ) total_mib = (model_size + kv) / (1024 * 1024) if total_mib <= pool_mib * pin_fraction: @@ -2899,7 +2899,7 @@ class LlamaCppBackend: # keep the ceiling at the native context (already the default). logger.debug( "Falling back to file-size-only GPU selection", - model_size_gb=round(model_size / (1024**3), 2), + model_size_gb = round(model_size / (1024**3), 2), ) gpu_indices, use_fit = self._select_gpus(model_size, gpus) if use_fit and not explicit_ctx: @@ -2912,7 +2912,7 @@ class LlamaCppBackend: if effective_ctx < original_ctx: kv_est = self._estimate_kv_cache_bytes( - effective_ctx, cache_type_kv, n_parallel=n_parallel + effective_ctx, cache_type_kv, n_parallel = n_parallel ) logger.info( f"Context auto-reduced: {original_ctx} -> {effective_ctx} " @@ -2921,7 +2921,7 @@ class LlamaCppBackend: ) kv_cache_bytes = self._estimate_kv_cache_bytes( - effective_ctx, cache_type_kv, n_parallel=n_parallel + effective_ctx, cache_type_kv, n_parallel = n_parallel ) logger.info( f"GGUF size: {model_size / (1024**3):.1f} GB, " @@ -2935,8 +2935,8 @@ class LlamaCppBackend: effective_ctx = n_ctx # fall back to original launch_mmproj_path = self._resolve_launch_mmproj_path( - model_path=model_path, - mmproj_path=mmproj_path, + model_path = model_path, + mmproj_path = mmproj_path, ) # Need both a resolved mmproj AND the config vision flag; a stray # mmproj passing the family-name heuristic must not flip a non-VLM @@ -3030,13 +3030,13 @@ class LlamaCppBackend: # fallback to -MTP in name. GPU: MTP-only. CPU/Mac: chain # with ngram-mod. See unsloth.ai/docs/models/qwen3.6#mtp-guide. spec_flags = self._build_speculative_flags( - speculative_type=speculative_type, - spec_draft_n_max=spec_draft_n_max, - extra_args=extra_args, - model_identifier=model_identifier, - model_path=model_path, - gpus=bool(gpus), - binary=binary, + speculative_type = speculative_type, + spec_draft_n_max = spec_draft_n_max, + extra_args = extra_args, + model_identifier = model_identifier, + model_path = model_path, + gpus = bool(gpus), + binary = binary, ) cmd.extend(spec_flags) @@ -3048,7 +3048,7 @@ class LlamaCppBackend: flags = detect_reasoning_flags( chat_template_override, self._model_identifier, - log_source="GGUF chat template override", + log_source = "GGUF chat template override", ) self._supports_reasoning = flags["supports_reasoning"] self._reasoning_style = flags["reasoning_style"] @@ -3059,10 +3059,10 @@ class LlamaCppBackend: self._supports_tools = flags["supports_tools"] self._chat_template_file = tempfile.NamedTemporaryFile( - mode="w", - suffix=".jinja", - delete=False, - prefix="unsloth_chat_template_", + mode = "w", + suffix = ".jinja", + delete = False, + prefix = "unsloth_chat_template_", ) self._chat_template_file.write(chat_template_override) self._chat_template_file.close() @@ -3243,15 +3243,15 @@ class LlamaCppBackend: self._llama_log_fh = None try: log_dir = _swa_cache_path().parent / "logs" / "llama-server" - log_dir.mkdir(parents=True, exist_ok=True) + log_dir.mkdir(parents = True, exist_ok = True) self._llama_log_path = ( log_dir / f"llama-{int(time.time())}-port-{self._port}.log" ) self._llama_log_fh = open( self._llama_log_path, "w", - encoding="utf-8", - buffering=1, + encoding = "utf-8", + buffering = 1, ) logger.info(f"llama-server stdout/stderr -> {self._llama_log_path}") except OSError as e: @@ -3260,16 +3260,16 @@ class LlamaCppBackend: self._llama_log_path = None self._process = subprocess.Popen( cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - env=env, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + env = env, **_windows_hidden_subprocess_kwargs(), ) # Start background thread to drain stdout and prevent pipe deadlock self._stdout_thread = threading.Thread( - target=self._drain_stdout, daemon=True, name="llama-stdout" + target = self._drain_stdout, daemon = True, name = "llama-stdout" ) self._stdout_thread.start() @@ -3308,7 +3308,7 @@ class LlamaCppBackend: ) # Wait for llama-server to become healthy - if not self._wait_for_health(timeout=600.0): + if not self._wait_for_health(timeout = 600.0): self._kill_process() _gguf = gguf_path or "" _is_ollama = ( @@ -3573,7 +3573,7 @@ class LlamaCppBackend: "fall back to spec-off if no nextn head is present. " "Engaging anyway (user override)." ) - _emit_mtp(chain_ngram=False) + _emit_mtp(chain_ngram = False) return flags if effective_mode == "mtp+ngram": if _mtp_too_small: @@ -3588,14 +3588,14 @@ class LlamaCppBackend: "may fall back to ngram-only if no nextn head is " "present. Engaging anyway (user override)." ) - _emit_mtp(chain_ngram=True) + _emit_mtp(chain_ngram = True) return flags # effective_mode == "auto": today's promotion path. llama.cpp # #22673: MTP is compatible with mmproj, so there's no vision gate. if is_mtp_model and not _mtp_too_small: # GPU: MTP-only. CPU/Mac: chain ngram-mod + MTP. - _emit_mtp(chain_ngram=not gpus) + _emit_mtp(chain_ngram = not gpus) elif is_mtp_model and _mtp_too_small: # Sub-3B fallback: drop the MTP draft head, keep ngram-mod # when the binary supports it. @@ -3807,11 +3807,11 @@ class LlamaCppBackend: return try: self._process.terminate() - self._process.wait(timeout=5) + self._process.wait(timeout = 5) except subprocess.TimeoutExpired: logger.warning("llama-server did not exit on SIGTERM, sending SIGKILL") self._process.kill() - self._process.wait(timeout=5) + self._process.wait(timeout = 5) except Exception as e: logger.warning(f"Error killing llama-server process: {e}") finally: @@ -3825,7 +3825,7 @@ class LlamaCppBackend: # /unload+/load Apply paths record the kill. self._last_kill_monotonic = time.monotonic() if self._stdout_thread is not None: - self._stdout_thread.join(timeout=2) + self._stdout_thread.join(timeout = 2) self._stdout_thread = None fh = getattr(self, "_llama_log_fh", None) if fh is not None: @@ -3968,10 +3968,10 @@ class LlamaCppBackend: return result = subprocess.run( ["pgrep", "-a", "-f", "llama-server"], - capture_output=True, - text=True, - timeout=5, - env=child_env_without_native_path_secret(), + capture_output = True, + text = True, + timeout = 5, + env = child_env_without_native_path_secret(), ) if result.returncode != 0: return @@ -3991,13 +3991,13 @@ class LlamaCppBackend: # unavailable. proc_exe = Path(f"/proc/{pid}/exe") try: - binary = proc_exe.resolve(strict=True) + binary = proc_exe.resolve(strict = True) except (OSError, ValueError): cmdline = parts[1] token = cmdline.split()[0] if cmdline.strip() else "" if not token: continue - binary = Path(token).resolve(strict=False) + binary = Path(token).resolve(strict = False) owned = binary in exact_binaries or any( binary.is_relative_to(root) for root in resolved_roots @@ -4013,7 +4013,7 @@ class LlamaCppBackend: except PermissionError: pass except Exception: - logger.warning("Error during orphan server cleanup", exc_info=True) + logger.warning("Error during orphan server cleanup", exc_info = True) def _cleanup(self): """atexit handler to ensure llama-server is terminated.""" @@ -4033,7 +4033,7 @@ class LlamaCppBackend: if self._process.poll() is not None: # Give the drain thread a moment to collect final output if self._stdout_thread is not None: - self._stdout_thread.join(timeout=2) + self._stdout_thread.join(timeout = 2) output = "\n".join(self._stdout_lines[-50:]) logger.error( f"llama-server exited with code {self._process.returncode}. " @@ -4042,7 +4042,7 @@ class LlamaCppBackend: return False try: - resp = httpx.get(url, timeout=2.0) + resp = httpx.get(url, timeout = 2.0) if resp.status_code == 200: return True except ( @@ -4173,7 +4173,7 @@ class LlamaCppBackend: def _cancel_watcher(): while not _cancel_closed.is_set(): - if cancel_event.wait(timeout=0.3): + if cancel_event.wait(timeout = 0.3): # Cancel requested. Keep polling until the response object # exists so we can close it, or until the main thread # finishes on its own (_cancel_closed is set in finally). @@ -4188,13 +4188,13 @@ class LlamaCppBackend: f"Error closing response in cancel watcher: {e}" ) # Response not created yet -- wait briefly and retry - _cancel_closed.wait(timeout=0.1) + _cancel_closed.wait(timeout = 0.1) return watcher = None if cancel_event is not None: watcher = threading.Thread( - target=_cancel_watcher, daemon=True, name="prefill-cancel" + target = _cancel_watcher, daemon = True, name = "prefill-cancel" ) watcher.start() @@ -4204,17 +4204,17 @@ class LlamaCppBackend: # prefill and streaming is handled by the watcher thread # which closes the response, unblocking any httpx read. prefill_timeout = httpx.Timeout( - connect=30, - read=120.0, - write=10, - pool=10, + connect = 30, + read = 120.0, + write = 10, + pool = 10, ) with client.stream( "POST", url, - json=payload, - timeout=prefill_timeout, - headers=headers, + json = payload, + timeout = prefill_timeout, + headers = headers, ) as response: _response_ref[0] = response if cancel_event is not None and cancel_event.is_set(): @@ -4299,19 +4299,19 @@ class LlamaCppBackend: # _stream_with_retry uses a 120 s read timeout so prefill # can finish. Cancel during streaming is handled by the # watcher thread (closes the response on cancel_event). - stream_timeout = httpx.Timeout(connect=10, read=0.5, write=10, pool=10) + stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10) _auth_headers = ( {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None ) with httpx.Client( - timeout=stream_timeout, limits=httpx.Limits(max_keepalive_connections=0) + timeout = stream_timeout, limits = httpx.Limits(max_keepalive_connections = 0) ) as client: with self._stream_with_retry( client, url, payload, cancel_event, - headers=_auth_headers, + headers = _auth_headers, ) as response: if response.status_code != 200: error_body = response.read().decode() @@ -4451,7 +4451,7 @@ class LlamaCppBackend: def _strip_tool_markup(text: str, *, final: bool = False) -> str: if not auto_heal_tool_calls: return text - return strip_tool_call_markup(text, final=final) + return strip_tool_call_markup(text, final = final) # XML prefixes that signal a tool call in content. # Empty when auto_heal is disabled so the buffer never @@ -4543,21 +4543,21 @@ class LlamaCppBackend: _last_emitted = "" stream_timeout = httpx.Timeout( - connect=10, - read=0.5, - write=10, - pool=10, + connect = 10, + read = 0.5, + write = 10, + pool = 10, ) with httpx.Client( - timeout=stream_timeout, - limits=httpx.Limits(max_keepalive_connections=0), + timeout = stream_timeout, + limits = httpx.Limits(max_keepalive_connections = 0), ) as client: with self._stream_with_retry( client, url, payload, cancel_event, - headers=_auth_headers, + headers = _auth_headers, ) as response: if response.status_code != 200: error_body = response.read().decode() @@ -4587,7 +4587,7 @@ class LlamaCppBackend: "type": "content", "text": _strip_tool_markup( cumulative_display, - final=True, + final = True, ), } else: @@ -4771,7 +4771,7 @@ class LlamaCppBackend: "type": "content", "text": _strip_tool_markup( cumulative_display, - final=True, + final = True, ), } elif reasoning_accum and not has_content_tokens: @@ -4909,7 +4909,7 @@ class LlamaCppBackend: tool_calls = _safety_tc content_text = _strip_tool_markup( content_accum, - final=True, + final = True, ) logger.info( f"Safety net: parsed {len(tool_calls)} tool call(s) " @@ -4943,7 +4943,7 @@ class LlamaCppBackend: if tool_calls and not has_structured_tc: content_text = _strip_tool_markup( content_text, - final=True, + final = True, ) if tool_calls: logger.info( @@ -4958,7 +4958,7 @@ class LlamaCppBackend: if content_accum: # Strip leaked tool-call XML before yielding content_accum = _strip_tool_markup( - content_accum, final=True + content_accum, final = True ) if content_accum: yield {"type": "content", "text": content_accum} @@ -5094,9 +5094,9 @@ class LlamaCppBackend: result = execute_tool( tool_name, arguments, - cancel_event=cancel_event, - timeout=_effective_timeout, - session_id=session_id, + cancel_event = cancel_event, + timeout = _effective_timeout, + session_id = session_id, ) yield { @@ -5210,19 +5210,19 @@ class LlamaCppBackend: _stream_done = False try: - stream_timeout = httpx.Timeout(connect=10, read=0.5, write=10, pool=10) + stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10) _auth_headers = ( {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None ) with httpx.Client( - timeout=stream_timeout, limits=httpx.Limits(max_keepalive_connections=0) + timeout = stream_timeout, limits = httpx.Limits(max_keepalive_connections = 0) ) as client: with self._stream_with_retry( client, url, stream_payload, cancel_event, - headers=_auth_headers, + headers = _auth_headers, ) as response: if response.status_code != 200: error_body = response.read().decode() @@ -5248,7 +5248,7 @@ class LlamaCppBackend: yield { "type": "content", "text": _strip_tool_markup( - cumulative, final=True + cumulative, final = True ), } else: @@ -5358,12 +5358,12 @@ class LlamaCppBackend: _auth_headers = ( {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None ) - with httpx.Client(timeout=10, headers=_auth_headers) as client: + with httpx.Client(timeout = 10, headers = _auth_headers) as client: def _detok(tid: int) -> str: # Non-200 means "marker not in vocab" -- keep probing. # Transport / JSON errors still raise. - r = client.post(f"{self.base_url}/detokenize", json={"tokens": [tid]}) + r = client.post(f"{self.base_url}/detokenize", json = {"tokens": [tid]}) if r.status_code != 200: return "" return r.json().get("content", "") @@ -5371,7 +5371,7 @@ class LlamaCppBackend: def _tok(text: str) -> list[int]: r = client.post( f"{self.base_url}/tokenize", - json={"content": text, "add_special": False}, + json = {"content": text, "add_special": False}, ) if r.status_code != 200: return [] @@ -5436,12 +5436,12 @@ class LlamaCppBackend: import os repo_path = snapshot_download( - "unsloth/Spark-TTS-0.5B", local_dir="Spark-TTS-0.5B" + "unsloth/Spark-TTS-0.5B", local_dir = "Spark-TTS-0.5B" ) model_repo_path = os.path.abspath(repo_path) LlamaCppBackend._codec_mgr.load_codec( - audio_type, device, model_repo_path=model_repo_path + audio_type, device, model_repo_path = model_repo_path ) logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}") @@ -5466,7 +5466,7 @@ class LlamaCppBackend: tpl, stop, need_ids = self._TTS_PROMPTS[audio_type] payload: dict = { - "prompt": tpl.format(text=text), + "prompt": tpl.format(text = text), "stream": False, "n_predict": max_new_tokens, "temperature": temperature, @@ -5484,9 +5484,9 @@ class LlamaCppBackend: {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None ) with httpx.Client( - timeout=httpx.Timeout(300, connect=10), headers=_auth_headers + timeout = httpx.Timeout(300, connect = 10), headers = _auth_headers ) as client: - resp = client.post(f"{self.base_url}/completion", json=payload) + resp = client.post(f"{self.base_url}/completion", json = payload) if resp.status_code != 200: raise RuntimeError( f"llama-server returned {resp.status_code}: {resp.text}" @@ -5503,5 +5503,5 @@ class LlamaCppBackend: device = "cuda" if torch.cuda.is_available() else "cpu" return LlamaCppBackend._codec_mgr.decode( - audio_type, device, token_ids=token_ids, text=data.get("content", "") + audio_type, device, token_ids = token_ids, text = data.get("content", "") ) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index ba8a155ead..589d803bb3 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -41,7 +41,7 @@ if sys.platform == "linux": _libc_name = ctypes.util.find_library("c") if _libc_name: - _libc = ctypes.CDLL(_libc_name, use_errno=True) + _libc = ctypes.CDLL(_libc_name, use_errno = True) except (OSError, AttributeError): pass @@ -159,9 +159,9 @@ def _find_blocked_commands(command: str) -> set[str]: # position after the `;` separator). try: if sys.platform == "win32": - tokens = shlex.split(command, posix=False) + tokens = shlex.split(command, posix = False) else: - lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|()`") + lexer = shlex.shlex(command, posix = True, punctuation_chars = ";&|()`") lexer.whitespace_split = True tokens = list(lexer) except ValueError: @@ -428,7 +428,7 @@ def _get_workdir(session_id: str | None = None) -> str: workdir = os.path.join(sandbox_root, "_invalid") else: workdir = os.path.join(sandbox_root, "_default") - os.makedirs(workdir, exist_ok=True) + os.makedirs(workdir, exist_ok = True) try: os.chmod(sandbox_root, 0o700) except OSError: @@ -548,7 +548,7 @@ def _render_html_result(arguments: dict) -> str: def execute_tool( name: str, arguments: dict, - cancel_event=None, + cancel_event = None, timeout: int | None = _TIMEOUT_UNSET, session_id: str | None = None, ) -> str: @@ -567,8 +567,8 @@ def execute_tool( if name == "web_search": return _web_search( arguments.get("query", ""), - url=arguments.get("url"), - timeout=effective_timeout, + url = arguments.get("url"), + timeout = effective_timeout, ) if name == "python": return _python_exec( @@ -627,7 +627,7 @@ class _PinnedHTTPSConnection(http.client.HTTPSConnection): # TLS handshake with the real hostname for SNI + cert verification. self.sock = self._context.wrap_socket( self.sock, - server_hostname=self._sni_hostname, + server_hostname = self._sni_hostname, ) @@ -640,7 +640,7 @@ class _SNIHTTPSHandler(urllib.request.HTTPSHandler): """ def __init__(self, hostname: str): - super().__init__(context=_tls_ctx) + super().__init__(context = _tls_ctx) self._sni_hostname = hostname def https_open(self, req): @@ -648,7 +648,7 @@ class _SNIHTTPSHandler(urllib.request.HTTPSHandler): def _sni_connection(self, host, **kwargs): kwargs["context"] = _tls_ctx - return _PinnedHTTPSConnection(host, sni_hostname=self._sni_hostname, **kwargs) + return _PinnedHTTPSConnection(host, sni_hostname = self._sni_hostname, **kwargs) def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str]: @@ -662,7 +662,7 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str import socket try: - infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM) except OSError as e: return False, f"Failed to resolve host: {e}", "" @@ -723,7 +723,7 @@ def _fetch_page_text( # Bracket IPv6 addresses so the netloc is valid in a URL. ip_str = f"[{pinned_ip}]" if ":" in pinned_ip else pinned_ip ip_netloc = f"{ip_str}:{cp.port}" if cp.port else ip_str - pinned_url = urlunparse(cp._replace(netloc=ip_netloc)) + pinned_url = urlunparse(cp._replace(netloc = ip_netloc)) opener = urllib.request.build_opener( _NoRedirect, @@ -732,13 +732,13 @@ def _fetch_page_text( req = urllib.request.Request( pinned_url, - headers={ + headers = { "User-Agent": ua, "Host": current_host, }, ) try: - resp = opener.open(req, timeout=timeout) + resp = opener.open(req, timeout = timeout) except _HTTPError as e: if e.code not in (301, 302, 303, 307, 308): return ( @@ -767,7 +767,7 @@ def _fetch_page_text( return "Failed to fetch URL: too many redirects." charset = resp.headers.get_content_charset() or "utf-8" - raw_html = raw_bytes.decode(charset, errors="replace") + raw_html = raw_bytes.decode(charset, errors = "replace") except _HTTPError as e: return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" except Exception as e: @@ -798,14 +798,14 @@ def _web_search( # Direct URL fetch mode if url and url.strip(): fetch_timeout = 60 if timeout is None else min(timeout, 60) - return _fetch_page_text(url.strip(), timeout=fetch_timeout) + return _fetch_page_text(url.strip(), timeout = fetch_timeout) if not query or not query.strip(): return "No query provided." try: from ddgs import DDGS - results = DDGS(timeout=timeout).text(query, max_results=max_results) + results = DDGS(timeout = timeout).text(query, max_results = max_results) if not results: return "No results found." parts = [] @@ -1883,7 +1883,7 @@ def _kill_process_tree(proc) -> None: pass -def _cancel_watcher(proc, cancel_event, poll_interval=0.2): +def _cancel_watcher(proc, cancel_event, poll_interval = 0.2): """Daemon thread that kills a process when cancel_event is set.""" while proc.poll() is None: if cancel_event is not None and cancel_event.is_set(): @@ -1900,7 +1900,7 @@ def _truncate(text: str, limit: int = _MAX_OUTPUT_CHARS) -> str: def _python_exec( code: str, - cancel_event=None, + cancel_event = None, timeout: int = _EXEC_TIMEOUT, session_id: str | None = None, ) -> str: @@ -1928,18 +1928,18 @@ def _python_exec( pass try: fd, tmp_path = tempfile.mkstemp( - suffix=".py", prefix="studio_exec_", dir=workdir + suffix = ".py", prefix = "studio_exec_", dir = workdir ) with os.fdopen(fd, "w") as f: f.write(code) safe_env = _build_safe_env(workdir) popen_kwargs = dict( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - cwd=workdir, - env=safe_env, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + cwd = workdir, + env = safe_env, ) if sys.platform != "win32": popen_kwargs["preexec_fn"] = _sandbox_preexec @@ -1951,16 +1951,16 @@ def _python_exec( # Spawn cancel watcher if we have a cancel event if cancel_event is not None: watcher = threading.Thread( - target=_cancel_watcher, args=(proc, cancel_event), daemon=True + target = _cancel_watcher, args = (proc, cancel_event), daemon = True ) watcher.start() try: - output, _ = proc.communicate(timeout=timeout) + output, _ = proc.communicate(timeout = timeout) except subprocess.TimeoutExpired: _kill_process_tree(proc) try: - proc.communicate(timeout=5) + proc.communicate(timeout = 5) except subprocess.TimeoutExpired: pass return _truncate(f"Execution timed out after {timeout} seconds.") @@ -2007,7 +2007,7 @@ def _python_exec( def _bash_exec( command: str, - cancel_event=None, + cancel_event = None, timeout: int = _EXEC_TIMEOUT, session_id: str | None = None, ) -> str: @@ -2024,11 +2024,11 @@ def _bash_exec( workdir = _get_workdir(session_id) safe_env = _build_safe_env(workdir) popen_kwargs = dict( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - cwd=workdir, - env=safe_env, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + cwd = workdir, + env = safe_env, ) if sys.platform != "win32": popen_kwargs["preexec_fn"] = _sandbox_preexec @@ -2039,16 +2039,16 @@ def _bash_exec( if cancel_event is not None: watcher = threading.Thread( - target=_cancel_watcher, args=(proc, cancel_event), daemon=True + target = _cancel_watcher, args = (proc, cancel_event), daemon = True ) watcher.start() try: - output, _ = proc.communicate(timeout=timeout) + output, _ = proc.communicate(timeout = timeout) except subprocess.TimeoutExpired: _kill_process_tree(proc) try: - proc.communicate(timeout=5) + proc.communicate(timeout = 5) except subprocess.TimeoutExpired: pass return _truncate(f"Execution timed out after {timeout} seconds.") diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 24c8560ec8..7dda429cdf 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -24,31 +24,31 @@ from pydantic import ( class LoadRequest(BaseModel): """Request to load a model for inference""" - model_path: str = Field(..., description="Model identifier or local path") + model_path: str = Field(..., description = "Model identifier or local path") native_path_lease: Optional[str] = Field( - None, description="Frontend-visible signed native path grant" + None, description = "Frontend-visible signed native path grant" ) hf_token: Optional[str] = Field( - None, description="HuggingFace token for gated models" + None, description = "HuggingFace token for gated models" ) max_seq_length: int = Field( 0, - ge=0, - le=1048576, - description="Maximum sequence length (0 = model default for GGUF)", + ge = 0, + le = 1048576, + description = "Maximum sequence length (0 = model default for GGUF)", ) - load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization") - is_lora: bool = Field(False, description="Whether this is a LoRA adapter") + load_in_4bit: bool = Field(True, description = "Load model in 4-bit quantization") + is_lora: bool = Field(False, description = "Whether this is a LoRA adapter") gguf_variant: Optional[str] = Field( - None, description="GGUF quantization variant (e.g. 'Q4_K_M')" + None, description = "GGUF quantization variant (e.g. 'Q4_K_M')" ) trust_remote_code: bool = Field( False, - description="Allow loading models with custom code (e.g. NVIDIA Nemotron). Only enable for repos you trust.", + description = "Allow loading models with custom code (e.g. NVIDIA Nemotron). Only enable for repos you trust.", ) chat_template_override: Optional[str] = Field( None, - description="Custom Jinja2 chat template to use instead of the model's default", + description = "Custom Jinja2 chat template to use instead of the model's default", ) @field_validator("chat_template_override") @@ -62,15 +62,15 @@ class LoadRequest(BaseModel): cache_type_kv: Optional[str] = Field( None, - description="KV cache data type for both K and V (e.g. 'f16', 'bf16', 'q8_0', 'q4_1', 'q5_1')", + description = "KV cache data type for both K and V (e.g. 'f16', 'bf16', 'q8_0', 'q4_1', 'q5_1')", ) gpu_ids: Optional[List[int]] = Field( None, - description="Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries. Not supported for GGUF models.", + description = "Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries. Not supported for GGUF models.", ) speculative_type: Optional[str] = Field( None, - description=( + description = ( "Speculative decoding mode for GGUF models. Canonical values: " "'auto' (platform-aware: MTP on MTP GGUFs, ngram-mod fallback " "for sub-3B), 'mtp' (force draft-mtp only on both GPU and CPU), " @@ -83,9 +83,9 @@ class LoadRequest(BaseModel): ) spec_draft_n_max: Optional[int] = Field( None, - ge=1, - le=16, - description=( + ge = 1, + le = 16, + description = ( "Max draft tokens per step for MTP speculative decoding " "(--spec-draft-n-max). Defaults to 2 on GPU and 3 on CPU/Mac " "when unset (upstream-bench sweet spot for dense Qwen3.6 MTP " @@ -95,7 +95,7 @@ class LoadRequest(BaseModel): ) llama_extra_args: Optional[List[str]] = Field( None, - description=( + description = ( "Extra arguments forwarded verbatim to llama-server for GGUF models. " "One token per list entry, e.g. ['--top-k', '20', '--seed', '42']. " "Studio-managed flags (model identity, port, context length, GPU placement, " @@ -108,7 +108,7 @@ class LoadRequest(BaseModel): class UnloadRequest(BaseModel): """Request to unload a model""" - model_path: str = Field(..., description="Model identifier to unload") + model_path: str = Field(..., description = "Model identifier to unload") class ValidateModelRequest(BaseModel): @@ -119,15 +119,15 @@ class ValidateModelRequest(BaseModel): This does NOT actually load weights into GPU memory. """ - model_path: str = Field(..., description="Model identifier or local path") + model_path: str = Field(..., description = "Model identifier or local path") native_path_lease: Optional[str] = Field( - None, description="Frontend-visible signed native path grant" + None, description = "Frontend-visible signed native path grant" ) hf_token: Optional[str] = Field( - None, description="HuggingFace token for gated models" + None, description = "HuggingFace token for gated models" ) gguf_variant: Optional[str] = Field( - None, description="GGUF quantization variant (e.g. 'Q4_K_M')" + None, description = "GGUF quantization variant (e.g. 'Q4_K_M')" ) @@ -139,107 +139,107 @@ class ValidateModelResponse(BaseModel): introspection (GGUF / LoRA / vision flags) is available. """ - valid: bool = Field(..., description="Whether the model identifier looks valid") - message: str = Field(..., description="Human-readable validation message") - identifier: Optional[str] = Field(None, description="Resolved model identifier") + valid: bool = Field(..., description = "Whether the model identifier looks valid") + message: str = Field(..., description = "Human-readable validation message") + identifier: Optional[str] = Field(None, description = "Resolved model identifier") display_name: Optional[str] = Field( - None, description="Display name derived from identifier" + None, description = "Display name derived from identifier" ) - is_gguf: bool = Field(False, description="Whether this is a GGUF model (llama.cpp)") - is_lora: bool = Field(False, description="Whether this is a LoRA adapter") - is_vision: bool = Field(False, description="Whether this is a vision-capable model") + is_gguf: bool = Field(False, description = "Whether this is a GGUF model (llama.cpp)") + is_lora: bool = Field(False, description = "Whether this is a LoRA adapter") + is_vision: bool = Field(False, description = "Whether this is a vision-capable model") requires_trust_remote_code: bool = Field( False, - description="Whether the model defaults require trust_remote_code to be enabled for loading.", + description = "Whether the model defaults require trust_remote_code to be enabled for loading.", ) class GenerateRequest(BaseModel): """Request for text generation (legacy /generate/stream endpoint)""" - messages: List[dict] = Field(..., description="Chat messages in OpenAI format") - system_prompt: str = Field("", description="System prompt") - temperature: float = Field(0.6, ge=0.0, le=2.0, description="Sampling temperature") - top_p: float = Field(0.95, ge=0.0, le=1.0, description="Top-p sampling") - top_k: int = Field(20, ge=-1, le=100, description="Top-k sampling") + messages: List[dict] = Field(..., description = "Chat messages in OpenAI format") + system_prompt: str = Field("", description = "System prompt") + temperature: float = Field(0.6, ge = 0.0, le = 2.0, description = "Sampling temperature") + top_p: float = Field(0.95, ge = 0.0, le = 1.0, description = "Top-p sampling") + top_k: int = Field(20, ge = -1, le = 100, description = "Top-k sampling") max_new_tokens: int = Field( - 2048, ge=1, le=4096, description="Maximum tokens to generate" + 2048, ge = 1, le = 4096, description = "Maximum tokens to generate" ) repetition_penalty: float = Field( - 1.0, ge=1.0, le=2.0, description="Repetition penalty" + 1.0, ge = 1.0, le = 2.0, description = "Repetition penalty" ) - presence_penalty: float = Field(0.0, ge=0.0, le=2.0, description="Presence penalty") + presence_penalty: float = Field(0.0, ge = 0.0, le = 2.0, description = "Presence penalty") image_base64: Optional[str] = Field( - None, description="Base64 encoded image for vision models" + None, description = "Base64 encoded image for vision models" ) class LoadResponse(BaseModel): """Response after loading a model""" - status: str = Field(..., description="Load status") - model: str = Field(..., description="Model identifier") - display_name: str = Field(..., description="Display name of the model") - is_vision: bool = Field(False, description="Whether model is a vision model") - is_lora: bool = Field(False, description="Whether model is a LoRA adapter") + status: str = Field(..., description = "Load status") + model: str = Field(..., description = "Model identifier") + display_name: str = Field(..., description = "Display name of the model") + is_vision: bool = Field(False, description = "Whether model is a vision model") + is_lora: bool = Field(False, description = "Whether model is a LoRA adapter") is_gguf: bool = Field( - False, description="Whether model is a GGUF model (llama.cpp)" + False, description = "Whether model is a GGUF model (llama.cpp)" ) - is_audio: bool = Field(False, description="Whether model is a TTS audio model") + is_audio: bool = Field(False, description = "Whether model is a TTS audio model") audio_type: Optional[str] = Field( - None, description="Audio codec type: snac, csm, bicodec, dac" + None, description = "Audio codec type: snac, csm, bicodec, dac" ) has_audio_input: bool = Field( - False, description="Whether model accepts audio input (ASR)" + False, description = "Whether model accepts audio input (ASR)" ) inference: dict = Field( - ..., description="Inference parameters (temperature, top_p, top_k, min_p)" + ..., description = "Inference parameters (temperature, top_p, top_k, min_p)" ) requires_trust_remote_code: bool = Field( False, - description="Whether the model defaults require trust_remote_code to be enabled for loading.", + description = "Whether the model defaults require trust_remote_code to be enabled for loading.", ) context_length: Optional[int] = Field( - None, description="Model's native context length (from GGUF metadata)" + None, description = "Model's native context length (from GGUF metadata)" ) max_context_length: Optional[int] = Field( - None, description="Maximum context length currently available on this hardware" + None, description = "Maximum context length currently available on this hardware" ) native_context_length: Optional[int] = Field( None, - description="Model's native context length from GGUF metadata (not capped by VRAM)", + description = "Model's native context length from GGUF metadata (not capped by VRAM)", ) supports_reasoning: bool = Field( False, - description="Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)", + description = "Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)", ) reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field( "enable_thinking", - description="Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", + description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", ) reasoning_always_on: bool = Field( False, - description="Whether reasoning is always on (hardcoded tags, not toggleable)", + description = "Whether reasoning is always on (hardcoded tags, not toggleable)", ) supports_preserve_thinking: bool = Field( False, - description="Whether the template understands the optional preserve_thinking kwarg (Qwen3.6-style)", + description = "Whether the template understands the optional preserve_thinking kwarg (Qwen3.6-style)", ) supports_tools: bool = Field( False, - description="Whether model supports tool calling (web search, etc.)", + description = "Whether model supports tool calling (web search, etc.)", ) cache_type_kv: Optional[str] = Field( None, - description="KV cache data type for K and V (e.g. 'f16', 'bf16', 'q8_0')", + description = "KV cache data type for K and V (e.g. 'f16', 'bf16', 'q8_0')", ) chat_template: Optional[str] = Field( None, - description="Jinja2 chat template string (from GGUF metadata or tokenizer)", + description = "Jinja2 chat template string (from GGUF metadata or tokenizer)", ) speculative_type: Optional[str] = Field( None, - description=( + description = ( "Canonical UI-facing requested speculative decoding mode " "('auto' / 'mtp' / 'ngram' / 'mtp+ngram' / 'off' / " "'ngram-simple'), round-tripped from the original LoadRequest " @@ -248,7 +248,7 @@ class LoadResponse(BaseModel): ) spec_draft_n_max: Optional[int] = Field( None, - description=( + description = ( "Active --spec-draft-n-max for MTP speculative decoding, or " "None when the platform default is in effect." ), @@ -258,8 +258,8 @@ class LoadResponse(BaseModel): class UnloadResponse(BaseModel): """Response after unloading a model""" - status: str = Field(..., description="Unload status") - model: str = Field(..., description="Model identifier that was unloaded") + status: str = Field(..., description = "Unload status") + model: str = Field(..., description = "Model identifier that was unloaded") class LoadProgressResponse(BaseModel): @@ -273,7 +273,7 @@ class LoadProgressResponse(BaseModel): phase: Optional[str] = Field( None, - description=( + description = ( "Load phase: 'mmap' (weights paging into RAM via mmap), " "'ready' (llama-server reported healthy), or null when no " "load is in flight." @@ -281,17 +281,17 @@ class LoadProgressResponse(BaseModel): ) bytes_loaded: int = Field( 0, - description=( + description = ( "Bytes of the model already resident in the llama-server " "process (VmRSS on Linux)." ), ) bytes_total: int = Field( 0, - description="Total bytes across all GGUF shards for the active model.", + description = "Total bytes across all GGUF shards for the active model.", ) fraction: float = Field( - 0.0, description="bytes_loaded / bytes_total, clamped to 0..1." + 0.0, description = "bytes_loaded / bytes_total, clamped to 0..1." ) @@ -299,81 +299,81 @@ class InferenceStatusResponse(BaseModel): """Current inference backend status""" active_model: Optional[str] = Field( - None, description="Currently active model identifier" + None, description = "Currently active model identifier" ) is_vision: bool = Field( - False, description="Whether the active model is a vision model" + False, description = "Whether the active model is a vision model" ) is_gguf: bool = Field( - False, description="Whether the active model is a GGUF model (llama.cpp)" + False, description = "Whether the active model is a GGUF model (llama.cpp)" ) gguf_variant: Optional[str] = Field( - None, description="GGUF quantization variant (e.g. Q4_K_M)" + None, description = "GGUF quantization variant (e.g. Q4_K_M)" ) is_audio: bool = Field( - False, description="Whether the active model is a TTS audio model" + False, description = "Whether the active model is a TTS audio model" ) audio_type: Optional[str] = Field( - None, description="Audio codec type: snac, csm, bicodec, dac" + None, description = "Audio codec type: snac, csm, bicodec, dac" ) has_audio_input: bool = Field( - False, description="Whether model accepts audio input (ASR)" + False, description = "Whether model accepts audio input (ASR)" ) loading: List[str] = Field( - default_factory=list, description="Models currently being loaded" + default_factory = list, description = "Models currently being loaded" ) loaded: List[str] = Field( - default_factory=list, description="Models currently loaded" + default_factory = list, description = "Models currently loaded" ) inference: Optional[Dict[str, Any]] = Field( - None, description="Recommended inference parameters for the active model" + None, description = "Recommended inference parameters for the active model" ) requires_trust_remote_code: bool = Field( False, - description="Whether the active model requires trust_remote_code to be enabled for loading.", + description = "Whether the active model requires trust_remote_code to be enabled for loading.", ) supports_reasoning: bool = Field( - False, description="Whether the active model supports reasoning/thinking mode" + False, description = "Whether the active model supports reasoning/thinking mode" ) reasoning_style: Literal["enable_thinking", "reasoning_effort"] = Field( "enable_thinking", - description="Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", + description = "Reasoning control style: 'enable_thinking' (boolean) or 'reasoning_effort' (low|medium|high)", ) reasoning_always_on: bool = Field( - False, description="Whether reasoning is always on (not toggleable)" + False, description = "Whether reasoning is always on (not toggleable)" ) supports_preserve_thinking: bool = Field( False, - description="Whether the active model's template understands the optional preserve_thinking kwarg", + description = "Whether the active model's template understands the optional preserve_thinking kwarg", ) supports_tools: bool = Field( - False, description="Whether the active model supports tool calling" + False, description = "Whether the active model supports tool calling" ) context_length: Optional[int] = Field( - None, description="Context length of the active model" + None, description = "Context length of the active model" ) max_context_length: Optional[int] = Field( None, - description="Maximum context length currently available for the active model", + description = "Maximum context length currently available for the active model", ) native_context_length: Optional[int] = Field( None, - description="Model's native context length from GGUF metadata (not capped by VRAM)", + description = "Model's native context length from GGUF metadata (not capped by VRAM)", ) cache_type_kv: Optional[str] = Field( None, - description="KV cache quantization dtype (e.g. 'q8_0'), or None for default", + description = "KV cache quantization dtype (e.g. 'q8_0'), or None for default", ) chat_template: Optional[str] = Field( - None, description="Model's default chat template (Jinja2 source), if any" + None, description = "Model's default chat template (Jinja2 source), if any" ) chat_template_override: Optional[str] = Field( None, - description="Active chat template override applied at load time, or None if model is using its default", + description = "Active chat template override applied at load time, or None if model is using its default", ) speculative_type: Optional[str] = Field( None, - description=( + description = ( "Canonical UI-facing requested speculative decoding mode " "('auto' / 'mtp' / 'ngram' / 'mtp+ngram' / 'off' / " "'ngram-simple'), round-tripped from the original LoadRequest. " @@ -382,32 +382,32 @@ class InferenceStatusResponse(BaseModel): ) spec_draft_n_max: Optional[int] = Field( None, - description=( + description = ( "Active --spec-draft-n-max for MTP speculative decoding, or " "None when the platform default is in effect." ), ) llama_cpp_supports_mtp: bool = Field( True, - description=( + description = ( "Whether llama.cpp supports MTP (--spec-type mtp/draft-mtp). " "False -> recommend `unsloth studio update`." ), ) llama_cpp_prebuilt_stale: bool = Field( False, - description=( + description = ( "Installed llama.cpp prebuilt is >=3 days behind the latest " "release. True -> show `unsloth studio update` banner." ), ) llama_cpp_installed_tag: Optional[str] = Field( None, - description="Installed llama.cpp tag, or None if unknown.", + description = "Installed llama.cpp tag, or None if unknown.", ) llama_cpp_latest_tag: Optional[str] = Field( None, - description="Latest published llama.cpp tag, or None if GitHub unreachable.", + description = "Latest published llama.cpp tag, or None if GitHub unreachable.", ) @@ -429,7 +429,7 @@ class TextContentPart(BaseModel): class ImageUrl(BaseModel): """Image URL object — supports data URIs and remote URLs.""" - url: str = Field(..., description="data:image/png;base64,... or https://...") + url: str = Field(..., description = "data:image/png;base64,... or https://...") detail: Optional[Literal["auto", "low", "high"]] = "auto" @@ -455,19 +455,19 @@ class InputDocumentContentPart(BaseModel): type: Literal["input_document"] file_data: Optional[str] = Field( None, - description="data:;base64, URI for inline payloads. Either file_data or file_url must be set; otherwise the part is dropped.", + description = "data:;base64, URI for inline payloads. Either file_data or file_url must be set; otherwise the part is dropped.", ) file_url: Optional[str] = Field( None, - description="Remote URL pointing to the document (https://...).", + description = "Remote URL pointing to the document (https://...).", ) filename: Optional[str] = Field( None, - description="Display filename, forwarded to providers as `title`/`filename`.", + description = "Display filename, forwarded to providers as `title`/`filename`.", ) media_type: Optional[str] = Field( None, - description='Override the media type sniffed from the data URI (e.g. "application/pdf").', + description = 'Override the media type sniffed from the data URI (e.g. "application/pdf").', ) @@ -489,7 +489,7 @@ class CompactionContentPart(BaseModel): type: Literal["compaction"] content: str = Field( ..., - description="Anthropic-produced summary of the compacted-away conversation prefix.", + description = "Anthropic-produced summary of the compacted-away conversation prefix.", ) @@ -524,25 +524,25 @@ class ChatMessage(BaseModel): """ role: Literal["system", "user", "assistant", "tool"] = Field( - ..., description="Message role" + ..., description = "Message role" ) content: Optional[Union[str, list[ContentPart]]] = Field( - None, description="Message content (string or multimodal parts)" + None, description = "Message content (string or multimodal parts)" ) tool_call_id: Optional[str] = Field( None, - description="OpenAI tool-result messages: id of the tool call this result belongs to.", + description = "OpenAI tool-result messages: id of the tool call this result belongs to.", ) tool_calls: Optional[list[dict]] = Field( None, - description="OpenAI assistant messages: structured tool calls the model decided to make.", + description = "OpenAI assistant messages: structured tool calls the model decided to make.", ) name: Optional[str] = Field( None, - description="OpenAI tool-result messages: name of the tool whose result this is.", + description = "OpenAI tool-result messages: name of the tool whose result this is.", ) - @model_validator(mode="after") + @model_validator(mode = "after") def _validate_role_shape(self) -> "ChatMessage": if self.tool_calls is not None and self.role != "assistant": raise ValueError('"tool_calls" is only valid on role="assistant" messages.') @@ -580,29 +580,29 @@ class ChatCompletionRequest(BaseModel): model: str = Field( "default", - description="Model identifier (informational; the active model is used)", + description = "Model identifier (informational; the active model is used)", ) - messages: list[ChatMessage] = Field(..., description="Conversation messages") + messages: list[ChatMessage] = Field(..., description = "Conversation messages") stream: bool = Field( False, - description=( + description = ( "Whether to stream the response via SSE. Default matches OpenAI's " "spec (`false`); opt into streaming by sending `stream: true`." ), ) - temperature: float = Field(0.6, ge=0.0, le=2.0) - top_p: float = Field(0.95, ge=0.0, le=1.0) + temperature: float = Field(0.6, ge = 0.0, le = 2.0) + top_p: float = Field(0.95, ge = 0.0, le = 1.0) max_tokens: Optional[int] = Field( - None, ge=1, description="Maximum tokens to generate (None = until EOS)" + None, ge = 1, description = "Maximum tokens to generate (None = until EOS)" ) - presence_penalty: float = Field(0.0, ge=0.0, le=2.0, description="Presence penalty") + presence_penalty: float = Field(0.0, ge = 0.0, le = 2.0, description = "Presence penalty") stop: Optional[Union[str, list[str]]] = Field( None, - description="OpenAI stop sequences: a single string or list of strings at which generation halts.", + description = "OpenAI stop sequences: a single string or list of strings at which generation halts.", ) tools: Optional[list[dict]] = Field( None, - description=( + description = ( "OpenAI function-tool definitions. When provided without `enable_tools=true`, " "Studio forwards the tools to the backend so the model returns structured " "tool_calls for the client to execute (standard OpenAI function calling)." @@ -610,29 +610,29 @@ class ChatCompletionRequest(BaseModel): ) tool_choice: Optional[Union[str, dict]] = Field( None, - description=( + description = ( "OpenAI tool choice: 'auto' | 'required' | 'none' | " "{'type': 'function', 'function': {'name': ...}}" ), ) # ── Unsloth extensions (ignored by standard OpenAI clients) ── - top_k: int = Field(20, ge=-1, le=100, description="[x-unsloth] Top-k sampling") + top_k: int = Field(20, ge = -1, le = 100, description = "[x-unsloth] Top-k sampling") min_p: float = Field( - 0.01, ge=0.0, le=1.0, description="[x-unsloth] Min-p sampling threshold" + 0.01, ge = 0.0, le = 1.0, description = "[x-unsloth] Min-p sampling threshold" ) repetition_penalty: float = Field( - 1.0, ge=1.0, le=2.0, description="[x-unsloth] Repetition penalty" + 1.0, ge = 1.0, le = 2.0, description = "[x-unsloth] Repetition penalty" ) image_base64: Optional[str] = Field( - None, description="[x-unsloth] Base64-encoded image for vision models" + None, description = "[x-unsloth] Base64-encoded image for vision models" ) audio_base64: Optional[str] = Field( - None, description="[x-unsloth] Base64-encoded WAV for audio-input models (ASR)" + None, description = "[x-unsloth] Base64-encoded WAV for audio-input models (ASR)" ) use_adapter: Optional[Union[bool, str]] = Field( None, - description=( + description = ( "[x-unsloth] Adapter control for compare mode. " "null = no change (default), " "false = disable adapters (base model), " @@ -642,25 +642,25 @@ class ChatCompletionRequest(BaseModel): ) enable_thinking: Optional[bool] = Field( None, - description="[x-unsloth] Enable/disable thinking/reasoning mode for supported models", + description = "[x-unsloth] Enable/disable thinking/reasoning mode for supported models", ) reasoning_effort: Optional[ Literal["none", "minimal", "low", "medium", "high", "max", "xhigh"] ] = Field( None, - description="[x-unsloth] Reasoning effort level ('none'|'minimal'|'low'|'medium'|'high'|'max'|'xhigh'). OpenAI `/v1/responses` accepts model-dependent subsets; Anthropic adaptive thinking uses `max` as the top tier on Claude 4.6 Opus/Sonnet (inbound `xhigh` is mapped to `max`) and `xhigh` on Claude 4.7 Opus; local Harmony/gpt-oss templates support low|medium|high.", + description = "[x-unsloth] Reasoning effort level ('none'|'minimal'|'low'|'medium'|'high'|'max'|'xhigh'). OpenAI `/v1/responses` accepts model-dependent subsets; Anthropic adaptive thinking uses `max` as the top tier on Claude 4.6 Opus/Sonnet (inbound `xhigh` is mapped to `max`) and `xhigh` on Claude 4.7 Opus; local Harmony/gpt-oss templates support low|medium|high.", ) preserve_thinking: Optional[bool] = Field( None, - description="[x-unsloth] When true, keep historical blocks from past assistant turns in the prompt (Qwen3.6 templates). Independent of enable_thinking / reasoning_effort.", + description = "[x-unsloth] When true, keep historical blocks from past assistant turns in the prompt (Qwen3.6 templates). Independent of enable_thinking / reasoning_effort.", ) enable_tools: Optional[bool] = Field( None, - description="[x-unsloth] Enable tool calling for supported models", + description = "[x-unsloth] Enable tool calling for supported models", ) enabled_tools: Optional[list[str]] = Field( None, - description=( + description = ( "[x-unsloth] List of enabled tool names. Local GGUF/safetensors models " "accept ['web_search', 'python', 'terminal', 'render_html']. External " "providers accept ['web_search', 'web_fetch', 'code_execution'] for " @@ -671,51 +671,51 @@ class ChatCompletionRequest(BaseModel): ) auto_heal_tool_calls: Optional[bool] = Field( True, - description="[x-unsloth] Auto-detect and fix malformed tool calls from model output.", + description = "[x-unsloth] Auto-detect and fix malformed tool calls from model output.", ) max_tool_calls_per_message: Optional[int] = Field( 25, - ge=0, - description="[x-unsloth] Maximum number of tool call iterations per message (0 = disabled, 9999 = unlimited).", + ge = 0, + description = "[x-unsloth] Maximum number of tool call iterations per message (0 = disabled, 9999 = unlimited).", ) tool_call_timeout: Optional[int] = Field( 300, - ge=1, - description="[x-unsloth] Timeout in seconds for each tool call execution (9999 = no limit).", + ge = 1, + description = "[x-unsloth] Timeout in seconds for each tool call execution (9999 = no limit).", ) session_id: Optional[str] = Field( None, - description="[x-unsloth] Session/thread ID for scoping tool execution sandbox.", + description = "[x-unsloth] Session/thread ID for scoping tool execution sandbox.", ) cancel_id: Optional[str] = Field( None, - description="[x-unsloth] Per-request cancellation token. Frontend sends a fresh UUID per run so /inference/cancel matches one specific generation.", + description = "[x-unsloth] Per-request cancellation token. Frontend sends a fresh UUID per run so /inference/cancel matches one specific generation.", ) # ── External provider routing (x-unsloth extensions) ────────── provider_id: Optional[str] = Field( None, - description="[x-unsloth] Saved provider config ID. If set with encrypted_api_key, routes to external LLM.", + description = "[x-unsloth] Saved provider config ID. If set with encrypted_api_key, routes to external LLM.", ) provider_type: Optional[str] = Field( None, - description="[x-unsloth] Provider type (e.g. 'openai', 'mistral'). Used if provider_id is not set.", + description = "[x-unsloth] Provider type (e.g. 'openai', 'mistral'). Used if provider_id is not set.", ) external_model: Optional[str] = Field( None, - description="[x-unsloth] Model ID at the external provider.", + description = "[x-unsloth] Model ID at the external provider.", ) encrypted_api_key: Optional[str] = Field( None, - description="[x-unsloth] RSA-encrypted, base64-encoded API key for the external provider.", + description = "[x-unsloth] RSA-encrypted, base64-encoded API key for the external provider.", ) provider_base_url: Optional[str] = Field( None, - description="[x-unsloth] Override base URL for the external provider.", + description = "[x-unsloth] Override base URL for the external provider.", ) enable_prompt_caching: Optional[bool] = Field( None, - description=( + description = ( "[x-unsloth] Opt in to provider-side prompt caching. On Anthropic, " "attaches cache_control={type:ephemeral} to the system block so the " "static prefix is reused across turns. On OpenAI cloud, caching is " @@ -726,7 +726,7 @@ class ChatCompletionRequest(BaseModel): ) prompt_cache_ttl: Optional[str] = Field( None, - description=( + description = ( "[x-unsloth] Anthropic cache_control TTL. Defaults to the 5-minute " "ephemeral pool when omitted. Pass `1h` to write into the 1-hour " "pool instead -- 1h writes are billed at 2x base input vs 1.25x " @@ -739,9 +739,9 @@ class ChatCompletionRequest(BaseModel): ) compaction_threshold: Optional[int] = Field( None, - ge=1, - le=2_000_000, - description=( + ge = 1, + le = 2_000_000, + description = ( "[x-unsloth] Server-side context compaction trigger, in tokens. " "Per-provider routing:\n" " - Anthropic (Opus 4.6+, Sonnet 4.6, Mythos preview): attaches " @@ -763,7 +763,7 @@ class ChatCompletionRequest(BaseModel): ) openai_code_exec_container_id: Optional[str] = Field( None, - description=( + description = ( "[x-unsloth] OpenAI shell-tool container id from the prior response " "in the same chat thread. When set and `code_execution` is in " "`enabled_tools`, the next /v1/responses call uses " @@ -775,7 +775,7 @@ class ChatCompletionRequest(BaseModel): ) anthropic_code_exec_container_id: Optional[str] = Field( None, - description=( + description = ( "[x-unsloth] Anthropic code_execution container id from the prior " "response in the same chat thread. When set and `code_execution` " "is in `enabled_tools`, the next /v1/messages call carries a " @@ -788,7 +788,7 @@ class ChatCompletionRequest(BaseModel): ), ) - @model_validator(mode="after") + @model_validator(mode = "after") def _resolve_missing_tool_call_ids(self) -> "ChatCompletionRequest": """Fill missing tool_call_id by walking back to the preceding assistant. @@ -873,26 +873,26 @@ class OpenAIContainerRequest(BaseModel): encrypted_api_key: str = Field( ..., - description="[x-unsloth] RSA-encrypted, base64-encoded OpenAI API key.", + description = "[x-unsloth] RSA-encrypted, base64-encoded OpenAI API key.", ) provider_base_url: Optional[str] = Field( None, - description="[x-unsloth] OpenAI base URL. Only api.openai.com is supported; non-cloud bases are rejected with 400.", + description = "[x-unsloth] OpenAI base URL. Only api.openai.com is supported; non-cloud bases are rejected with 400.", ) class CreateOpenAIContainerBody(OpenAIContainerRequest): name: str = Field( ..., - min_length=1, - max_length=256, - description="Human-readable container name. Surfaces in the picker UI.", + min_length = 1, + max_length = 256, + description = "Human-readable container name. Surfaces in the picker UI.", ) ttl_minutes: int = Field( 20, - ge=1, - le=20, - description=( + ge = 1, + le = 20, + description = ( "Idle-timeout TTL the new container will inherit (anchor=" "last_active_at). OpenAI hard-caps this at 20 minutes and " "rejects larger values with integer_above_max_value." @@ -903,7 +903,7 @@ class CreateOpenAIContainerBody(OpenAIContainerRequest): class DeleteOpenAIContainerBody(OpenAIContainerRequest): container_id: str = Field( ..., - description="OpenAI container id (cntr_...) to delete.", + description = "OpenAI container id (cntr_...) to delete.", ) @@ -943,9 +943,9 @@ class ChunkChoice(BaseModel): class ChatCompletionChunk(BaseModel): """A single SSE chunk in OpenAI streaming format.""" - id: str = Field(default_factory=lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}") + id: str = Field(default_factory = lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}") object: Literal["chat.completion.chunk"] = "chat.completion.chunk" - created: int = Field(default_factory=lambda: int(time.time())) + created: int = Field(default_factory = lambda: int(time.time())) model: str = "default" choices: list[ChunkChoice] usage: Optional[CompletionUsage] = None @@ -981,12 +981,12 @@ class CompletionUsage(BaseModel): class ChatCompletion(BaseModel): """Non-streaming chat completion response.""" - id: str = Field(default_factory=lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}") + id: str = Field(default_factory = lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}") object: Literal["chat.completion"] = "chat.completion" - created: int = Field(default_factory=lambda: int(time.time())) + created: int = Field(default_factory = lambda: int(time.time())) model: str = "default" choices: list[CompletionChoice] - usage: CompletionUsage = Field(default_factory=CompletionUsage) + usage: CompletionUsage = Field(default_factory = CompletionUsage) # ===================================================================== @@ -1008,7 +1008,7 @@ class ResponsesInputImagePart(BaseModel): """Image content part in a Responses API message (type=input_image).""" type: Literal["input_image"] - image_url: str = Field(..., description="data:image/png;base64,... or https://...") + image_url: str = Field(..., description = "data:image/png;base64,... or https://...") detail: Optional[Literal["auto", "low", "high"]] = "auto" @@ -1074,15 +1074,15 @@ class ResponsesFunctionCallInputItem(BaseModel): type: Literal["function_call"] id: Optional[str] = Field( - None, description="Item id assigned by the server (e.g. fc_...)" + None, description = "Item id assigned by the server (e.g. fc_...)" ) call_id: str = Field( ..., - description="Correlation id matching a function_call_output on the next turn.", + description = "Correlation id matching a function_call_output on the next turn.", ) name: str arguments: str = Field( - ..., description="JSON string of the arguments the model produced." + ..., description = "JSON string of the arguments the model produced." ) status: Optional[Literal["in_progress", "completed", "incomplete"]] = None @@ -1098,7 +1098,7 @@ class ResponsesFunctionCallOutputInputItem(BaseModel): id: Optional[str] = None call_id: str output: Union[str, list] = Field( - ..., description="String or content-array result of the tool call." + ..., description = "String or content-array result of the tool call." ) status: Optional[Literal["in_progress", "completed", "incomplete"]] = None @@ -1174,18 +1174,18 @@ class ResponsesFunctionTool(BaseModel): class ResponsesRequest(BaseModel): """OpenAI Responses API request.""" - model: str = Field("default", description="Model identifier") + model: str = Field("default", description = "Model identifier") input: Union[str, list[ResponsesInputItem]] = Field( - default=[], - description="Input text or list of messages / function_call / function_call_output items", + default = [], + description = "Input text or list of messages / function_call / function_call_output items", ) instructions: Optional[str] = Field( - None, description="System / developer instructions" + None, description = "System / developer instructions" ) - temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - top_p: Optional[float] = Field(None, ge=0.0, le=1.0) - max_output_tokens: Optional[int] = Field(None, ge=1) - stream: bool = Field(False, description="Whether to stream the response via SSE") + temperature: Optional[float] = Field(None, ge = 0.0, le = 2.0) + top_p: Optional[float] = Field(None, ge = 0.0, le = 1.0) + max_output_tokens: Optional[int] = Field(None, ge = 1) + stream: bool = Field(False, description = "Whether to stream the response via SSE") # OpenAI function-calling fields — forwarded to llama-server via the # Chat Completions pass-through (see routes/inference.py). Typed as a @@ -1194,7 +1194,7 @@ class ResponsesRequest(BaseModel): # picks out only ``type=="function"`` entries for forwarding. tools: Optional[list[dict]] = Field( None, - description=( + description = ( "Responses-shape function tool definitions. Entries with " '`type="function"` are translated to the Chat Completions nested ' "shape before being forwarded to llama-server; other tool types " @@ -1204,7 +1204,7 @@ class ResponsesRequest(BaseModel): ) tool_choice: Optional[Any] = Field( None, - description=( + description = ( "'auto' | 'required' | 'none' | {'type': 'function', 'name': ...} — " "the Responses-shape forcing object is translated to the Chat " "Completions nested shape internally." @@ -1231,17 +1231,17 @@ class ResponsesOutputTextContent(BaseModel): type: Literal["output_text"] = "output_text" text: str - annotations: list = Field(default_factory=list) + annotations: list = Field(default_factory = list) class ResponsesOutputMessage(BaseModel): """An output message in the Responses API response.""" type: Literal["message"] = "message" - id: str = Field(default_factory=lambda: f"msg_{uuid.uuid4().hex[:12]}") + id: str = Field(default_factory = lambda: f"msg_{uuid.uuid4().hex[:12]}") status: Literal["completed", "in_progress"] = "completed" role: Literal["assistant"] = "assistant" - content: list[ResponsesOutputTextContent] = Field(default_factory=list) + content: list[ResponsesOutputTextContent] = Field(default_factory = list) class ResponsesOutputFunctionCall(BaseModel): @@ -1254,11 +1254,11 @@ class ResponsesOutputFunctionCall(BaseModel): """ type: Literal["function_call"] = "function_call" - id: str = Field(default_factory=lambda: f"fc_{uuid.uuid4().hex[:12]}") + id: str = Field(default_factory = lambda: f"fc_{uuid.uuid4().hex[:12]}") call_id: str name: str arguments: str = Field( - ..., description="JSON string of the arguments the model produced." + ..., description = "JSON string of the arguments the model produced." ) status: Literal["completed", "in_progress", "incomplete"] = "completed" @@ -1277,24 +1277,24 @@ class ResponsesUsage(BaseModel): class ResponsesResponse(BaseModel): """Top-level Responses API response object.""" - id: str = Field(default_factory=lambda: f"resp_{uuid.uuid4().hex[:12]}") + id: str = Field(default_factory = lambda: f"resp_{uuid.uuid4().hex[:12]}") object: Literal["response"] = "response" - created_at: int = Field(default_factory=lambda: int(time.time())) + created_at: int = Field(default_factory = lambda: int(time.time())) status: Literal["completed", "in_progress", "failed"] = "completed" model: str = "default" - output: list[ResponsesOutputItem] = Field(default_factory=list) - usage: ResponsesUsage = Field(default_factory=ResponsesUsage) + output: list[ResponsesOutputItem] = Field(default_factory = list) + usage: ResponsesUsage = Field(default_factory = ResponsesUsage) error: Optional[Any] = None incomplete_details: Optional[Any] = None instructions: Optional[str] = None - metadata: dict = Field(default_factory=dict) + metadata: dict = Field(default_factory = dict) temperature: Optional[float] = None top_p: Optional[float] = None max_output_tokens: Optional[int] = None previous_response_id: Optional[str] = None text: Optional[Any] = None tool_choice: Optional[Any] = None - tools: list = Field(default_factory=list) + tools: list = Field(default_factory = list) truncation: Optional[Any] = None @@ -1373,13 +1373,13 @@ class AnthropicMessagesRequest(BaseModel): metadata: Optional[dict] = None # [x-unsloth] extensions — mirror the OpenAI endpoint convenience fields min_p: Optional[float] = Field( - None, ge=0.0, le=1.0, description="[x-unsloth] Min-p sampling threshold" + None, ge = 0.0, le = 1.0, description = "[x-unsloth] Min-p sampling threshold" ) repetition_penalty: Optional[float] = Field( - None, ge=1.0, le=2.0, description="[x-unsloth] Repetition penalty" + None, ge = 1.0, le = 2.0, description = "[x-unsloth] Repetition penalty" ) presence_penalty: Optional[float] = Field( - None, ge=0.0, le=2.0, description="[x-unsloth] Presence penalty" + None, ge = 0.0, le = 2.0, description = "[x-unsloth] Presence penalty" ) enable_tools: Optional[bool] = None enabled_tools: Optional[list[str]] = None @@ -1414,11 +1414,11 @@ AnthropicResponseBlock = Union[ class AnthropicMessagesResponse(BaseModel): - id: str = Field(default_factory=lambda: f"msg_{uuid.uuid4().hex[:24]}") + id: str = Field(default_factory = lambda: f"msg_{uuid.uuid4().hex[:24]}") type: Literal["message"] = "message" role: Literal["assistant"] = "assistant" - content: list[AnthropicResponseBlock] = Field(default_factory=list) + content: list[AnthropicResponseBlock] = Field(default_factory = list) model: str = "default" stop_reason: Optional[str] = None stop_sequence: Optional[str] = None - usage: AnthropicUsage = Field(default_factory=AnthropicUsage) + usage: AnthropicUsage = Field(default_factory = AnthropicUsage) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 715afed35d..efa9f0b6ad 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -248,8 +248,8 @@ def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict: flags = ( detect_reasoning_flags( chat_template, - model_identifier=model_id, - log_source="safetensors", + model_identifier = model_id, + log_source = "safetensors", ) if chat_template else { @@ -286,7 +286,7 @@ def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict: flags["reasoning_style"] = "reasoning_effort" flags["supports_tools"] = False except Exception: - logger.debug("gpt_oss_check_failed", exc_info=True) + logger.debug("gpt_oss_check_failed", exc_info = True) return flags @@ -458,26 +458,26 @@ def _validate_native_mmproj_companion( mm_lstat = os.lstat(mm) except OSError as exc: raise HTTPException( - status_code=400, - detail="Native vision companion is no longer accessible.", + status_code = 400, + detail = "Native vision companion is no longer accessible.", ) from exc if _stat_module.S_ISLNK(mm_lstat.st_mode) or not _stat_module.S_ISREG( mm_lstat.st_mode ): raise HTTPException( - status_code=400, - detail="Native vision companion must be a regular file.", + status_code = 400, + detail = "Native vision companion must be a regular file.", ) try: - if mm.resolve(strict=True).parent != gguf.resolve(strict=True).parent: + if mm.resolve(strict = True).parent != gguf.resolve(strict = True).parent: raise HTTPException( - status_code=400, - detail="Native vision companion must live next to the selected GGUF.", + status_code = 400, + detail = "Native vision companion must live next to the selected GGUF.", ) except OSError as exc: raise HTTPException( - status_code=400, - detail="Native vision companion is no longer accessible.", + status_code = 400, + detail = "Native vision companion is no longer accessible.", ) from exc @@ -547,13 +547,13 @@ def _resolve_model_identifier_for_request( try: grant = verify_native_path_lease( request.native_path_lease, - operation=operation, - expected_kind="model", - expected_path_type="file", - allowed_suffixes=(".gguf",), + operation = operation, + expected_kind = "model", + expected_path_type = "file", + allowed_suffixes = (".gguf",), ) except NativePathLeaseError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + raise HTTPException(status_code = 400, detail = str(exc)) from exc display_label = ( grant.display_label or Path(request.model_path).name or "Native model" ) @@ -568,7 +568,7 @@ def get_llama_cpp_backend() -> LlamaCppBackend: return _llama_cpp_backend -@router.post("/load", response_model=LoadResponse) +@router.post("/load", response_model = LoadResponse) async def load_model( request: LoadRequest, fastapi_request: Request, @@ -591,7 +591,7 @@ async def load_model( try: extra_llama_args = validate_extra_args(request.llama_extra_args) except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) + raise HTTPException(status_code = 400, detail = str(exc)) # Re-narrow []-from-None back to None so the inheritance path # below can tell "caller omitted" from "caller explicit []". extra_llama_args: Optional[list[str]] = ( @@ -599,7 +599,7 @@ async def load_model( ) model_identifier, model_log_label, native_grant_backed = ( - _resolve_model_identifier_for_request(request, operation="load-model") + _resolve_model_identifier_for_request(request, operation = "load-model") ) # Version switching is handled automatically by the subprocess-based # inference backend — no need for ensure_transformers_version() here. @@ -632,34 +632,34 @@ async def load_model( ) _gguf_is_audio = getattr(llama_backend, "_is_audio", False) return LoadResponse( - status="already_loaded", - model=model_log_label + status = "already_loaded", + model = model_log_label if native_grant_backed else llama_backend.model_identifier, - display_name=model_log_label + display_name = model_log_label if native_grant_backed else llama_backend.model_identifier, - is_vision=llama_backend._is_vision, - is_lora=False, - is_gguf=True, - is_audio=_gguf_is_audio, - audio_type=_gguf_audio, - has_audio_input=False, - inference=inference_config, - requires_trust_remote_code=bool( + is_vision = llama_backend._is_vision, + is_lora = False, + is_gguf = True, + is_audio = _gguf_is_audio, + audio_type = _gguf_audio, + has_audio_input = False, + inference = inference_config, + requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), - context_length=llama_backend.context_length, - max_context_length=llama_backend.max_context_length, - native_context_length=llama_backend.native_context_length, - supports_reasoning=llama_backend.supports_reasoning, - reasoning_style=llama_backend.reasoning_style, - reasoning_always_on=llama_backend.reasoning_always_on, - supports_preserve_thinking=llama_backend.supports_preserve_thinking, - supports_tools=llama_backend.supports_tools, - chat_template=llama_backend.chat_template, - speculative_type=llama_backend.requested_spec_mode, - spec_draft_n_max=llama_backend.spec_draft_n_max, + context_length = llama_backend.context_length, + max_context_length = llama_backend.max_context_length, + native_context_length = llama_backend.native_context_length, + supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, + reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, + supports_tools = llama_backend.supports_tools, + chat_template = llama_backend.chat_template, + speculative_type = llama_backend.requested_spec_mode, + spec_draft_n_max = llama_backend.spec_draft_n_max, ) else: if ( @@ -684,29 +684,29 @@ async def load_model( _sf_supports_reasoning = _sf_flags["supports_reasoning"] _sf_reasoning_style = _sf_flags["reasoning_style"] return LoadResponse( - status="already_loaded", - model=model_log_label + status = "already_loaded", + model = model_log_label if native_grant_backed else backend.active_model_name, - display_name=model_log_label + display_name = model_log_label if native_grant_backed else backend.active_model_name, - is_vision=_model_info.get("is_vision", False), - is_lora=_model_info.get("is_lora", False), - is_gguf=False, - is_audio=_model_info.get("is_audio", False), - audio_type=_model_info.get("audio_type"), - has_audio_input=_model_info.get("has_audio_input", False), - inference=inference_config, - requires_trust_remote_code=bool( + is_vision = _model_info.get("is_vision", False), + is_lora = _model_info.get("is_lora", False), + is_gguf = False, + is_audio = _model_info.get("is_audio", False), + audio_type = _model_info.get("audio_type"), + has_audio_input = _model_info.get("has_audio_input", False), + inference = inference_config, + requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), - supports_reasoning=_sf_supports_reasoning, - reasoning_style=_sf_reasoning_style, - reasoning_always_on=_sf_flags["reasoning_always_on"], - supports_preserve_thinking=_sf_flags["supports_preserve_thinking"], - supports_tools=_sf_flags["supports_tools"], - chat_template=_chat_template, + supports_reasoning = _sf_supports_reasoning, + reasoning_style = _sf_reasoning_style, + reasoning_always_on = _sf_flags["reasoning_always_on"], + supports_preserve_thinking = _sf_flags["supports_preserve_thinking"], + supports_tools = _sf_flags["supports_tools"], + chat_template = _chat_template, ) # is_lora auto-detected from adapter_config.json on disk/HF. @@ -714,15 +714,15 @@ async def load_model( # network checks before the worker starts. with _hf_offline_if_dns_dead(): config = ModelConfig.from_identifier( - model_id=model_identifier, - hf_token=request.hf_token, - gguf_variant=request.gguf_variant, + model_id = model_identifier, + hf_token = request.hf_token, + gguf_variant = request.gguf_variant, ) if not config: raise HTTPException( - status_code=400, - detail=f"Invalid model identifier: {model_log_label}", + status_code = 400, + detail = f"Invalid model identifier: {model_log_label}", ) # Normalize gpu_ids: empty list means auto-selection, same as None @@ -732,8 +732,8 @@ async def load_model( if config.is_gguf: if effective_gpu_ids is not None: raise HTTPException( - status_code=400, - detail="gpu_ids is not supported for GGUF models yet.", + status_code = 400, + detail = "gpu_ids is not supported for GGUF models yet.", ) llama_backend = get_llama_cpp_backend() @@ -786,13 +786,13 @@ async def load_model( fields_set = getattr(request, "model_fields_set", set()) stripped = strip_shadowing_flags( llama_backend.extra_args, - strip_context="max_seq_length" in fields_set, - strip_cache="cache_type_kv" in fields_set, - strip_spec=( + strip_context = "max_seq_length" in fields_set, + strip_cache = "cache_type_kv" in fields_set, + strip_spec = ( "speculative_type" in fields_set or "spec_draft_n_max" in fields_set ), - strip_template="chat_template_override" in fields_set, + strip_template = "chat_template_override" in fields_set, ) try: extra_llama_args = validate_extra_args(stripped) @@ -823,18 +823,18 @@ async def load_model( # HF mode: download via huggingface_hub then start llama-server success = await asyncio.to_thread( llama_backend.load_model, - hf_repo=config.gguf_hf_repo, - hf_variant=config.gguf_variant, - hf_token=request.hf_token, - model_identifier=config.identifier, - is_vision=config.is_vision, - n_ctx=request.max_seq_length, - chat_template_override=request.chat_template_override, - cache_type_kv=request.cache_type_kv, - speculative_type=request.speculative_type, - spec_draft_n_max=request.spec_draft_n_max, - n_parallel=_n_parallel, - extra_args=extra_llama_args, + hf_repo = config.gguf_hf_repo, + hf_variant = config.gguf_variant, + hf_token = request.hf_token, + model_identifier = config.identifier, + is_vision = config.is_vision, + n_ctx = request.max_seq_length, + chat_template_override = request.chat_template_override, + cache_type_kv = request.cache_type_kv, + speculative_type = request.speculative_type, + spec_draft_n_max = request.spec_draft_n_max, + n_parallel = _n_parallel, + extra_args = extra_llama_args, ) else: # Local mode: llama-server loads via -m @@ -844,27 +844,27 @@ async def load_model( ) success = await asyncio.to_thread( llama_backend.load_model, - gguf_path=config.gguf_file, - mmproj_path=config.gguf_mmproj_file, + gguf_path = config.gguf_file, + mmproj_path = config.gguf_mmproj_file, # Pass the resolved variant so _extra_args_source # is keyed off the same string the inheritance # check at the top of /load uses (#5401 followup). - hf_variant=config.gguf_variant, - model_identifier=config.identifier, - is_vision=config.is_vision, - n_ctx=request.max_seq_length, - chat_template_override=request.chat_template_override, - cache_type_kv=request.cache_type_kv, - speculative_type=request.speculative_type, - spec_draft_n_max=request.spec_draft_n_max, - n_parallel=_n_parallel, - extra_args=extra_llama_args, + hf_variant = config.gguf_variant, + model_identifier = config.identifier, + is_vision = config.is_vision, + n_ctx = request.max_seq_length, + chat_template_override = request.chat_template_override, + cache_type_kv = request.cache_type_kv, + speculative_type = request.speculative_type, + spec_draft_n_max = request.spec_draft_n_max, + n_parallel = _n_parallel, + extra_args = extra_llama_args, ) if not success: raise HTTPException( - status_code=500, - detail=f"Failed to load GGUF model: {model_log_label if native_grant_backed else config.display_name}", + status_code = 500, + detail = f"Failed to load GGUF model: {model_log_label if native_grant_backed else config.display_name}", ) logger.info( @@ -884,33 +884,33 @@ async def load_model( inference_config = load_inference_config(config.identifier) return LoadResponse( - status="loaded", - model=model_log_label if native_grant_backed else config.identifier, - display_name=model_log_label + status = "loaded", + model = model_log_label if native_grant_backed else config.identifier, + display_name = model_log_label if native_grant_backed else config.display_name, - is_vision=llama_backend.is_vision, - is_lora=False, - is_gguf=True, - is_audio=_gguf_is_audio, - audio_type=_gguf_audio, - has_audio_input=False, - inference=inference_config, - requires_trust_remote_code=bool( + is_vision = llama_backend.is_vision, + is_lora = False, + is_gguf = True, + is_audio = _gguf_is_audio, + audio_type = _gguf_audio, + has_audio_input = False, + inference = inference_config, + requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), - context_length=llama_backend.context_length, - max_context_length=llama_backend.max_context_length, - native_context_length=llama_backend.native_context_length, - supports_reasoning=llama_backend.supports_reasoning, - reasoning_style=llama_backend.reasoning_style, - reasoning_always_on=llama_backend.reasoning_always_on, - supports_preserve_thinking=llama_backend.supports_preserve_thinking, - supports_tools=llama_backend.supports_tools, - cache_type_kv=llama_backend.cache_type_kv, - chat_template=llama_backend.chat_template, - speculative_type=llama_backend.requested_spec_mode, - spec_draft_n_max=llama_backend.spec_draft_n_max, + context_length = llama_backend.context_length, + max_context_length = llama_backend.max_context_length, + native_context_length = llama_backend.native_context_length, + supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, + reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, + supports_tools = llama_backend.supports_tools, + cache_type_kv = llama_backend.cache_type_kv, + chat_template = llama_backend.chat_template, + speculative_type = llama_backend.requested_spec_mode, + spec_draft_n_max = llama_backend.spec_draft_n_max, ) # ── Standard path: load via Unsloth/transformers ────────── @@ -988,12 +988,12 @@ async def load_model( # for download progress polling and other requests. success = await asyncio.to_thread( backend.load_model, - config=config, - max_seq_length=request.max_seq_length, - load_in_4bit=load_in_4bit, - hf_token=request.hf_token, - trust_remote_code=request.trust_remote_code, - gpu_ids=effective_gpu_ids, + config = config, + max_seq_length = request.max_seq_length, + load_in_4bit = load_in_4bit, + hf_token = request.hf_token, + trust_remote_code = request.trust_remote_code, + gpu_ids = effective_gpu_ids, ) if not success: @@ -1005,15 +1005,15 @@ async def load_model( ) if yaml_trust: raise HTTPException( - status_code=400, - detail=( + status_code = 400, + detail = ( f"Model '{config.display_name}' requires trust_remote_code to be enabled. " f"Please enable 'Trust remote code' in Chat Settings and try again." ), ) raise HTTPException( - status_code=500, - detail=f"Failed to load model: {model_log_label if native_grant_backed else config.display_name}", + status_code = 500, + detail = f"Failed to load model: {model_log_label if native_grant_backed else config.display_name}", ) logger.info( @@ -1036,27 +1036,27 @@ async def load_model( _sf_flags = _detect_safetensors_features(backend, _chat_template) return LoadResponse( - status="loaded", - model=model_log_label if native_grant_backed else config.identifier, - display_name=model_log_label + status = "loaded", + model = model_log_label if native_grant_backed else config.identifier, + display_name = model_log_label if native_grant_backed else config.display_name, - is_vision=config.is_vision, - is_lora=config.is_lora, - is_gguf=False, - is_audio=config.is_audio, - audio_type=config.audio_type, - has_audio_input=config.has_audio_input, - inference=inference_config, - requires_trust_remote_code=bool( + is_vision = config.is_vision, + is_lora = config.is_lora, + is_gguf = False, + is_audio = config.is_audio, + audio_type = config.audio_type, + has_audio_input = config.has_audio_input, + inference = inference_config, + requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), - supports_reasoning=_sf_flags["supports_reasoning"], - reasoning_style=_sf_flags["reasoning_style"], - reasoning_always_on=_sf_flags["reasoning_always_on"], - supports_preserve_thinking=_sf_flags["supports_preserve_thinking"], - supports_tools=_sf_flags["supports_tools"], - chat_template=_chat_template, + supports_reasoning = _sf_flags["supports_reasoning"], + reasoning_style = _sf_flags["reasoning_style"], + reasoning_always_on = _sf_flags["reasoning_always_on"], + supports_preserve_thinking = _sf_flags["supports_preserve_thinking"], + supports_tools = _sf_flags["supports_tools"], + chat_template = _chat_template, ) except HTTPException: @@ -1069,9 +1069,9 @@ async def load_model( model_log_label, redacted_msg, ) - raise HTTPException(status_code=400, detail=redacted_msg) + raise HTTPException(status_code = 400, detail = redacted_msg) logger.warning("Rejected inference GPU selection: %s", e) - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException(status_code = 400, detail = str(e)) except Exception as e: # Surface a friendlier message for models that Unsloth cannot load not_supported_hints = [ @@ -1091,17 +1091,17 @@ async def load_model( if any(h.lower() in msg.lower() for h in not_supported_hints): msg = f"This model is not supported yet. Try a different model. (Original error: {msg})" raise HTTPException( - status_code=500, - detail=f"Failed to load native model {model_log_label}: {msg}", + status_code = 500, + detail = f"Failed to load native model {model_log_label}: {msg}", ) - logger.error(f"Error loading model: {e}", exc_info=True) + logger.error(f"Error loading model: {e}", exc_info = True) msg = str(e) if any(h.lower() in msg.lower() for h in not_supported_hints): msg = f"This model is not supported yet. Try a different model. (Original error: {msg})" - raise HTTPException(status_code=500, detail=f"Failed to load model: {msg}") + raise HTTPException(status_code = 500, detail = f"Failed to load model: {msg}") -@router.post("/validate", response_model=ValidateModelResponse) +@router.post("/validate", response_model = ValidateModelResponse) async def validate_model( request: ValidateModelRequest, current_subject: str = Depends(get_current_subject), @@ -1116,31 +1116,31 @@ async def validate_model( model_log_label = request.model_path try: model_identifier, model_log_label, native_grant_backed = ( - _resolve_model_identifier_for_request(request, operation="validate-model") + _resolve_model_identifier_for_request(request, operation = "validate-model") ) config = ModelConfig.from_identifier( - model_id=model_identifier, - hf_token=request.hf_token, - gguf_variant=request.gguf_variant, + model_id = model_identifier, + hf_token = request.hf_token, + gguf_variant = request.gguf_variant, ) if not config: raise HTTPException( - status_code=400, - detail=f"Invalid model identifier: {model_log_label}", + status_code = 400, + detail = f"Invalid model identifier: {model_log_label}", ) return ValidateModelResponse( - valid=True, - message="Model identifier is valid.", - identifier=model_log_label if native_grant_backed else config.identifier, - display_name=model_log_label + valid = True, + message = "Model identifier is valid.", + identifier = model_log_label if native_grant_backed else config.identifier, + display_name = model_log_label if native_grant_backed else getattr(config, "display_name", config.identifier), - is_gguf=getattr(config, "is_gguf", False), - is_lora=getattr(config, "is_lora", False), - is_vision=getattr(config, "is_vision", False), - requires_trust_remote_code=bool( + is_gguf = getattr(config, "is_gguf", False), + is_lora = getattr(config, "is_lora", False), + is_vision = getattr(config, "is_vision", False), + requires_trust_remote_code = bool( load_inference_config(config.identifier).get("trust_remote_code", False) ), ) @@ -1165,20 +1165,20 @@ async def validate_model( if any(h.lower() in msg.lower() for h in not_supported_hints): msg = f"This model is not supported yet. Try a different model. (Original error: {msg})" raise HTTPException( - status_code=400, - detail=f"Invalid native model {model_log_label}: {msg}", + status_code = 400, + detail = f"Invalid native model {model_log_label}: {msg}", ) logger.error( f"Error validating model identifier '{request.model_path}': {e}", - exc_info=True, + exc_info = True, ) raise HTTPException( - status_code=400, - detail=f"Invalid model: {str(e)}", + status_code = 400, + detail = f"Invalid model: {str(e)}", ) -@router.post("/unload", response_model=UnloadResponse) +@router.post("/unload", response_model = UnloadResponse) async def unload_model( request: UnloadRequest, current_subject: str = Depends(get_current_subject), @@ -1199,17 +1199,17 @@ async def unload_model( ): llama_backend.unload_model() logger.info(f"Unloaded GGUF model: {request.model_path}") - return UnloadResponse(status="unloaded", model=request.model_path) + return UnloadResponse(status = "unloaded", model = request.model_path) # Otherwise, unload from Unsloth backend backend = get_inference_backend() backend.unload_model(request.model_path) logger.info(f"Unloaded model: {request.model_path}") - return UnloadResponse(status="unloaded", model=request.model_path) + return UnloadResponse(status = "unloaded", model = request.model_path) except Exception as e: - logger.error(f"Error unloading model: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to unload model: {str(e)}") + logger.error(f"Error unloading model: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = f"Failed to unload model: {str(e)}") @studio_router.post("/cancel") @@ -1268,7 +1268,7 @@ async def generate_stream( if not backend.active_model_name: raise HTTPException( - status_code=400, detail="No model loaded. Call POST /inference/load first." + status_code = 400, detail = "No model loaded. Call POST /inference/load first." ) # Decode image if provided (for vision models) @@ -1283,8 +1283,8 @@ async def generate_stream( model_info = backend.models.get(backend.active_model_name, {}) if not model_info.get("is_vision"): raise HTTPException( - status_code=400, - detail="Image provided but current model is text-only. Load a vision model.", + status_code = 400, + detail = "Image provided but current model is text-only. Load a vision model.", ) image_data = base64.b64decode(request.image_base64) @@ -1295,40 +1295,40 @@ async def generate_stream( raise except Exception as e: raise HTTPException( - status_code=400, detail=f"Failed to decode image: {str(e)}" + status_code = 400, detail = f"Failed to decode image: {str(e)}" ) async def stream(): try: for chunk in backend.generate_chat_response( - messages=request.messages, - system_prompt=request.system_prompt, - image=image, - temperature=request.temperature, - top_p=request.top_p, - top_k=request.top_k, - max_new_tokens=request.max_new_tokens, - repetition_penalty=request.repetition_penalty, + messages = request.messages, + system_prompt = request.system_prompt, + image = image, + temperature = request.temperature, + top_p = request.top_p, + top_k = request.top_k, + max_new_tokens = request.max_new_tokens, + repetition_penalty = request.repetition_penalty, ): yield f"data: {json.dumps({'content': chunk})}\n\n" yield "data: [DONE]\n\n" except Exception as e: backend.reset_generation_state() - logger.error(f"Error during generation: {e}", exc_info=True) + logger.error(f"Error during generation: {e}", exc_info = True) yield f"data: {json.dumps({'error': _friendly_error(e)})}\n\n" return StreamingResponse( stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", }, ) -@router.get("/status", response_model=InferenceStatusResponse) +@router.get("/status", response_model = InferenceStatusResponse) async def get_status( current_subject: str = Depends(get_current_subject), ): @@ -1374,36 +1374,36 @@ async def get_status( _inference_cfg = load_inference_config(_model_id) if _model_id else None _audio_type = getattr(llama_backend, "_audio_type", None) return InferenceStatusResponse( - active_model=_display_model_id, - is_vision=llama_backend.is_vision, - is_gguf=True, - gguf_variant=llama_backend.hf_variant, - is_audio=getattr(llama_backend, "_is_audio", False), - audio_type=_audio_type, - has_audio_input=False, - loading=[], - loaded=[_display_model_id] if _display_model_id else [], - inference=_inference_cfg, - requires_trust_remote_code=bool( + active_model = _display_model_id, + is_vision = llama_backend.is_vision, + is_gguf = True, + gguf_variant = llama_backend.hf_variant, + is_audio = getattr(llama_backend, "_is_audio", False), + audio_type = _audio_type, + has_audio_input = False, + loading = [], + loaded = [_display_model_id] if _display_model_id else [], + inference = _inference_cfg, + requires_trust_remote_code = bool( (_inference_cfg or {}).get("trust_remote_code", False) ), - supports_reasoning=llama_backend.supports_reasoning, - reasoning_style=llama_backend.reasoning_style, - reasoning_always_on=llama_backend.reasoning_always_on, - supports_preserve_thinking=llama_backend.supports_preserve_thinking, - supports_tools=llama_backend.supports_tools, - chat_template=llama_backend.chat_template, - context_length=llama_backend.context_length, - max_context_length=llama_backend.max_context_length, - native_context_length=llama_backend.native_context_length, - cache_type_kv=llama_backend.cache_type_kv, - chat_template_override=llama_backend.chat_template_override, - speculative_type=llama_backend.requested_spec_mode, - spec_draft_n_max=llama_backend.spec_draft_n_max, - llama_cpp_supports_mtp=_supports_mtp, - llama_cpp_prebuilt_stale=_stale, - llama_cpp_installed_tag=_installed_tag, - llama_cpp_latest_tag=_latest_tag, + supports_reasoning = llama_backend.supports_reasoning, + reasoning_style = llama_backend.reasoning_style, + reasoning_always_on = llama_backend.reasoning_always_on, + supports_preserve_thinking = llama_backend.supports_preserve_thinking, + supports_tools = llama_backend.supports_tools, + chat_template = llama_backend.chat_template, + context_length = llama_backend.context_length, + max_context_length = llama_backend.max_context_length, + native_context_length = llama_backend.native_context_length, + cache_type_kv = llama_backend.cache_type_kv, + chat_template_override = llama_backend.chat_template_override, + speculative_type = llama_backend.requested_spec_mode, + spec_draft_n_max = llama_backend.spec_draft_n_max, + llama_cpp_supports_mtp = _supports_mtp, + llama_cpp_prebuilt_stale = _stale, + llama_cpp_installed_tag = _installed_tag, + llama_cpp_latest_tag = _latest_tag, ) # Otherwise, report Unsloth backend status @@ -1436,36 +1436,36 @@ async def get_status( ) return InferenceStatusResponse( - active_model=backend.active_model_name, - is_vision=is_vision, - is_gguf=False, - is_audio=is_audio, - audio_type=audio_type, - has_audio_input=has_audio_input, - loading=list(getattr(backend, "loading_models", set())), - loaded=list(backend.models.keys()), - inference=inference_config, - requires_trust_remote_code=bool( + active_model = backend.active_model_name, + is_vision = is_vision, + is_gguf = False, + is_audio = is_audio, + audio_type = audio_type, + has_audio_input = has_audio_input, + loading = list(getattr(backend, "loading_models", set())), + loaded = list(backend.models.keys()), + inference = inference_config, + requires_trust_remote_code = bool( (inference_config or {}).get("trust_remote_code", False) ), - supports_reasoning=_sf_flags["supports_reasoning"], - reasoning_style=_sf_flags["reasoning_style"], - reasoning_always_on=_sf_flags["reasoning_always_on"], - supports_preserve_thinking=_sf_flags["supports_preserve_thinking"], - supports_tools=_sf_flags["supports_tools"], - chat_template=chat_template, - llama_cpp_supports_mtp=_supports_mtp, - llama_cpp_prebuilt_stale=_stale, - llama_cpp_installed_tag=_installed_tag, - llama_cpp_latest_tag=_latest_tag, + supports_reasoning = _sf_flags["supports_reasoning"], + reasoning_style = _sf_flags["reasoning_style"], + reasoning_always_on = _sf_flags["reasoning_always_on"], + supports_preserve_thinking = _sf_flags["supports_preserve_thinking"], + supports_tools = _sf_flags["supports_tools"], + chat_template = chat_template, + llama_cpp_supports_mtp = _supports_mtp, + llama_cpp_prebuilt_stale = _stale, + llama_cpp_installed_tag = _installed_tag, + llama_cpp_latest_tag = _latest_tag, ) except Exception as e: - logger.error(f"Error getting status: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to get status: {str(e)}") + logger.error(f"Error getting status: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = f"Failed to get status: {str(e)}") -@router.get("/load-progress", response_model=LoadProgressResponse) +@router.get("/load-progress", response_model = LoadProgressResponse) async def get_load_progress( current_subject: str = Depends(get_current_subject), ): @@ -1514,12 +1514,12 @@ async def generate_audio( # Extract text from the last user message _, chat_messages, _ = _extract_content_parts(payload.messages) if not chat_messages: - raise HTTPException(status_code=400, detail="No messages provided.") + raise HTTPException(status_code = 400, detail = "No messages provided.") last_user_msg = next( (m for m in reversed(chat_messages) if m["role"] == "user"), None ) if not last_user_msg: - raise HTTPException(status_code=400, detail="No user message found.") + raise HTTPException(status_code = 400, detail = "No user message found.") text = last_user_msg["content"] # Pick backend — both return (wav_bytes, sample_rate) @@ -1527,34 +1527,34 @@ async def generate_audio( if llama_backend.is_loaded and getattr(llama_backend, "_is_audio", False): model_name = llama_backend.model_identifier gen = lambda: llama_backend.generate_audio_response( - text=text, - audio_type=llama_backend._audio_type, - temperature=payload.temperature, - top_p=payload.top_p, - top_k=payload.top_k, - min_p=payload.min_p, - max_new_tokens=payload.max_tokens or 2048, - repetition_penalty=payload.repetition_penalty, + text = text, + audio_type = llama_backend._audio_type, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_new_tokens = payload.max_tokens or 2048, + repetition_penalty = payload.repetition_penalty, ) else: backend = get_inference_backend() if not backend.active_model_name: - raise HTTPException(status_code=400, detail="No model loaded.") + raise HTTPException(status_code = 400, detail = "No model loaded.") model_info = backend.models.get(backend.active_model_name, {}) if not model_info.get("is_audio"): raise HTTPException( - status_code=400, detail="Active model is not an audio model." + status_code = 400, detail = "Active model is not an audio model." ) model_name = backend.active_model_name gen = lambda: backend.generate_audio_response( - text=text, - temperature=payload.temperature, - top_p=payload.top_p, - top_k=payload.top_k, - min_p=payload.min_p, - max_new_tokens=payload.max_tokens or 2048, - repetition_penalty=payload.repetition_penalty, - use_adapter=payload.use_adapter, + text = text, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_new_tokens = payload.max_tokens or 2048, + repetition_penalty = payload.repetition_penalty, + use_adapter = payload.use_adapter, ) try: @@ -1562,12 +1562,12 @@ async def generate_audio( None, gen ) except Exception as e: - logger.error(f"Audio generation error: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) + logger.error(f"Audio generation error: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = str(e)) audio_b64 = base64.b64encode(wav_bytes).decode("ascii") return JSONResponse( - content={ + content = { "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", "object": "chat.completion.audio", "model": model_name, @@ -1603,9 +1603,9 @@ def _decode_audio_base64(b64: str) -> np.ndarray: # torchaudio.load needs a file path or file-like object with format hint # Write to a temp file so torchaudio can auto-detect the format with tempfile.NamedTemporaryFile( - suffix=".audio", - delete=False, - dir=str(ensure_dir(tmp_root())), + suffix = ".audio", + delete = False, + dir = str(ensure_dir(tmp_root())), ) as tmp: tmp.write(raw) tmp_path = tmp.name @@ -1616,11 +1616,11 @@ def _decode_audio_base64(b64: str) -> np.ndarray: # Convert to mono if stereo if waveform.shape[0] > 1: - waveform = waveform.mean(dim=0, keepdim=True) + waveform = waveform.mean(dim = 0, keepdim = True) # Resample to 16kHz if needed if sr != 16000: - resampler = torchaudio.transforms.Resample(orig_freq=sr, new_freq=16000) + resampler = torchaudio.transforms.Resample(orig_freq = sr, new_freq = 16000) waveform = resampler(waveform) return waveform.squeeze(0).numpy() @@ -1800,21 +1800,21 @@ async def _proxy_to_external_provider( config = providers_db.get_provider(payload.provider_id) if config is None: raise HTTPException( - status_code=404, - detail=f"Provider config not found: {payload.provider_id}", + status_code = 404, + detail = f"Provider config not found: {payload.provider_id}", ) if not config["is_enabled"]: raise HTTPException( - status_code=400, - detail=f"Provider '{config['display_name']}' is disabled.", + status_code = 400, + detail = f"Provider '{config['display_name']}' is disabled.", ) provider_type = provider_type or config["provider_type"] base_url = base_url or config["base_url"] if not provider_type: raise HTTPException( - status_code=400, - detail="Either provider_id or provider_type is required for external provider routing.", + status_code = 400, + detail = "Either provider_id or provider_type is required for external provider routing.", ) # Fall back to registry default base URL @@ -1822,8 +1822,8 @@ async def _proxy_to_external_provider( base_url = get_base_url(provider_type) if not base_url: raise HTTPException( - status_code=400, - detail=f"Unknown provider type: {provider_type}", + status_code = 400, + detail = f"Unknown provider type: {provider_type}", ) api_key = "" @@ -1831,17 +1831,17 @@ async def _proxy_to_external_provider( try: api_key = decrypt_api_key(payload.encrypted_api_key) except Exception as exc: - logger.warning("external_provider.decrypt_failed", error=str(exc)) + logger.warning("external_provider.decrypt_failed", error = str(exc)) raise HTTPException( - status_code=400, - detail="Failed to decrypt API key. The server key may have changed — try refreshing the page.", + status_code = 400, + detail = "Failed to decrypt API key. The server key may have changed — try refreshing the page.", ) model = payload.external_model or payload.model if model == "default": raise HTTPException( - status_code=400, - detail="external_model is required when using an external provider.", + status_code = 400, + detail = "external_model is required when using an external provider.", ) # Build messages preserving multimodal content for vision-capable providers @@ -1852,33 +1852,33 @@ async def _proxy_to_external_provider( chat_messages = _build_external_messages( payload.messages, _supports_vision, - provider_type=provider_type, + provider_type = provider_type, ) client = ExternalProviderClient( - provider_type=provider_type, - base_url=base_url, - api_key=api_key, + provider_type = provider_type, + base_url = base_url, + api_key = api_key, ) async def _stream(): gen = client.stream_chat_completion( - messages=chat_messages, - model=model, - temperature=payload.temperature, - top_p=payload.top_p, - max_tokens=payload.max_tokens, - presence_penalty=payload.presence_penalty, - top_k=payload.top_k, - enable_thinking=payload.enable_thinking, - reasoning_effort=payload.reasoning_effort, - enabled_tools=payload.enabled_tools, - enable_prompt_caching=payload.enable_prompt_caching, - openai_code_exec_container_id=payload.openai_code_exec_container_id, - anthropic_code_exec_container_id=payload.anthropic_code_exec_container_id, - prompt_cache_ttl=payload.prompt_cache_ttl, - compaction_threshold=payload.compaction_threshold, - stream=payload.stream, + messages = chat_messages, + model = model, + temperature = payload.temperature, + top_p = payload.top_p, + max_tokens = payload.max_tokens, + presence_penalty = payload.presence_penalty, + top_k = payload.top_k, + enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + enabled_tools = payload.enabled_tools, + enable_prompt_caching = payload.enable_prompt_caching, + openai_code_exec_container_id = payload.openai_code_exec_container_id, + anthropic_code_exec_container_id = payload.anthropic_code_exec_container_id, + prompt_cache_ttl = payload.prompt_cache_ttl, + compaction_threshold = payload.compaction_threshold, + stream = payload.stream, ) try: sent_done = False @@ -1889,7 +1889,7 @@ async def _proxy_to_external_provider( if not sent_done: yield "data: [DONE]\n\n" except Exception as exc: - logger.error("external_provider.stream_error", error=str(exc)) + logger.error("external_provider.stream_error", error = str(exc)) finally: try: await gen.aclose() @@ -1899,8 +1899,8 @@ async def _proxy_to_external_provider( return StreamingResponse( _stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "X-Accel-Buffering": "no", }, @@ -1923,8 +1923,8 @@ def _resolve_openai_cloud_client( base_url = body.provider_base_url or get_base_url("openai") if not base_url or "api.openai.com" not in base_url: raise HTTPException( - status_code=400, - detail=( + status_code = 400, + detail = ( "OpenAI container management is only available on the " "managed cloud (api.openai.com). The provider's base URL " f"points at {base_url!r}." @@ -1933,15 +1933,15 @@ def _resolve_openai_cloud_client( try: api_key = decrypt_api_key(body.encrypted_api_key) except Exception as exc: - logger.warning("external_provider.decrypt_failed", error=str(exc)) + logger.warning("external_provider.decrypt_failed", error = str(exc)) raise HTTPException( - status_code=400, - detail="Failed to decrypt API key. The server key may have changed — try refreshing the page.", + status_code = 400, + detail = "Failed to decrypt API key. The server key may have changed — try refreshing the page.", ) return ExternalProviderClient( - provider_type="openai", - base_url=base_url, - api_key=api_key, + provider_type = "openai", + base_url = base_url, + api_key = api_key, ) @@ -1953,22 +1953,22 @@ def _summarize_container(raw: dict) -> OpenAIContainerSummary: if isinstance(minutes, int): expires_minutes = minutes return OpenAIContainerSummary( - id=str(raw.get("id") or ""), - name=raw.get("name"), - created_at=raw.get("created_at") + id = str(raw.get("id") or ""), + name = raw.get("name"), + created_at = raw.get("created_at") if isinstance(raw.get("created_at"), int) else None, - last_active_at=raw.get("last_active_at") + last_active_at = raw.get("last_active_at") if isinstance(raw.get("last_active_at"), int) else None, - expires_after_minutes=expires_minutes, - status=raw.get("status") if isinstance(raw.get("status"), str) else None, + expires_after_minutes = expires_minutes, + status = raw.get("status") if isinstance(raw.get("status"), str) else None, ) @router.post( "/external/openai/containers/list", - response_model=ListOpenAIContainersResponse, + response_model = ListOpenAIContainersResponse, ) async def list_openai_containers( body: OpenAIContainerRequest, @@ -1982,19 +1982,19 @@ async def list_openai_containers( except httpx.HTTPStatusError as exc: detail = exc.response.text[:500] if exc.response is not None else str(exc) raise HTTPException( - status_code=exc.response.status_code if exc.response else 502, - detail=f"OpenAI rejected /containers list: {detail}", + status_code = exc.response.status_code if exc.response else 502, + detail = f"OpenAI rejected /containers list: {detail}", ) except httpx.HTTPError as exc: raise HTTPException( - status_code=502, - detail=f"Failed to reach OpenAI: {exc}", + status_code = 502, + detail = f"Failed to reach OpenAI: {exc}", ) # OpenAI keeps expired containers in /v1/containers indefinitely # with status="expired" — they're effectively dead but still # listed. Hide them so the picker only shows usable containers. return ListOpenAIContainersResponse( - containers=[ + containers = [ _summarize_container(c) for c in raw if isinstance(c, dict) and c.get("status") != "expired" @@ -2006,7 +2006,7 @@ async def list_openai_containers( @router.post( "/external/openai/containers/create", - response_model=OpenAIContainerSummary, + response_model = OpenAIContainerSummary, ) async def create_openai_container( body: CreateOpenAIContainerBody, @@ -2017,31 +2017,31 @@ async def create_openai_container( try: try: raw = await client.create_openai_container( - name=body.name, - ttl_minutes=body.ttl_minutes, + name = body.name, + ttl_minutes = body.ttl_minutes, ) except httpx.HTTPStatusError as exc: detail = exc.response.text[:500] if exc.response is not None else str(exc) raise HTTPException( - status_code=exc.response.status_code if exc.response else 502, - detail=f"OpenAI rejected /containers create: {detail}", + status_code = exc.response.status_code if exc.response else 502, + detail = f"OpenAI rejected /containers create: {detail}", ) except httpx.HTTPError as exc: raise HTTPException( - status_code=502, - detail=f"Failed to reach OpenAI: {exc}", + status_code = 502, + detail = f"Failed to reach OpenAI: {exc}", ) if not isinstance(raw, dict): raise HTTPException( - status_code=502, - detail="OpenAI returned an unexpected container payload.", + status_code = 502, + detail = "OpenAI returned an unexpected container payload.", ) return _summarize_container(raw) finally: await client.close() -@router.post("/external/openai/containers/delete", status_code=204) +@router.post("/external/openai/containers/delete", status_code = 204) async def delete_openai_container( body: DeleteOpenAIContainerBody, current_subject: str = Depends(get_current_subject), @@ -2070,8 +2070,8 @@ async def delete_openai_container( detail, ) raise HTTPException( - status_code=exc.response.status_code if exc.response else 502, - detail=f"OpenAI rejected /containers delete: {detail}", + status_code = exc.response.status_code if exc.response else 502, + detail = f"OpenAI rejected /containers delete: {detail}", ) except httpx.HTTPError as exc: logger.warning( @@ -2080,8 +2080,8 @@ async def delete_openai_container( exc, ) raise HTTPException( - status_code=502, - detail=f"Failed to reach OpenAI: {exc}", + status_code = 502, + detail = f"Failed to reach OpenAI: {exc}", ) finally: await client.close() @@ -2140,8 +2140,8 @@ async def openai_chat_completions( backend = get_inference_backend() if not backend.active_model_name: raise HTTPException( - status_code=400, - detail="No model loaded. Call POST /inference/load first.", + status_code = 400, + detail = "No model loaded. Call POST /inference/load first.", ) model_name = backend.active_model_name or payload.model @@ -2154,8 +2154,8 @@ async def openai_chat_completions( # ── Whisper without audio: return clear error ── if model_info.get("audio_type") == "whisper" and not payload.audio_base64: raise HTTPException( - status_code=400, - detail="Whisper models require audio input. Please upload an audio file.", + status_code = 400, + detail = "Whisper models require audio input. Please upload an audio file.", ) # ── Audio INPUT path: decode WAV and route to audio input generation ── @@ -2169,20 +2169,20 @@ async def openai_chat_completions( def audio_input_generate(): if model_info.get("audio_type") == "whisper": return backend.generate_whisper_response( - audio_array=audio_array, - cancel_event=cancel_event, + audio_array = audio_array, + cancel_event = cancel_event, ) return backend.generate_audio_input_response( - messages=chat_messages, - system_prompt=system_prompt, - audio_array=audio_array, - temperature=payload.temperature, - top_p=payload.top_p, - top_k=payload.top_k, - min_p=payload.min_p, - max_new_tokens=payload.max_tokens or 2048, - repetition_penalty=payload.repetition_penalty, - cancel_event=cancel_event, + messages = chat_messages, + system_prompt = system_prompt, + audio_array = audio_array, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_new_tokens = payload.max_tokens or 2048, + repetition_penalty = payload.repetition_penalty, + cancel_event = cancel_event, ) if payload.stream: @@ -2193,17 +2193,17 @@ async def openai_chat_completions( async def audio_input_stream(): try: first_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(role="assistant"), - finish_reason=None, + delta = ChoiceDelta(role = "assistant"), + finish_reason = None, ) ], ) - yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" gen = audio_input_generate() _DONE = object() @@ -2218,34 +2218,34 @@ async def openai_chat_completions( break if chunk_text: chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(content=chunk_text), - finish_reason=None, + delta = ChoiceDelta(content = chunk_text), + finish_reason = None, ) ], ) - yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {chunk.model_dump_json(exclude_none = True)}\n\n" final_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ - ChunkChoice(delta=ChoiceDelta(), finish_reason="stop") + id = completion_id, + created = created, + model = model_name, + choices = [ + ChunkChoice(delta = ChoiceDelta(), finish_reason = "stop") ], ) - yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {final_chunk.model_dump_json(exclude_none = True)}\n\n" yield "data: [DONE]\n\n" except asyncio.CancelledError: cancel_event.set() raise except Exception as e: logger.error( - f"Error during audio input streaming: {e}", exc_info=True + f"Error during audio input streaming: {e}", exc_info = True ) yield f"data: {json.dumps({'error': {'message': _friendly_error(e), 'type': 'server_error'}})}\n\n" finally: @@ -2253,8 +2253,8 @@ async def openai_chat_completions( return StreamingResponse( audio_input_stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -2263,17 +2263,17 @@ async def openai_chat_completions( else: full_text = "".join(audio_input_generate()) response = ChatCompletion( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ CompletionChoice( - message=CompletionMessage(content=full_text), - finish_reason="stop", + message = CompletionMessage(content = full_text), + finish_reason = "stop", ) ], ) - return JSONResponse(content=response.model_dump()) + return JSONResponse(content = response.model_dump()) # ── Standard OpenAI function-calling pass-through (GGUF only) ──── # When a client (opencode / Claude Code via OpenAI compat / Cursor / @@ -2304,8 +2304,8 @@ async def openai_chat_completions( ): if payload.audio_base64: raise HTTPException( - status_code=400, - detail="Audio input is not supported for GGUF chat models yet.", + status_code = 400, + detail = "Audio input is not supported for GGUF chat models yet.", ) # Preserve the vision guard that would otherwise run in the @@ -2321,8 +2321,8 @@ async def openai_chat_completions( ) ): raise HTTPException( - status_code=400, - detail="Image provided but current GGUF model does not support vision.", + status_code = 400, + detail = "Image provided but current GGUF model does not support vision.", ) cancel_event = threading.Event() @@ -2352,16 +2352,16 @@ async def openai_chat_completions( if not chat_messages: raise HTTPException( - status_code=400, - detail="At least one non-system message is required.", + status_code = 400, + detail = "At least one non-system message is required.", ) # ── GGUF path: proxy to llama-server /v1/chat/completions ── if using_gguf: if payload.audio_base64: raise HTTPException( - status_code=400, - detail="Audio input is not supported for GGUF chat models yet.", + status_code = 400, + detail = "Audio input is not supported for GGUF chat models yet.", ) gguf_messages, has_gguf_image = _openai_messages_for_gguf_chat( @@ -2468,29 +2468,29 @@ async def openai_chat_completions( def gguf_generate_with_tools(): return llama_backend.generate_chat_completion_with_tools( - messages=gguf_messages, - tools=tools_to_use, - temperature=payload.temperature, - top_p=payload.top_p, - top_k=payload.top_k, - min_p=payload.min_p, - max_tokens=payload.max_tokens, - repetition_penalty=payload.repetition_penalty, - presence_penalty=payload.presence_penalty, - cancel_event=cancel_event, - enable_thinking=payload.enable_thinking, - reasoning_effort=payload.reasoning_effort, - preserve_thinking=payload.preserve_thinking, - auto_heal_tool_calls=payload.auto_heal_tool_calls + messages = gguf_messages, + tools = tools_to_use, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_tokens = payload.max_tokens, + repetition_penalty = payload.repetition_penalty, + presence_penalty = payload.presence_penalty, + cancel_event = cancel_event, + enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, + auto_heal_tool_calls = payload.auto_heal_tool_calls if payload.auto_heal_tool_calls is not None else True, - max_tool_iterations=payload.max_tool_calls_per_message + max_tool_iterations = payload.max_tool_calls_per_message if payload.max_tool_calls_per_message is not None else 25, - tool_call_timeout=payload.tool_call_timeout + tool_call_timeout = payload.tool_call_timeout if payload.tool_call_timeout is not None else 300, - session_id=payload.session_id, + session_id = payload.session_id, ) _tool_sentinel = object() @@ -2502,17 +2502,17 @@ async def openai_chat_completions( async def gguf_tool_stream(): try: first_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(role="assistant"), - finish_reason=None, + delta = ChoiceDelta(role = "assistant"), + finish_reason = None, ) ], ) - yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" # Iterate the synchronous generator in a thread so # the event loop stays free for disconnect detection. @@ -2571,48 +2571,48 @@ async def openai_chat_completions( if not new_text: continue chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(content=new_text), - finish_reason=None, + delta = ChoiceDelta(content = new_text), + finish_reason = None, ) ], ) - yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {chunk.model_dump_json(exclude_none = True)}\n\n" final_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(), - finish_reason="stop", + delta = ChoiceDelta(), + finish_reason = "stop", ) ], ) - yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {final_chunk.model_dump_json(exclude_none = True)}\n\n" # Usage chunk (OpenAI-standard: choices=[], usage populated) if _stream_usage or _stream_timings: usage_obj = CompletionUsage( - prompt_tokens=(_stream_usage or {}).get("prompt_tokens", 0), - completion_tokens=(_stream_usage or {}).get( + prompt_tokens = (_stream_usage or {}).get("prompt_tokens", 0), + completion_tokens = (_stream_usage or {}).get( "completion_tokens", 0 ), - total_tokens=(_stream_usage or {}).get("total_tokens", 0), + total_tokens = (_stream_usage or {}).get("total_tokens", 0), ) usage_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[], - usage=usage_obj, - timings=_stream_timings, + id = completion_id, + created = created, + model = model_name, + choices = [], + usage = usage_obj, + timings = _stream_timings, ) - yield f"data: {usage_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {usage_chunk.model_dump_json(exclude_none = True)}\n\n" yield "data: [DONE]\n\n" except asyncio.CancelledError: @@ -2635,8 +2635,8 @@ async def openai_chat_completions( return StreamingResponse( gguf_tool_stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -2647,19 +2647,19 @@ async def openai_chat_completions( def gguf_generate(): return llama_backend.generate_chat_completion( - messages=gguf_messages, - image_b64=image_b64, - temperature=payload.temperature, - top_p=payload.top_p, - top_k=payload.top_k, - min_p=payload.min_p, - max_tokens=payload.max_tokens, - repetition_penalty=payload.repetition_penalty, - presence_penalty=payload.presence_penalty, - cancel_event=cancel_event, - enable_thinking=payload.enable_thinking, - reasoning_effort=payload.reasoning_effort, - preserve_thinking=payload.preserve_thinking, + messages = gguf_messages, + image_b64 = image_b64, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_tokens = payload.max_tokens, + repetition_penalty = payload.repetition_penalty, + presence_penalty = payload.presence_penalty, + cancel_event = cancel_event, + enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, ) _gguf_sentinel = object() @@ -2673,17 +2673,17 @@ async def openai_chat_completions( try: # First chunk: role first_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(role="assistant"), - finish_reason=None, + delta = ChoiceDelta(role = "assistant"), + finish_reason = None, ) ], ) - yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" # Iterate the synchronous generator in a thread so # the event loop stays free for disconnect detection. @@ -2720,56 +2720,56 @@ async def openai_chat_completions( if not new_text: continue chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(content=new_text), - finish_reason=None, + delta = ChoiceDelta(content = new_text), + finish_reason = None, ) ], ) - yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {chunk.model_dump_json(exclude_none = True)}\n\n" # Final chunk final_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(), - finish_reason="stop", + delta = ChoiceDelta(), + finish_reason = "stop", ) ], ) - yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {final_chunk.model_dump_json(exclude_none = True)}\n\n" # Usage chunk (OpenAI-standard: choices=[], usage populated) if _stream_usage or _stream_timings: usage_obj = CompletionUsage( - prompt_tokens=(_stream_usage or {}).get("prompt_tokens", 0), - completion_tokens=(_stream_usage or {}).get( + prompt_tokens = (_stream_usage or {}).get("prompt_tokens", 0), + completion_tokens = (_stream_usage or {}).get( "completion_tokens", 0 ), - total_tokens=(_stream_usage or {}).get("total_tokens", 0), + total_tokens = (_stream_usage or {}).get("total_tokens", 0), ) usage_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[], - usage=usage_obj, - timings=_stream_timings, + id = completion_id, + created = created, + model = model_name, + choices = [], + usage = usage_obj, + timings = _stream_timings, ) - yield f"data: {usage_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {usage_chunk.model_dump_json(exclude_none = True)}\n\n" yield "data: [DONE]\n\n" except asyncio.CancelledError: cancel_event.set() raise except Exception as e: - logger.error(f"Error during GGUF streaming: {e}", exc_info=True) + logger.error(f"Error during GGUF streaming: {e}", exc_info = True) error_chunk = { "error": { "message": _friendly_error(e), @@ -2782,8 +2782,8 @@ async def openai_chat_completions( return StreamingResponse( gguf_stream_chunks(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -2798,21 +2798,21 @@ async def openai_chat_completions( full_text = token response = ChatCompletion( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ CompletionChoice( - message=CompletionMessage(content=full_text), - finish_reason="stop", + message = CompletionMessage(content = full_text), + finish_reason = "stop", ) ], ) - return JSONResponse(content=response.model_dump()) + return JSONResponse(content = response.model_dump()) except Exception as e: - logger.error(f"Error during GGUF completion: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) + logger.error(f"Error during GGUF completion: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = str(e)) # ── Standard Unsloth path ───────────────────────────────── @@ -2829,8 +2829,8 @@ async def openai_chat_completions( model_info = backend.models.get(backend.active_model_name, {}) if not model_info.get("is_vision"): raise HTTPException( - status_code=400, - detail="Image provided but current model is text-only. Load a vision model.", + status_code = 400, + detail = "Image provided but current model is text-only. Load a vision model.", ) image_data = base64.b64decode(image_b64) @@ -2840,7 +2840,7 @@ async def openai_chat_completions( except HTTPException: raise except Exception as e: - raise HTTPException(status_code=400, detail=f"Failed to decode image: {e}") + raise HTTPException(status_code = 400, detail = f"Failed to decode image: {e}") # Classify capability flags from the loaded template. _sf_model_info = backend.models.get(backend.active_model_name, {}) @@ -2956,28 +2956,28 @@ async def openai_chat_completions( def sf_generate_with_tools(): return backend.generate_chat_completion_with_tools( - messages=_sf_chat_messages, - tools=_sf_tools_to_use, - system_prompt=_sf_system_prompt or "", - temperature=payload.temperature, - top_p=payload.top_p, - top_k=payload.top_k, - min_p=payload.min_p, - max_tokens=payload.max_tokens, - repetition_penalty=payload.repetition_penalty, - cancel_event=cancel_event, - enable_thinking=payload.enable_thinking, - reasoning_effort=payload.reasoning_effort, - preserve_thinking=payload.preserve_thinking, - auto_heal_tool_calls=payload.auto_heal_tool_calls + messages = _sf_chat_messages, + tools = _sf_tools_to_use, + system_prompt = _sf_system_prompt or "", + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_tokens = payload.max_tokens, + repetition_penalty = payload.repetition_penalty, + cancel_event = cancel_event, + enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, + auto_heal_tool_calls = payload.auto_heal_tool_calls if payload.auto_heal_tool_calls is not None else True, - max_tool_iterations=_sf_tool_budget, - tool_call_timeout=payload.tool_call_timeout + max_tool_iterations = _sf_tool_budget, + tool_call_timeout = payload.tool_call_timeout if payload.tool_call_timeout is not None else 300, - session_id=payload.session_id, - use_adapter=payload.use_adapter, + session_id = payload.session_id, + use_adapter = payload.use_adapter, ) _sf_tool_sentinel = object() @@ -2988,17 +2988,17 @@ async def openai_chat_completions( async def sf_tool_stream(): try: first_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(role="assistant"), - finish_reason=None, + delta = ChoiceDelta(role = "assistant"), + finish_reason = None, ) ], ) - yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" gen = sf_generate_with_tools() prev_text = "" @@ -3041,30 +3041,30 @@ async def openai_chat_completions( if not new_text: continue chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(content=new_text), - finish_reason=None, + delta = ChoiceDelta(content = new_text), + finish_reason = None, ) ], ) - yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {chunk.model_dump_json(exclude_none = True)}\n\n" final_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(), - finish_reason="stop", + delta = ChoiceDelta(), + finish_reason = "stop", ) ], ) - yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {final_chunk.model_dump_json(exclude_none = True)}\n\n" yield "data: [DONE]\n\n" except asyncio.CancelledError: @@ -3089,8 +3089,8 @@ async def openai_chat_completions( if payload.stream: return StreamingResponse( sf_tool_stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -3112,39 +3112,39 @@ async def openai_chat_completions( content_text = await asyncio.to_thread(_drain_to_text) response = ChatCompletion( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ CompletionChoice( - message=CompletionMessage(content=content_text), - finish_reason="stop", + message = CompletionMessage(content = content_text), + finish_reason = "stop", ) ], ) - return JSONResponse(content=response.model_dump()) + return JSONResponse(content = response.model_dump()) except Exception: backend.reset_generation_state() # CWE-209: generic detail; full trace in log. logger.exception("safetensors tool completion error") raise HTTPException( - status_code=500, - detail="An internal error occurred.", + status_code = 500, + detail = "An internal error occurred.", ) finally: _sf_tracker.__exit__(None, None, None) # Shared generation kwargs gen_kwargs = dict( - messages=chat_messages, - system_prompt=system_prompt, - image=image, - temperature=payload.temperature, - top_p=payload.top_p, - top_k=payload.top_k, - min_p=payload.min_p, - max_new_tokens=payload.max_tokens or 2048, - repetition_penalty=payload.repetition_penalty, + messages = chat_messages, + system_prompt = system_prompt, + image = image, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_new_tokens = payload.max_tokens or 2048, + repetition_penalty = payload.repetition_penalty, ) # Forward reasoning kwargs; the worker/template wrapper peels off # any the template doesn't accept. @@ -3159,15 +3159,15 @@ async def openai_chat_completions( def generate(): return backend.generate_with_adapter_control( - use_adapter=payload.use_adapter, - cancel_event=cancel_event, + use_adapter = payload.use_adapter, + cancel_event = cancel_event, **gen_kwargs, ) else: def generate(): return backend.generate_chat_response( - cancel_event=cancel_event, **gen_kwargs + cancel_event = cancel_event, **gen_kwargs ) # ── Streaming response ──────────────────────────────────────── @@ -3179,17 +3179,17 @@ async def openai_chat_completions( async def stream_chunks(): try: first_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(role="assistant"), - finish_reason=None, + delta = ChoiceDelta(role = "assistant"), + finish_reason = None, ) ], ) - yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" prev_text = "" # Run sync generator in thread pool to avoid blocking @@ -3220,30 +3220,30 @@ async def openai_chat_completions( if not new_text: continue chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(content=new_text), - finish_reason=None, + delta = ChoiceDelta(content = new_text), + finish_reason = None, ) ], ) - yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {chunk.model_dump_json(exclude_none = True)}\n\n" final_chunk = ChatCompletionChunk( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ ChunkChoice( - delta=ChoiceDelta(), - finish_reason="stop", + delta = ChoiceDelta(), + finish_reason = "stop", ) ], ) - yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n" + yield f"data: {final_chunk.model_dump_json(exclude_none = True)}\n\n" yield "data: [DONE]\n\n" except asyncio.CancelledError: @@ -3252,7 +3252,7 @@ async def openai_chat_completions( raise except Exception as e: backend.reset_generation_state() - logger.error(f"Error during OpenAI streaming: {e}", exc_info=True) + logger.error(f"Error during OpenAI streaming: {e}", exc_info = True) error_chunk = { "error": { "message": _friendly_error(e), @@ -3265,8 +3265,8 @@ async def openai_chat_completions( return StreamingResponse( stream_chunks(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -3281,22 +3281,22 @@ async def openai_chat_completions( full_text = token response = ChatCompletion( - id=completion_id, - created=created, - model=model_name, - choices=[ + id = completion_id, + created = created, + model = model_name, + choices = [ CompletionChoice( - message=CompletionMessage(content=full_text), - finish_reason="stop", + message = CompletionMessage(content = full_text), + finish_reason = "stop", ) ], ) - return JSONResponse(content=response.model_dump()) + return JSONResponse(content = response.model_dump()) except Exception as e: backend.reset_generation_state() - logger.error(f"Error during OpenAI completion: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) + logger.error(f"Error during OpenAI completion: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = str(e)) # ===================================================================== @@ -3336,26 +3336,26 @@ async def serve_sandbox_file( jwt_token = token else: raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Missing authentication token", + status_code = status.HTTP_401_UNAUTHORIZED, + detail = "Missing authentication token", ) from fastapi.security import HTTPAuthorizationCredentials - creds = HTTPAuthorizationCredentials(scheme="Bearer", credentials=jwt_token) + creds = HTTPAuthorizationCredentials(scheme = "Bearer", credentials = jwt_token) await get_current_subject(creds) # ── Filename sanitization ─────────────────────────────────── safe_filename = os.path.basename(filename) if not safe_filename or safe_filename in (".", ".."): - raise HTTPException(status_code=404, detail="Not found") + raise HTTPException(status_code = 404, detail = "Not found") # ── Extension allowlist ───────────────────────────────────── ext = os.path.splitext(safe_filename)[1].lower() media_type = _SANDBOX_MEDIA_TYPES.get(ext) if not media_type: raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="File type not allowed", + status_code = status.HTTP_403_FORBIDDEN, + detail = "File type not allowed", ) # ── Path containment check ────────────────────────────────── @@ -3363,24 +3363,24 @@ async def serve_sandbox_file( sandbox_root = os.path.realpath(os.path.join(home, "studio_sandbox")) safe_session = os.path.basename(session_id.replace("..", "")) if not safe_session: - raise HTTPException(status_code=404, detail="Not found") + raise HTTPException(status_code = 404, detail = "Not found") file_path = os.path.realpath( os.path.join(sandbox_root, safe_session, safe_filename) ) if not file_path.startswith(sandbox_root + os.sep): raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Access denied", + status_code = status.HTTP_403_FORBIDDEN, + detail = "Access denied", ) if not os.path.isfile(file_path): - raise HTTPException(status_code=404, detail="Not found") + raise HTTPException(status_code = 404, detail = "Not found") return FileResponse( - path=file_path, - media_type=media_type, - headers={ + path = file_path, + media_type = media_type, + headers = { "Cache-Control": "private, no-store", "X-Content-Type-Options": "nosniff", }, @@ -3448,8 +3448,8 @@ async def openai_completions( llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: raise HTTPException( - status_code=503, - detail="No GGUF model loaded. Load a GGUF model first.", + status_code = 503, + detail = "No GGUF model loaded. Load a GGUF model first.", ) body = await request.json() @@ -3468,12 +3468,12 @@ async def openai_completions( # cancel-scope trace: an anonymous async for leaves the # iterator unclosed, so Python's asyncgen GC finalizer runs # cleanup on a later pass in a different asyncio task. - client = httpx.AsyncClient(timeout=600) + client = httpx.AsyncClient(timeout = 600) resp = None bytes_iter = None try: - req = client.build_request("POST", target_url, json=body) - resp = await client.send(req, stream=True) + req = client.build_request("POST", target_url, json = body) + resp = await client.send(req, stream = True) bytes_iter = resp.aiter_bytes() async for chunk in bytes_iter: yield chunk @@ -3495,14 +3495,14 @@ async def openai_completions( except Exception: pass - return StreamingResponse(_stream(), media_type="text/event-stream") + return StreamingResponse(_stream(), media_type = "text/event-stream") else: async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json=body, timeout=600) + resp = await client.post(target_url, json = body, timeout = 600) return Response( - content=resp.content, - status_code=resp.status_code, - media_type="application/json", + content = resp.content, + status_code = resp.status_code, + media_type = "application/json", ) @@ -3527,19 +3527,19 @@ async def openai_embeddings( llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: raise HTTPException( - status_code=503, - detail="No GGUF model loaded. Load a GGUF model first.", + status_code = 503, + detail = "No GGUF model loaded. Load a GGUF model first.", ) body = await request.json() target_url = f"{llama_backend.base_url}/v1/embeddings" async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json=body, timeout=600) + resp = await client.post(target_url, json = body, timeout = 600) return Response( - content=resp.content, - status_code=resp.status_code, - media_type="application/json", + content = resp.content, + status_code = resp.status_code, + media_type = "application/json", ) @@ -3662,19 +3662,19 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]: # Simple string input if isinstance(payload.input, str): if payload.input: - messages.append(ChatMessage(role="user", content=payload.input)) + messages.append(ChatMessage(role = "user", content = payload.input)) if system_parts: merged = "\n\n".join(p for p in system_parts if p) - return [ChatMessage(role="system", content=merged), *messages] + return [ChatMessage(role = "system", content = merged), *messages] return messages for item in payload.input: if isinstance(item, ResponsesFunctionCallInputItem): messages.append( ChatMessage( - role="assistant", - content=None, - tool_calls=[ + role = "assistant", + content = None, + tool_calls = [ { "id": item.call_id, "type": "function", @@ -3696,9 +3696,9 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]: output = json.dumps(output) messages.append( ChatMessage( - role="tool", - tool_call_id=item.call_id, - content=output, + role = "tool", + tool_call_id = item.call_id, + content = output, ) ) continue @@ -3718,7 +3718,7 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]: continue if isinstance(item.content, str): - messages.append(ChatMessage(role=item.role, content=item.content)) + messages.append(ChatMessage(role = item.role, content = item.content)) continue # Assistant-replay turns come back as content = [output_text, ...]. @@ -3728,7 +3728,7 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]: if item.role == "assistant": text = _responses_message_text(item.content) if text: - messages.append(ChatMessage(role="assistant", content=text)) + messages.append(ChatMessage(role = "assistant", content = text)) continue # User (and any other remaining roles) — keep multimodal when @@ -3736,12 +3736,12 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]: parts: list = [] for part in item.content: if isinstance(part, (ResponsesInputTextPart, ResponsesOutputTextPart)): - parts.append(TextContentPart(type="text", text=part.text)) + parts.append(TextContentPart(type = "text", text = part.text)) elif isinstance(part, ResponsesInputImagePart): parts.append( ImageContentPart( - type="image_url", - image_url=ImageUrl(url=part.image_url, detail=part.detail), + type = "image_url", + image_url = ImageUrl(url = part.image_url, detail = part.detail), ) ) # ResponsesUnknownContentPart and anything else: drop. @@ -3750,13 +3750,13 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]: # that reject multimodal arrays (e.g. legacy templates) still # accept the message. if len(parts) == 1 and isinstance(parts[0], TextContentPart): - messages.append(ChatMessage(role=item.role, content=parts[0].text)) + messages.append(ChatMessage(role = item.role, content = parts[0].text)) else: - messages.append(ChatMessage(role=item.role, content=parts)) + messages.append(ChatMessage(role = item.role, content = parts)) if system_parts: merged = "\n\n".join(p for p in system_parts if p) - return [ChatMessage(role="system", content=merged), *messages] + return [ChatMessage(role = "system", content = merged), *messages] return messages @@ -3771,9 +3771,9 @@ def _build_chat_request( further modification. """ chat_kwargs: dict = dict( - model=payload.model, - messages=messages, - stream=stream, + model = payload.model, + messages = messages, + stream = stream, ) if payload.temperature is not None: chat_kwargs["temperature"] = payload.temperature @@ -3814,10 +3814,10 @@ def _chat_tool_calls_to_responses_output(tool_calls: list[dict]) -> list[dict]: fn = tc.get("function") or {} items.append( ResponsesOutputFunctionCall( - call_id=tc.get("id", ""), - name=fn.get("name", ""), - arguments=fn.get("arguments", "") or "", - status="completed", + call_id = tc.get("id", ""), + name = fn.get("name", ""), + arguments = fn.get("arguments", "") or "", + status = "completed", ).model_dump() ) return items @@ -3829,7 +3829,7 @@ async def _responses_non_streaming( request: Request, ) -> JSONResponse: """Handle a non-streaming Responses API call.""" - chat_req = _build_chat_request(payload, messages, stream=False) + chat_req = _build_chat_request(payload, messages, stream = False) result = await openai_chat_completions(chat_req, request) # openai_chat_completions returns a JSONResponse for non-streaming @@ -3864,31 +3864,31 @@ async def _responses_non_streaming( msg_id = f"msg_{uuid.uuid4().hex[:12]}" output_items.append( ResponsesOutputMessage( - id=msg_id, - status="completed", - role="assistant", - content=[ResponsesOutputTextContent(text=text)], + id = msg_id, + status = "completed", + role = "assistant", + content = [ResponsesOutputTextContent(text = text)], ).model_dump() ) output_items.extend(_chat_tool_calls_to_responses_output(tool_calls)) response = ResponsesResponse( - id=resp_id, - created_at=int(time.time()), - status="completed", - model=body.get("model", payload.model), - output=output_items, - usage=ResponsesUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=input_tokens + output_tokens, + id = resp_id, + created_at = int(time.time()), + status = "completed", + model = body.get("model", payload.model), + output = output_items, + usage = ResponsesUsage( + input_tokens = input_tokens, + output_tokens = output_tokens, + total_tokens = input_tokens + output_tokens, ), - temperature=payload.temperature, - top_p=payload.top_p, - max_output_tokens=payload.max_output_tokens, - instructions=payload.instructions, + temperature = payload.temperature, + top_p = payload.top_p, + max_output_tokens = payload.max_output_tokens, + instructions = payload.instructions, ) - return JSONResponse(content=response.model_dump()) + return JSONResponse(content = response.model_dump()) async def _responses_stream( @@ -3922,7 +3922,7 @@ async def _responses_stream( msg_id = f"msg_{uuid.uuid4().hex[:12]}" created_at = int(time.time()) - chat_req = _build_chat_request(payload, messages, stream=True) + chat_req = _build_chat_request(payload, messages, stream = True) llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: @@ -3934,8 +3934,8 @@ async def _responses_stream( # 3.13. Surface a typed 400 so the client sees a useful error # instead of a dangling stream. raise HTTPException( - status_code=400, - detail=( + status_code = 400, + detail = ( "Streaming /v1/responses requires a GGUF model loaded via " "llama-server. Use non-streaming /v1/responses, " "/v1/chat/completions, or load a GGUF model." @@ -3949,12 +3949,12 @@ async def _responses_stream( for m in messages ): raise HTTPException( - status_code=400, - detail="Image provided but current GGUF model does not support vision.", + status_code = 400, + detail = "Image provided but current GGUF model does not support vision.", ) body = _build_openai_passthrough_body( - chat_req, backend_ctx=llama_backend.context_length + chat_req, backend_ctx = llama_backend.context_length ) target_url = f"{llama_backend.base_url}/v1/chat/completions" @@ -3986,7 +3986,7 @@ async def _responses_stream( ], } ] - for st in sorted(tool_call_state.values(), key=lambda s: s["output_index"]): + for st in sorted(tool_call_state.values(), key = lambda s: s["output_index"]): items.append( { "type": "function_call", @@ -4022,13 +4022,13 @@ async def _responses_stream( # no `async with`, explicit aclose of lines_iter BEFORE resp / # client so the innermost httpcore byte stream is finalised in # this task (not via Python's asyncgen GC in a sibling task). - client = httpx.AsyncClient(timeout=600) + client = httpx.AsyncClient(timeout = 600) resp = None lines_iter = None try: - req = client.build_request("POST", target_url, json=body) + req = client.build_request("POST", target_url, json = body) try: - resp = await client.send(req, stream=True) + resp = await client.send(req, stream = True) except httpx.RequestError as e: logger.error("responses stream: upstream unreachable: %s", e) yield f"event: response.failed\ndata: {json.dumps({'type': 'response.failed', 'response': {'id': resp_id, 'object': 'response', 'created_at': created_at, 'status': 'failed', 'model': payload.model, 'output': [], 'error': {'code': 502, 'message': _friendly_error(e)}}})}\n\n" @@ -4036,7 +4036,7 @@ async def _responses_stream( if resp.status_code != 200: err_bytes = await resp.aread() - err_text = err_bytes.decode("utf-8", errors="replace") + err_text = err_bytes.decode("utf-8", errors = "replace") logger.error( "responses stream upstream error: status=%s body=%s", resp.status_code, @@ -4162,7 +4162,7 @@ async def _responses_stream( pass # ── Closing events for tool calls ── - for st in sorted(tool_call_state.values(), key=lambda s: s["output_index"]): + for st in sorted(tool_call_state.values(), key = lambda s: s["output_index"]): # If id/name never arrived (malformed upstream), synthesise so # the client still sees a coherent frame sequence. if not st["opened"]: @@ -4249,8 +4249,8 @@ async def _responses_stream( return StreamingResponse( event_generator(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -4274,7 +4274,7 @@ async def openai_responses( """ messages = _normalise_responses_input(payload) if not messages: - raise HTTPException(status_code=400, detail="No input provided.") + raise HTTPException(status_code = 400, detail = "No input provided.") if payload.stream: return await _responses_stream(payload, messages, request) @@ -4360,8 +4360,8 @@ def _normalize_anthropic_openai_images( has_image = True if not is_vision: raise HTTPException( - status_code=400, - detail="Image provided but current GGUF model does not support vision.", + status_code = 400, + detail = "Image provided but current GGUF model does not support vision.", ) url = (part.get("image_url") or {}).get("url", "") @@ -4375,12 +4375,12 @@ def _normalize_anthropic_openai_images( raw = base64.b64decode(b64data) img = Image.open(io.BytesIO(raw)).convert("RGB") buf = io.BytesIO() - img.save(buf, format="PNG") + img.save(buf, format = "PNG") png_b64 = base64.b64encode(buf.getvalue()).decode("ascii") except Exception: raise HTTPException( - status_code=400, - detail="Failed to process image.", + status_code = 400, + detail = "Failed to process image.", ) part["image_url"] = {"url": f"data:image/png;base64,{png_b64}"} @@ -4404,8 +4404,8 @@ async def anthropic_messages( llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: raise HTTPException( - status_code=503, - detail="No GGUF model loaded. Load a GGUF model first.", + status_code = 503, + detail = "No GGUF model loaded. Load a GGUF model first.", ) model_name = getattr(llama_backend, "model_identifier", None) or payload.model @@ -4466,13 +4466,13 @@ async def anthropic_messages( name, type_, schema = td.get("name"), td.get("type"), td.get("input_schema") if schema is None and not isinstance(type_, str): raise HTTPException( - status_code=400, - detail=f"Tool {name!r} is missing required field 'input_schema'.", + status_code = 400, + detail = f"Tool {name!r} is missing required field 'input_schema'.", ) if schema is not None and (not isinstance(name, str) or not name): raise HTTPException( - status_code=400, - detail="Client tool is missing required field 'name'.", + status_code = 400, + detail = "Client tool is missing required field 'name'.", ) # Detect client tools from the raw payload (presence of input_schema) @@ -4488,8 +4488,8 @@ async def anthropic_messages( # would silently drop the client tools. Reject explicitly instead. if requested_studio_tools and _has_client_tool: raise HTTPException( - status_code=400, - detail=( + status_code = 400, + detail = ( "Mixing Anthropic server tools (e.g. web_search_20250305) " "with custom client tools in a single request is not " "supported. Send them in separate requests." @@ -4534,13 +4534,13 @@ async def anthropic_messages( payload.max_tokens, message_id, model_name, - stop=stop, - min_p=min_p, - repetition_penalty=repetition_penalty, - presence_penalty=presence_penalty, - tool_choice=openai_tool_choice, - session_id=payload.session_id, - cancel_id=payload.cancel_id, + stop = stop, + min_p = min_p, + repetition_penalty = repetition_penalty, + presence_penalty = presence_penalty, + tool_choice = openai_tool_choice, + session_id = payload.session_id, + cancel_id = payload.cancel_id, ) return await _anthropic_passthrough_non_streaming( llama_backend, @@ -4552,11 +4552,11 @@ async def anthropic_messages( payload.max_tokens, message_id, model_name, - stop=stop, - min_p=min_p, - repetition_penalty=repetition_penalty, - presence_penalty=presence_penalty, - tool_choice=openai_tool_choice, + stop = stop, + min_p = min_p, + repetition_penalty = repetition_penalty, + presence_penalty = presence_penalty, + tool_choice = openai_tool_choice, ) if server_tools: @@ -4630,21 +4630,21 @@ async def anthropic_messages( def _run_tool_gen(): return llama_backend.generate_chat_completion_with_tools( - messages=openai_messages, - tools=openai_tools, - temperature=temperature, - top_p=top_p, - top_k=top_k, - min_p=min_p, - repetition_penalty=repetition_penalty, - presence_penalty=presence_penalty, - max_tokens=payload.max_tokens, - stop=stop, - cancel_event=cancel_event, - max_tool_iterations=25, - auto_heal_tool_calls=True, - tool_call_timeout=300, - session_id=payload.session_id, + messages = openai_messages, + tools = openai_tools, + temperature = temperature, + top_p = top_p, + top_k = top_k, + min_p = min_p, + repetition_penalty = repetition_penalty, + presence_penalty = presence_penalty, + max_tokens = payload.max_tokens, + stop = stop, + cancel_event = cancel_event, + max_tool_iterations = 25, + auto_heal_tool_calls = True, + tool_call_timeout = 300, + session_id = payload.session_id, ) if payload.stream: @@ -4664,16 +4664,16 @@ async def anthropic_messages( # ── No-tool path ────────────────────────────────────────── def _run_plain_gen(): return llama_backend.generate_chat_completion( - messages=openai_messages, - temperature=temperature, - top_p=top_p, - top_k=top_k, - min_p=min_p, - repetition_penalty=repetition_penalty, - presence_penalty=presence_penalty, - max_tokens=payload.max_tokens, - stop=stop, - cancel_event=cancel_event, + messages = openai_messages, + temperature = temperature, + top_p = top_p, + top_k = top_k, + min_p = min_p, + repetition_penalty = repetition_penalty, + presence_penalty = presence_penalty, + max_tokens = payload.max_tokens, + stop = stop, + cancel_event = cancel_event, ) if payload.stream: @@ -4729,8 +4729,8 @@ async def _anthropic_tool_stream( return StreamingResponse( _stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -4778,8 +4778,8 @@ async def _anthropic_plain_stream( return StreamingResponse( _stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -4818,13 +4818,13 @@ async def _anthropic_tool_non_streaming(run_gen, message_id, model_name): ): content_blocks[-1].text += new else: - content_blocks.append(AnthropicResponseTextBlock(text=new)) + content_blocks.append(AnthropicResponseTextBlock(text = new)) elif etype == "tool_start": content_blocks.append( AnthropicResponseToolUseBlock( - id=event["tool_call_id"], - name=event["tool_name"], - input=event.get("arguments", {}), + id = event["tool_call_id"], + name = event["tool_name"], + input = event.get("arguments", {}), ) ) elif etype == "tool_end": @@ -4833,16 +4833,16 @@ async def _anthropic_tool_non_streaming(run_gen, message_id, model_name): usage = event.get("usage", {}) resp = AnthropicMessagesResponse( - id=message_id, - model=model_name, - content=content_blocks, - stop_reason="end_turn", - usage=AnthropicUsage( - input_tokens=usage.get("prompt_tokens", 0), - output_tokens=usage.get("completion_tokens", 0), + id = message_id, + model = model_name, + content = content_blocks, + stop_reason = "end_turn", + usage = AnthropicUsage( + input_tokens = usage.get("prompt_tokens", 0), + output_tokens = usage.get("completion_tokens", 0), ), ) - return JSONResponse(content=resp.model_dump()) + return JSONResponse(content = resp.model_dump()) async def _anthropic_plain_non_streaming(run_gen, message_id, model_name): @@ -4864,19 +4864,19 @@ async def _anthropic_plain_non_streaming(run_gen, message_id, model_name): full_text = "".join(text_parts) content_blocks = [] if full_text: - content_blocks.append(AnthropicResponseTextBlock(text=full_text)) + content_blocks.append(AnthropicResponseTextBlock(text = full_text)) resp = AnthropicMessagesResponse( - id=message_id, - model=model_name, - content=content_blocks, - stop_reason="end_turn", - usage=AnthropicUsage( - input_tokens=usage.get("prompt_tokens", 0), - output_tokens=usage.get("completion_tokens", 0), + id = message_id, + model = model_name, + content = content_blocks, + stop_reason = "end_turn", + usage = AnthropicUsage( + input_tokens = usage.get("prompt_tokens", 0), + output_tokens = usage.get("completion_tokens", 0), ), ) - return JSONResponse(content=resp.model_dump()) + return JSONResponse(content = resp.model_dump()) # ===================================================================== @@ -4892,14 +4892,14 @@ def _build_passthrough_payload( top_k, max_tokens, stream, - stop=None, - min_p=None, - repetition_penalty=None, - presence_penalty=None, - tool_choice="auto", - response_format=None, - chat_template_kwargs=None, - backend_ctx=None, + stop = None, + min_p = None, + repetition_penalty = None, + presence_penalty = None, + tool_choice = "auto", + response_format = None, + chat_template_kwargs = None, + backend_ctx = None, ): body = { "messages": openai_messages, @@ -4953,13 +4953,13 @@ async def _anthropic_passthrough_stream( max_tokens, message_id, model_name, - stop=None, - min_p=None, - repetition_penalty=None, - presence_penalty=None, - tool_choice="auto", - session_id=None, - cancel_id=None, + stop = None, + min_p = None, + repetition_penalty = None, + presence_penalty = None, + tool_choice = "auto", + session_id = None, + cancel_id = None, ): """Streaming client-side pass-through: forward tools to llama-server and translate its streaming response to Anthropic SSE without executing anything.""" @@ -4972,12 +4972,12 @@ async def _anthropic_passthrough_stream( top_k, max_tokens, True, - stop=stop, - min_p=min_p, - repetition_penalty=repetition_penalty, - presence_penalty=presence_penalty, - tool_choice=tool_choice, - backend_ctx=llama_backend.context_length, + stop = stop, + min_p = min_p, + repetition_penalty = repetition_penalty, + presence_penalty = presence_penalty, + tool_choice = tool_choice, + backend_ctx = llama_backend.context_length, ) # cancel_id mirrors the OpenAI passthrough so a per-run cancel POST @@ -5018,15 +5018,15 @@ async def _anthropic_passthrough_stream( # `try: ... except Exception: pass` so anyio cleanup noise from # nested aclose paths can't bubble out. client = httpx.AsyncClient( - timeout=600, - limits=httpx.Limits(max_keepalive_connections=0), + timeout = 600, + limits = httpx.Limits(max_keepalive_connections = 0), ) resp = None lines_iter = None cancel_watcher = None try: - req = client.build_request("POST", target_url, json=body) - resp = await client.send(req, stream=True) + req = client.build_request("POST", target_url, json = body) + resp = await client.send(req, stream = True) # See _openai_passthrough_stream for rationale: aiter_lines() # blocks during llama-server prefill, so the in-loop cancel @@ -5086,8 +5086,8 @@ async def _anthropic_passthrough_stream( return StreamingResponse( _stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -5105,11 +5105,11 @@ async def _anthropic_passthrough_non_streaming( max_tokens, message_id, model_name, - stop=None, - min_p=None, - repetition_penalty=None, - presence_penalty=None, - tool_choice="auto", + stop = None, + min_p = None, + repetition_penalty = None, + presence_penalty = None, + tool_choice = "auto", ): """Non-streaming client-side pass-through.""" target_url = f"{llama_backend.base_url}/v1/chat/completions" @@ -5121,21 +5121,21 @@ async def _anthropic_passthrough_non_streaming( top_k, max_tokens, False, - stop=stop, - min_p=min_p, - repetition_penalty=repetition_penalty, - presence_penalty=presence_penalty, - tool_choice=tool_choice, - backend_ctx=llama_backend.context_length, + stop = stop, + min_p = min_p, + repetition_penalty = repetition_penalty, + presence_penalty = presence_penalty, + tool_choice = tool_choice, + backend_ctx = llama_backend.context_length, ) async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json=body, timeout=600) + resp = await client.post(target_url, json = body, timeout = 600) if resp.status_code != 200: raise HTTPException( - status_code=resp.status_code, - detail=f"llama-server error: {resp.text[:500]}", + status_code = resp.status_code, + detail = f"llama-server error: {resp.text[:500]}", ) data = resp.json() @@ -5148,7 +5148,7 @@ async def _anthropic_passthrough_non_streaming( if text: text = _TOOL_XML_RE.sub("", text).strip() if text: - content_blocks.append(AnthropicResponseTextBlock(text=text)) + content_blocks.append(AnthropicResponseTextBlock(text = text)) tool_calls = message.get("tool_calls") or [] for tc in tool_calls: @@ -5159,9 +5159,9 @@ async def _anthropic_passthrough_non_streaming( args = {} content_blocks.append( AnthropicResponseToolUseBlock( - id=tc.get("id", ""), - name=fn.get("name", ""), - input=args, + id = tc.get("id", ""), + name = fn.get("name", ""), + input = args, ) ) @@ -5174,16 +5174,16 @@ async def _anthropic_passthrough_non_streaming( usage = data.get("usage") or {} resp_obj = AnthropicMessagesResponse( - id=message_id, - model=model_name, - content=content_blocks, - stop_reason=stop_reason, - usage=AnthropicUsage( - input_tokens=usage.get("prompt_tokens", 0), - output_tokens=usage.get("completion_tokens", 0), + id = message_id, + model = model_name, + content = content_blocks, + stop_reason = stop_reason, + usage = AnthropicUsage( + input_tokens = usage.get("prompt_tokens", 0), + output_tokens = usage.get("completion_tokens", 0), ), ) - return JSONResponse(content=resp_obj.model_dump()) + return JSONResponse(content = resp_obj.model_dump()) # ===================================================================== @@ -5221,7 +5221,7 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: transparently. """ messages = _drop_empty_assistant_sentinels( - [m.model_dump(exclude_none=True) for m in payload.messages] + [m.model_dump(exclude_none = True) for m in payload.messages] ) if not payload.image_base64: @@ -5235,12 +5235,12 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: raw = _b64.b64decode(payload.image_base64) img = _Image.open(_BytesIO(raw)).convert("RGB") buf = _BytesIO() - img.save(buf, format="PNG") + img.save(buf, format = "PNG") png_b64 = _b64.b64encode(buf.getvalue()).decode("ascii") except Exception: raise HTTPException( - status_code=400, - detail="Failed to process image.", + status_code = 400, + detail = "Failed to process image.", ) data_url = f"data:image/png;base64,{png_b64}" @@ -5271,7 +5271,7 @@ def _openai_messages_for_gguf_chat(payload, is_vision: bool) -> tuple[list[dict] image attached to its original turn. """ messages = _drop_empty_assistant_sentinels( - [m.model_dump(exclude_none=True) for m in payload.messages] + [m.model_dump(exclude_none = True) for m in payload.messages] ) has_message_image = any( isinstance(msg.get("content"), list) @@ -5318,7 +5318,7 @@ def _extract_response_format(payload): return rf if isinstance(rf, dict) else None -def _build_openai_passthrough_body(payload, backend_ctx=None) -> dict: +def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: """Assemble the llama-server request body from a ChatCompletionRequest. Only explicitly-known OpenAI / llama-server fields are forwarded so that @@ -5341,14 +5341,14 @@ def _build_openai_passthrough_body(payload, backend_ctx=None) -> dict: payload.top_k, payload.max_tokens, payload.stream, - stop=payload.stop, - min_p=payload.min_p, - repetition_penalty=payload.repetition_penalty, - presence_penalty=payload.presence_penalty, - tool_choice=tool_choice, - response_format=_extract_response_format(payload), - chat_template_kwargs=tpl_kwargs, - backend_ctx=backend_ctx, + stop = payload.stop, + min_p = payload.min_p, + repetition_penalty = payload.repetition_penalty, + presence_penalty = payload.presence_penalty, + tool_choice = tool_choice, + response_format = _extract_response_format(payload), + chat_template_kwargs = tpl_kwargs, + backend_ctx = backend_ctx, ) @@ -5370,7 +5370,7 @@ async def _openai_passthrough_stream( """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body( - payload, backend_ctx=llama_backend.context_length + payload, backend_ctx = llama_backend.context_length ) _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) @@ -5386,13 +5386,13 @@ async def _openai_passthrough_stream( # and non-200 upstream statuses surface as real HTTP errors -- # OpenAI SDKs rely on status codes to raise APIError/BadRequestError. client = httpx.AsyncClient( - timeout=600, - limits=httpx.Limits(max_keepalive_connections=0), + timeout = 600, + limits = httpx.Limits(max_keepalive_connections = 0), ) resp = None try: - req = client.build_request("POST", target_url, json=body) - resp = await client.send(req, stream=True) + req = client.build_request("POST", target_url, json = body) + resp = await client.send(req, stream = True) except httpx.RequestError as e: # llama-server subprocess crashed / still starting / unreachable. logger.error("openai passthrough stream: upstream unreachable: %s", e) @@ -5406,13 +5406,13 @@ async def _openai_passthrough_stream( except Exception: pass raise HTTPException( - status_code=502, - detail=_friendly_error(e), + status_code = 502, + detail = _friendly_error(e), ) if resp.status_code != 200: err_bytes = await resp.aread() - err_text = err_bytes.decode("utf-8", errors="replace") + err_text = err_bytes.decode("utf-8", errors = "replace") logger.error( "openai passthrough upstream error: status=%s body=%s", resp.status_code, @@ -5428,8 +5428,8 @@ async def _openai_passthrough_stream( except Exception: pass raise HTTPException( - status_code=upstream_status, - detail=f"llama-server error: {err_text[:500]}", + status_code = upstream_status, + detail = f"llama-server error: {err_text[:500]}", ) async def _stream(): @@ -5502,8 +5502,8 @@ async def _openai_passthrough_stream( return StreamingResponse( _stream(), - media_type="text/event-stream", - headers={ + media_type = "text/event-stream", + headers = { "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", @@ -5528,26 +5528,26 @@ async def _openai_passthrough_non_streaming( """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body( - payload, backend_ctx=llama_backend.context_length + payload, backend_ctx = llama_backend.context_length ) try: async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json=body, timeout=600) + resp = await client.post(target_url, json = body, timeout = 600) except httpx.RequestError as e: # llama-server subprocess crashed / still starting / unreachable. # Surface the same friendly message the sync chat path emits so # operators don't see a bare 500 with no diagnostic. logger.error("openai passthrough non-streaming: upstream unreachable: %s", e) raise HTTPException( - status_code=502, - detail=_friendly_error(e), + status_code = 502, + detail = _friendly_error(e), ) if resp.status_code != 200: raise HTTPException( - status_code=resp.status_code, - detail=f"llama-server error: {resp.text[:500]}", + status_code = resp.status_code, + detail = f"llama-server error: {resp.text[:500]}", ) # Guided-decoding fence wrap. llama-server returns raw JSON that matches @@ -5576,7 +5576,7 @@ async def _openai_passthrough_non_streaming( msg["content"] = f"```json\n{stripped}\n```" changed = True if changed: - return JSONResponse(content=data) + return JSONResponse(content = data) except Exception as exc: # Wrap is best-effort; fall through to the verbatim body if # the response is not JSON-shaped or the structure is unusual. @@ -5589,4 +5589,4 @@ async def _openai_passthrough_non_streaming( # parse+re-serialize round-trip and keeps the response truly # verbatim (matches the docstring). Status is guaranteed 200 by # the check above. - return Response(content=resp.content, media_type="application/json") + return Response(content = resp.content, media_type = "application/json")