7337729e57* fix(studio/llama_cpp): disable trust_env on the loopback health probe _wait_for_health() polls http://127.0.0.1:<port>/health with the default httpx trust_env=True, so an ambient HTTP(S)_PROXY in the environment is applied to the loopback request. A proxy that returns 503 for 127.0.0.1 makes every probe fail, so the loop runs until timeout and Studio load hangs (trust_env=False returns 200 immediately). Pass trust_env=False so the local readiness probe never goes through a proxy. This mirrors the existing trust_env=False handling in the sibling llama_http / external_provider HTTP clients. * test(offline_gguf_cache): accept trust_env kwarg in fake_get mock _wait_for_health now calls httpx.get(..., trust_env=False); update the retry test's fake_get to accept the kwarg so it doesn't raise TypeError. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio/llama_cpp): bypass proxies for loopback clients * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio/routes): bypass proxies for llama streams * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: wasimysaid <wasimysdev@gmail.com>
52 lines
1.7 KiB
Python
52 lines
1.7 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Shared pooled httpx.AsyncClient for NON-streaming calls to the local llama-server.
|
|
|
|
Streaming generation must NOT use this. It relies on ``Connection: close`` and
|
|
``max_keepalive_connections=0`` so a client disconnect tears down the upstream
|
|
socket and stops GPU decode (PR #5749). This pooled client is only for short
|
|
request/response proxy calls (non-streaming completions, embeddings) where
|
|
reusing a connection removes per-request setup cost. Per-request ``timeout`` is
|
|
still passed at each call site.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import weakref
|
|
|
|
import httpx
|
|
|
|
_LIMITS = httpx.Limits(max_connections = 64, max_keepalive_connections = 32)
|
|
|
|
|
|
def _new_client() -> httpx.AsyncClient:
|
|
return httpx.AsyncClient(limits = _LIMITS, trust_env = False)
|
|
|
|
|
|
# One client per running event loop: an httpx client binds its transport to the
|
|
# loop it first runs on, so a single global instance breaks across a lifespan
|
|
# restart or a second test loop. Weak keys let a finished loop drop its client.
|
|
_clients: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, httpx.AsyncClient]" = (
|
|
weakref.WeakKeyDictionary()
|
|
)
|
|
|
|
|
|
def nonstreaming_client() -> httpx.AsyncClient:
|
|
loop = asyncio.get_running_loop()
|
|
client = _clients.get(loop)
|
|
if client is None or client.is_closed:
|
|
client = _new_client()
|
|
_clients[loop] = client
|
|
return client
|
|
|
|
|
|
async def aclose() -> None:
|
|
clients = list(_clients.values())
|
|
_clients.clear()
|
|
for client in clients:
|
|
try:
|
|
await client.aclose()
|
|
except Exception:
|
|
pass
|