31439d9eed* fix: extend llama.cpp first-token timeout * fix: timeout label pluralization * studio: distinguish llama stream timeout phases * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix/adjust timeout handling for PR #5841 * Fix lint failure for PR #5841 * Fix/adjust stream timeout handling for PR #5841 * Fix/adjust first token timeout for PR #5841 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix/adjust passthrough timeouts for PR #5841 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix/adjust preheader stream cancellation for PR #5841 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix/adjust timeout PR diff for PR #5841 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix/adjust Python 3.9 stream iteration for PR #5841 * Fix first body timeout for PR #5841 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix first token timeout deadlines for PR #5841 --------- Co-authored-by: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: wasimysaid <wasimysdev@gmail.com>
87 lines
2.5 KiB
Python
87 lines
2.5 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
|
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
_backend = os.path.join(os.path.dirname(__file__), "..")
|
|
sys.path.insert(0, _backend)
|
|
|
|
import routes.inference as inf_mod # noqa: E402
|
|
|
|
|
|
def test_non_streaming_generation_timeout_has_read_deadline():
|
|
timeout = inf_mod._llama_non_streaming_generation_timeout()
|
|
assert timeout.read == inf_mod._DEFAULT_FIRST_TOKEN_TIMEOUT_S
|
|
|
|
|
|
def test_stream_first_item_deadline_after_headers():
|
|
async def _run():
|
|
class _Never:
|
|
async def __anext__(self):
|
|
await asyncio.Future()
|
|
|
|
started = time.monotonic()
|
|
try:
|
|
async for _ in inf_mod._aiter_llama_stream_items(
|
|
_Never(),
|
|
first_token_deadline = started + 0.02,
|
|
):
|
|
pass
|
|
except inf_mod.httpx.ReadTimeout:
|
|
pass
|
|
else:
|
|
raise AssertionError("first item deadline did not fire")
|
|
assert time.monotonic() - started < 0.5
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
def test_preheader_send_cleanup_on_disconnect_and_cancel():
|
|
async def _run(cancel_parent):
|
|
state = SimpleNamespace(disconnected = False, closed = False, cancelled = False)
|
|
started = asyncio.Event()
|
|
|
|
class _Client:
|
|
async def send(
|
|
self,
|
|
req,
|
|
stream = False,
|
|
):
|
|
started.set()
|
|
try:
|
|
await asyncio.Future()
|
|
except asyncio.CancelledError:
|
|
state.cancelled = True
|
|
raise
|
|
|
|
async def aclose(self):
|
|
state.closed = True
|
|
|
|
class _Request:
|
|
async def is_disconnected(self):
|
|
return state.disconnected
|
|
|
|
task = asyncio.create_task(
|
|
inf_mod._send_stream_with_preheader_cancel(_Client(), object(), request = _Request())
|
|
)
|
|
await started.wait()
|
|
if cancel_parent:
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
else:
|
|
raise AssertionError("helper cancellation did not propagate")
|
|
else:
|
|
state.disconnected = True
|
|
assert await task is None
|
|
assert state.closed
|
|
assert state.cancelled
|
|
|
|
asyncio.run(_run(False))
|
|
asyncio.run(_run(True))
|