Compare commits
6 commits
main
...
test/hf-sp
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd0a90fcc5 | ||
|
|
f4382e288b | ||
|
|
edf9d28930 | ||
|
|
0347e766d4 | ||
|
|
e6cb03557e | ||
|
|
836dac6d4d |
8 changed files with 231 additions and 391 deletions
|
|
@ -1,6 +1,7 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||||
|
|
||||||
|
import os
|
||||||
import secrets
|
import secrets
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
|
|
@ -21,7 +22,28 @@ ALGORITHM = "HS256"
|
||||||
ACCESS_TOKEN_EXPIRE_MINUTES = 60
|
ACCESS_TOKEN_EXPIRE_MINUTES = 60
|
||||||
REFRESH_TOKEN_EXPIRE_DAYS = 7
|
REFRESH_TOKEN_EXPIRE_DAYS = 7
|
||||||
|
|
||||||
security = HTTPBearer() # Reads Authorization: Bearer <token>
|
security = HTTPBearer(auto_error = False) # Reads Authorization: Bearer <token>
|
||||||
|
|
||||||
|
|
||||||
|
def is_auth_disabled() -> bool:
|
||||||
|
"""
|
||||||
|
Return True when auth should be bypassed.
|
||||||
|
|
||||||
|
For the HF Spaces branch, auth is disabled by default so users can access
|
||||||
|
Studio without login/signup. You can explicitly re-enable auth with:
|
||||||
|
UNSLOTH_STUDIO_ENABLE_AUTH=1
|
||||||
|
"""
|
||||||
|
raw_enable = os.getenv("UNSLOTH_STUDIO_ENABLE_AUTH", "").strip().lower()
|
||||||
|
if raw_enable in {"1", "true", "yes", "on"}:
|
||||||
|
return False
|
||||||
|
|
||||||
|
raw = os.getenv("UNSLOTH_STUDIO_DISABLE_AUTH", "").strip().lower()
|
||||||
|
if raw in {"1", "true", "yes", "on"}:
|
||||||
|
return True
|
||||||
|
if raw in {"0", "false", "no", "off"}:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _get_secret_for_subject(subject: str) -> str:
|
def _get_secret_for_subject(subject: str) -> str:
|
||||||
|
|
@ -103,7 +125,7 @@ def reload_secret() -> None:
|
||||||
|
|
||||||
|
|
||||||
async def get_current_subject(
|
async def get_current_subject(
|
||||||
credentials: HTTPAuthorizationCredentials = Depends(security),
|
credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Validate JWT and require the password-change flow to be completed."""
|
"""Validate JWT and require the password-change flow to be completed."""
|
||||||
return await _get_current_subject(
|
return await _get_current_subject(
|
||||||
|
|
@ -113,7 +135,7 @@ async def get_current_subject(
|
||||||
|
|
||||||
|
|
||||||
async def get_current_subject_allow_password_change(
|
async def get_current_subject_allow_password_change(
|
||||||
credentials: HTTPAuthorizationCredentials = Depends(security),
|
credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Validate JWT but allow access to the password-change endpoint."""
|
"""Validate JWT but allow access to the password-change endpoint."""
|
||||||
return await _get_current_subject(
|
return await _get_current_subject(
|
||||||
|
|
@ -123,7 +145,7 @@ async def get_current_subject_allow_password_change(
|
||||||
|
|
||||||
|
|
||||||
async def _get_current_subject(
|
async def _get_current_subject(
|
||||||
credentials: HTTPAuthorizationCredentials,
|
credentials: Optional[HTTPAuthorizationCredentials],
|
||||||
*,
|
*,
|
||||||
allow_password_change: bool,
|
allow_password_change: bool,
|
||||||
) -> str:
|
) -> str:
|
||||||
|
|
@ -136,6 +158,15 @@ async def _get_current_subject(
|
||||||
async def secure_endpoint(current_subject: str = Depends(get_current_subject)):
|
async def secure_endpoint(current_subject: str = Depends(get_current_subject)):
|
||||||
...
|
...
|
||||||
"""
|
"""
|
||||||
|
if is_auth_disabled():
|
||||||
|
return "hf-space-user"
|
||||||
|
|
||||||
|
if credentials is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code = status.HTTP_401_UNAUTHORIZED,
|
||||||
|
detail = "Missing bearer token",
|
||||||
|
)
|
||||||
|
|
||||||
token = credentials.credentials
|
token = credentials.credentials
|
||||||
subject = _decode_subject_without_verification(token)
|
subject = _decode_subject_without_verification(token)
|
||||||
if subject is None:
|
if subject is None:
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,10 @@ class AuthStatusResponse(BaseModel):
|
||||||
...,
|
...,
|
||||||
description = "True if the seeded admin must still change the default password",
|
description = "True if the seeded admin must still change the default password",
|
||||||
)
|
)
|
||||||
|
auth_disabled: bool = Field(
|
||||||
|
default = False,
|
||||||
|
description = "True when auth is bypassed (HF Spaces mode).",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ChangePasswordRequest(BaseModel):
|
class ChangePasswordRequest(BaseModel):
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ from auth.authentication import (
|
||||||
create_refresh_token,
|
create_refresh_token,
|
||||||
get_current_subject,
|
get_current_subject,
|
||||||
get_current_subject_allow_password_change,
|
get_current_subject_allow_password_change,
|
||||||
|
is_auth_disabled,
|
||||||
refresh_access_token,
|
refresh_access_token,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -34,6 +35,14 @@ async def auth_status() -> AuthStatusResponse:
|
||||||
- initialized = False -> frontend should wait for the seeded admin bootstrap.
|
- initialized = False -> frontend should wait for the seeded admin bootstrap.
|
||||||
- initialized = True -> frontend should show login or force the first password change.
|
- initialized = True -> frontend should show login or force the first password change.
|
||||||
"""
|
"""
|
||||||
|
if is_auth_disabled():
|
||||||
|
return AuthStatusResponse(
|
||||||
|
initialized = True,
|
||||||
|
default_username = storage.DEFAULT_ADMIN_USERNAME,
|
||||||
|
requires_password_change = False,
|
||||||
|
auth_disabled = True,
|
||||||
|
)
|
||||||
|
|
||||||
return AuthStatusResponse(
|
return AuthStatusResponse(
|
||||||
initialized = storage.is_initialized(),
|
initialized = storage.is_initialized(),
|
||||||
default_username = storage.DEFAULT_ADMIN_USERNAME,
|
default_username = storage.DEFAULT_ADMIN_USERNAME,
|
||||||
|
|
@ -42,6 +51,7 @@ async def auth_status() -> AuthStatusResponse:
|
||||||
)
|
)
|
||||||
if storage.is_initialized()
|
if storage.is_initialized()
|
||||||
else True,
|
else True,
|
||||||
|
auth_disabled = False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,8 +7,9 @@ Training API routes
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
|
from starlette.websockets import WebSocketState
|
||||||
from typing import Dict, Optional, Any
|
from typing import Dict, Optional, Any
|
||||||
import structlog
|
import structlog
|
||||||
from loggers import get_logger
|
from loggers import get_logger
|
||||||
|
|
@ -494,101 +495,108 @@ async def get_training_metrics(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/progress")
|
@router.websocket("/progress/ws")
|
||||||
async def stream_training_progress(
|
async def ws_training_progress(websocket: WebSocket):
|
||||||
request: Request,
|
|
||||||
current_subject: str = Depends(get_current_subject),
|
|
||||||
):
|
|
||||||
"""
|
"""
|
||||||
Stream training progress updates using Server-Sent Events (SSE).
|
Stream training progress updates over WebSocket.
|
||||||
|
|
||||||
This endpoint provides real-time updates on training progress.
|
This endpoint provides real-time updates on training progress.
|
||||||
Supports reconnection via the SSE spec:
|
Supports reconnection via query params:
|
||||||
- Sends `id:` with each event so the browser tracks position.
|
- `last_event_id`: resume from a specific step on reconnect.
|
||||||
- Sends `retry:` to control reconnection interval.
|
- `token`: JWT auth token (WebSocket can't use Authorization header).
|
||||||
- Sends named `event:` types (progress, heartbeat, complete, error).
|
|
||||||
- Reads `Last-Event-ID` header on reconnect to replay missed steps.
|
|
||||||
"""
|
"""
|
||||||
# Read Last-Event-ID header for reconnection resume
|
# Auth: WebSocket can't use Authorization header, so accept token as query param
|
||||||
last_event_id = request.headers.get("last-event-id")
|
from auth.authentication import is_auth_disabled
|
||||||
|
|
||||||
|
if not is_auth_disabled():
|
||||||
|
token = websocket.query_params.get("token")
|
||||||
|
if not token:
|
||||||
|
await websocket.close(code = 4001, reason = "Missing auth token")
|
||||||
|
return
|
||||||
|
from auth.authentication import _decode_subject_without_verification
|
||||||
|
from auth.storage import get_user_and_secret
|
||||||
|
|
||||||
|
subject = _decode_subject_without_verification(token)
|
||||||
|
if subject is None:
|
||||||
|
await websocket.close(code = 4001, reason = "Invalid token")
|
||||||
|
return
|
||||||
|
record = get_user_and_secret(subject)
|
||||||
|
if record is None:
|
||||||
|
await websocket.close(code = 4001, reason = "Invalid or expired token")
|
||||||
|
return
|
||||||
|
|
||||||
|
await websocket.accept()
|
||||||
|
|
||||||
|
# Read optional last_event_id from query params for reconnection resume
|
||||||
|
last_event_id = websocket.query_params.get("last_event_id")
|
||||||
resume_from_step: Optional[int] = None
|
resume_from_step: Optional[int] = None
|
||||||
if last_event_id is not None:
|
if last_event_id is not None:
|
||||||
try:
|
try:
|
||||||
resume_from_step = int(last_event_id)
|
resume_from_step = int(last_event_id)
|
||||||
logger.info(f"SSE reconnect: resuming from step {resume_from_step}")
|
logger.info(f"WebSocket reconnect: resuming from step {resume_from_step}")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
logger.warning(f"Invalid Last-Event-ID: {last_event_id}")
|
logger.warning(f"Invalid last_event_id: {last_event_id}")
|
||||||
|
|
||||||
async def event_generator():
|
backend = get_training_backend()
|
||||||
backend = get_training_backend()
|
job_id: str = getattr(backend, "current_job_id", "") or ""
|
||||||
job_id: str = getattr(backend, "current_job_id", "") or ""
|
|
||||||
|
|
||||||
# ── Helpers ──────────────────────────────────────────────
|
# ── Helpers ──────────────────────────────────────────────
|
||||||
def build_progress(
|
def build_progress(
|
||||||
step: int,
|
step: int,
|
||||||
loss: float,
|
loss: float,
|
||||||
learning_rate: float,
|
learning_rate: float,
|
||||||
total_steps: int,
|
total_steps: int,
|
||||||
epoch: Optional[float] = None,
|
epoch: Optional[float] = None,
|
||||||
progress: Optional[Any] = None,
|
progress: Optional[Any] = None,
|
||||||
grad_norm_override: Optional[float] = None,
|
grad_norm_override: Optional[float] = None,
|
||||||
eval_loss_override: Optional[float] = None,
|
eval_loss_override: Optional[float] = None,
|
||||||
) -> TrainingProgress:
|
) -> TrainingProgress:
|
||||||
total = max(total_steps, 0)
|
total = max(total_steps, 0)
|
||||||
if step < 0 or total == 0:
|
if step < 0 or total == 0:
|
||||||
progress_percent = 0.0
|
progress_percent = 0.0
|
||||||
else:
|
else:
|
||||||
progress_percent = (
|
progress_percent = (
|
||||||
float(step) / float(total) * 100.0 if total > 0 else 0.0
|
float(step) / float(total) * 100.0 if total > 0 else 0.0
|
||||||
)
|
|
||||||
|
|
||||||
# Get actual values from progress object if available
|
|
||||||
elapsed_seconds = (
|
|
||||||
getattr(progress, "elapsed_seconds", None) if progress else None
|
|
||||||
)
|
|
||||||
eta_seconds = getattr(progress, "eta_seconds", None) if progress else None
|
|
||||||
grad_norm = grad_norm_override
|
|
||||||
if grad_norm is None and progress:
|
|
||||||
grad_norm = getattr(progress, "grad_norm", None)
|
|
||||||
num_tokens = getattr(progress, "num_tokens", None) if progress else None
|
|
||||||
eval_loss = eval_loss_override
|
|
||||||
if eval_loss is None and progress:
|
|
||||||
eval_loss = getattr(progress, "eval_loss", None)
|
|
||||||
|
|
||||||
return TrainingProgress(
|
|
||||||
job_id = job_id,
|
|
||||||
step = step,
|
|
||||||
total_steps = total,
|
|
||||||
loss = loss,
|
|
||||||
learning_rate = learning_rate,
|
|
||||||
progress_percent = progress_percent,
|
|
||||||
epoch = epoch,
|
|
||||||
elapsed_seconds = elapsed_seconds,
|
|
||||||
eta_seconds = eta_seconds,
|
|
||||||
grad_norm = grad_norm,
|
|
||||||
num_tokens = num_tokens,
|
|
||||||
eval_loss = eval_loss,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def format_sse(
|
elapsed_seconds = (
|
||||||
data: str,
|
getattr(progress, "elapsed_seconds", None) if progress else None
|
||||||
event: str = "progress",
|
)
|
||||||
event_id: Optional[int] = None,
|
eta_seconds = getattr(progress, "eta_seconds", None) if progress else None
|
||||||
) -> str:
|
grad_norm = grad_norm_override
|
||||||
"""Format a single SSE message with id/event/data fields."""
|
if grad_norm is None and progress:
|
||||||
lines = []
|
grad_norm = getattr(progress, "grad_norm", None)
|
||||||
if event_id is not None:
|
num_tokens = getattr(progress, "num_tokens", None) if progress else None
|
||||||
lines.append(f"id: {event_id}")
|
eval_loss = eval_loss_override
|
||||||
lines.append(f"event: {event}")
|
if eval_loss is None and progress:
|
||||||
lines.append(f"data: {data}")
|
eval_loss = getattr(progress, "eval_loss", None)
|
||||||
lines.append("") # trailing blank line
|
|
||||||
lines.append("") # double newline terminates the event
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
# ── Retry directive ──────────────────────────────────────
|
return TrainingProgress(
|
||||||
# Tell the browser to reconnect after 3 seconds if the connection drops
|
job_id = job_id,
|
||||||
yield "retry: 3000\n\n"
|
step = step,
|
||||||
|
total_steps = total,
|
||||||
|
loss = loss,
|
||||||
|
learning_rate = learning_rate,
|
||||||
|
progress_percent = progress_percent,
|
||||||
|
epoch = epoch,
|
||||||
|
elapsed_seconds = elapsed_seconds,
|
||||||
|
eta_seconds = eta_seconds,
|
||||||
|
grad_norm = grad_norm,
|
||||||
|
num_tokens = num_tokens,
|
||||||
|
eval_loss = eval_loss,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def send_event(event: str, event_id: Optional[int], payload: TrainingProgress):
|
||||||
|
"""Send a typed event over WebSocket as JSON."""
|
||||||
|
if websocket.client_state != WebSocketState.CONNECTED:
|
||||||
|
return
|
||||||
|
await websocket.send_json({
|
||||||
|
"event": event,
|
||||||
|
"id": event_id,
|
||||||
|
"data": payload.model_dump(),
|
||||||
|
})
|
||||||
|
|
||||||
|
try:
|
||||||
# ── Replay missed steps on reconnect ─────────────────────
|
# ── Replay missed steps on reconnect ─────────────────────
|
||||||
if resume_from_step is not None and backend.step_history:
|
if resume_from_step is not None and backend.step_history:
|
||||||
replayed = 0
|
replayed = 0
|
||||||
|
|
@ -629,12 +637,10 @@ async def stream_training_progress(
|
||||||
progress = tp_replay,
|
progress = tp_replay,
|
||||||
grad_norm_override = grad_norm_by_step.get(step_val),
|
grad_norm_override = grad_norm_by_step.get(step_val),
|
||||||
)
|
)
|
||||||
yield format_sse(
|
await send_event("progress", step_val, payload)
|
||||||
payload.model_dump_json(), event = "progress", event_id = step_val
|
|
||||||
)
|
|
||||||
replayed += 1
|
replayed += 1
|
||||||
if replayed:
|
if replayed:
|
||||||
logger.info(f"SSE reconnect: replayed {replayed} missed steps")
|
logger.info(f"WebSocket reconnect: replayed {replayed} missed steps")
|
||||||
|
|
||||||
# ── Initial status (only on fresh connections) ───────────
|
# ── Initial status (only on fresh connections) ───────────
|
||||||
if resume_from_step is None:
|
if resume_from_step is None:
|
||||||
|
|
@ -651,11 +657,9 @@ async def stream_training_progress(
|
||||||
epoch = initial_epoch,
|
epoch = initial_epoch,
|
||||||
progress = tp,
|
progress = tp,
|
||||||
)
|
)
|
||||||
yield format_sse(
|
await send_event("progress", 0, initial_progress)
|
||||||
initial_progress.model_dump_json(), event = "progress", event_id = 0
|
|
||||||
)
|
|
||||||
|
|
||||||
# If not active, send final state and exit
|
# If not active, send final state and close
|
||||||
if not is_active:
|
if not is_active:
|
||||||
if backend.step_history:
|
if backend.step_history:
|
||||||
final_step = backend.step_history[-1]
|
final_step = backend.step_history[-1]
|
||||||
|
|
@ -675,23 +679,18 @@ async def stream_training_progress(
|
||||||
final_epoch,
|
final_epoch,
|
||||||
progress = tp,
|
progress = tp,
|
||||||
)
|
)
|
||||||
yield format_sse(
|
await send_event("complete", final_step, payload)
|
||||||
payload.model_dump_json(), event = "complete", event_id = final_step
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
yield format_sse(
|
await send_event(
|
||||||
build_progress(-1, 0.0, 0.0, 0, progress = tp).model_dump_json(),
|
"complete", 0,
|
||||||
event = "complete",
|
build_progress(-1, 0.0, 0.0, 0, progress = tp),
|
||||||
event_id = 0,
|
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
# ── Live polling loop ────────────────────────────────────
|
# ── Live polling loop ────────────────────────────────────
|
||||||
last_step = resume_from_step if resume_from_step is not None else -1
|
last_step = resume_from_step if resume_from_step is not None else -1
|
||||||
no_update_count = 0
|
no_update_count = 0
|
||||||
max_no_updates = (
|
max_no_updates = 1800 # Timeout after 30 min
|
||||||
1800 # Timeout after 30 minutes (large models need time for compilation)
|
|
||||||
)
|
|
||||||
|
|
||||||
while backend.is_training_active():
|
while backend.is_training_active():
|
||||||
try:
|
try:
|
||||||
|
|
@ -723,11 +722,7 @@ async def stream_training_progress(
|
||||||
current_epoch,
|
current_epoch,
|
||||||
progress = tp_inner,
|
progress = tp_inner,
|
||||||
)
|
)
|
||||||
yield format_sse(
|
await send_event("progress", current_step, progress_payload)
|
||||||
progress_payload.model_dump_json(),
|
|
||||||
event = "progress",
|
|
||||||
event_id = current_step,
|
|
||||||
)
|
|
||||||
last_step = current_step
|
last_step = current_step
|
||||||
no_update_count = 0
|
no_update_count = 0
|
||||||
else:
|
else:
|
||||||
|
|
@ -742,17 +737,11 @@ async def stream_training_progress(
|
||||||
current_epoch,
|
current_epoch,
|
||||||
progress = tp_inner,
|
progress = tp_inner,
|
||||||
)
|
)
|
||||||
yield format_sse(
|
await send_event("heartbeat", current_step, heartbeat_payload)
|
||||||
heartbeat_payload.model_dump_json(),
|
|
||||||
event = "heartbeat",
|
|
||||||
event_id = current_step,
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
# No steps yet, but training is active (model loading, etc.)
|
# No steps yet, but training is active (model loading, etc.)
|
||||||
no_update_count += 1
|
no_update_count += 1
|
||||||
if no_update_count % 5 == 0:
|
if no_update_count % 5 == 0:
|
||||||
# Pull total_steps and status from trainer so
|
|
||||||
# the frontend can show "Tokenizing…" etc.
|
|
||||||
tp_prep = getattr(
|
tp_prep = getattr(
|
||||||
getattr(backend, "trainer", None),
|
getattr(backend, "trainer", None),
|
||||||
"training_progress",
|
"training_progress",
|
||||||
|
|
@ -762,17 +751,9 @@ async def stream_training_progress(
|
||||||
getattr(tp_prep, "total_steps", 0) if tp_prep else 0
|
getattr(tp_prep, "total_steps", 0) if tp_prep else 0
|
||||||
)
|
)
|
||||||
preparing_payload = build_progress(
|
preparing_payload = build_progress(
|
||||||
0,
|
0, 0.0, 0.0, prep_total, progress = tp_prep,
|
||||||
0.0,
|
|
||||||
0.0,
|
|
||||||
prep_total,
|
|
||||||
progress = tp_prep,
|
|
||||||
)
|
|
||||||
yield format_sse(
|
|
||||||
preparing_payload.model_dump_json(),
|
|
||||||
event = "heartbeat",
|
|
||||||
event_id = 0,
|
|
||||||
)
|
)
|
||||||
|
await send_event("heartbeat", 0, preparing_payload)
|
||||||
|
|
||||||
# Timeout check
|
# Timeout check
|
||||||
if no_update_count > max_no_updates:
|
if no_update_count > max_no_updates:
|
||||||
|
|
@ -783,10 +764,8 @@ async def stream_training_progress(
|
||||||
timeout_payload = build_progress(
|
timeout_payload = build_progress(
|
||||||
last_step, 0.0, 0.0, 0, progress = tp_timeout
|
last_step, 0.0, 0.0, 0, progress = tp_timeout
|
||||||
)
|
)
|
||||||
yield format_sse(
|
await send_event(
|
||||||
timeout_payload.model_dump_json(),
|
"error", last_step if last_step >= 0 else 0, timeout_payload
|
||||||
event = "error",
|
|
||||||
event_id = last_step if last_step >= 0 else 0,
|
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
@ -798,10 +777,8 @@ async def stream_training_progress(
|
||||||
getattr(backend, "trainer", None), "training_progress", None
|
getattr(backend, "trainer", None), "training_progress", None
|
||||||
)
|
)
|
||||||
error_payload = build_progress(0, 0.0, 0.0, 0, progress = tp_error)
|
error_payload = build_progress(0, 0.0, 0.0, 0, progress = tp_error)
|
||||||
yield format_sse(
|
await send_event(
|
||||||
error_payload.model_dump_json(),
|
"error", last_step if last_step >= 0 else 0, error_payload
|
||||||
event = "error",
|
|
||||||
event_id = last_step if last_step >= 0 else 0,
|
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
@ -822,18 +799,16 @@ async def stream_training_progress(
|
||||||
final_epoch,
|
final_epoch,
|
||||||
progress = final_tp,
|
progress = final_tp,
|
||||||
)
|
)
|
||||||
yield format_sse(
|
await send_event(
|
||||||
final_payload.model_dump_json(),
|
"complete", final_step if final_step >= 0 else 0, final_payload
|
||||||
event = "complete",
|
|
||||||
event_id = final_step if final_step >= 0 else 0,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return StreamingResponse(
|
except WebSocketDisconnect:
|
||||||
event_generator(),
|
logger.info("WebSocket client disconnected")
|
||||||
media_type = "text/event-stream",
|
except Exception as e:
|
||||||
headers = {
|
logger.error(f"WebSocket error: {e}", exc_info = True)
|
||||||
"Cache-Control": "no-cache",
|
try:
|
||||||
"Connection": "keep-alive",
|
if websocket.client_state == WebSocketState.CONNECTED:
|
||||||
"X-Accel-Buffering": "no",
|
await websocket.send_json({"event": "error", "id": None, "data": {"error": str(e)}})
|
||||||
},
|
except Exception:
|
||||||
)
|
pass
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,22 @@ import {
|
||||||
refreshSession,
|
refreshSession,
|
||||||
} from "@/features/auth";
|
} from "@/features/auth";
|
||||||
|
|
||||||
|
type AuthStatus = {
|
||||||
|
initialized: boolean;
|
||||||
|
requires_password_change: boolean;
|
||||||
|
auth_disabled?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function getAuthStatus(): Promise<AuthStatus | null> {
|
||||||
|
try {
|
||||||
|
const res = await fetch("/api/auth/status");
|
||||||
|
if (!res.ok) return null;
|
||||||
|
return (await res.json()) as AuthStatus;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function hasActiveSession(): Promise<boolean> {
|
async function hasActiveSession(): Promise<boolean> {
|
||||||
if (hasAuthToken()) return true;
|
if (hasAuthToken()) return true;
|
||||||
if (!hasRefreshToken()) return false;
|
if (!hasRefreshToken()) return false;
|
||||||
|
|
@ -17,28 +33,22 @@ async function hasActiveSession(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function checkAuthInitialized(): Promise<boolean> {
|
async function checkAuthInitialized(): Promise<boolean> {
|
||||||
try {
|
const status = await getAuthStatus();
|
||||||
const res = await fetch("/api/auth/status");
|
if (status?.auth_disabled) return true;
|
||||||
if (!res.ok) return true; // fallback to login on error
|
return status?.initialized ?? true; // fallback to login on error
|
||||||
const data = (await res.json()) as { initialized: boolean };
|
|
||||||
return data.initialized;
|
|
||||||
} catch {
|
|
||||||
return true; // fallback to login on error
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function checkPasswordChangeRequired(): Promise<boolean> {
|
async function checkPasswordChangeRequired(): Promise<boolean> {
|
||||||
try {
|
const status = await getAuthStatus();
|
||||||
const res = await fetch("/api/auth/status");
|
if (status?.auth_disabled) return false;
|
||||||
if (!res.ok) return mustChangePassword();
|
if (!status) return mustChangePassword();
|
||||||
const data = (await res.json()) as { requires_password_change: boolean };
|
return status.requires_password_change || mustChangePassword();
|
||||||
return data.requires_password_change || mustChangePassword();
|
|
||||||
} catch {
|
|
||||||
return mustChangePassword();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function requireAuth(): Promise<void> {
|
export async function requireAuth(): Promise<void> {
|
||||||
|
const status = await getAuthStatus();
|
||||||
|
if (status?.auth_disabled) return;
|
||||||
|
|
||||||
if (await hasActiveSession()) {
|
if (await hasActiveSession()) {
|
||||||
if (await checkPasswordChangeRequired()) {
|
if (await checkPasswordChangeRequired()) {
|
||||||
throw redirect({ to: "/change-password" });
|
throw redirect({ to: "/change-password" });
|
||||||
|
|
@ -52,11 +62,21 @@ export async function requireAuth(): Promise<void> {
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function requireGuest(): Promise<void> {
|
export async function requireGuest(): Promise<void> {
|
||||||
|
const status = await getAuthStatus();
|
||||||
|
if (status?.auth_disabled) {
|
||||||
|
throw redirect({ to: getPostAuthRoute() });
|
||||||
|
}
|
||||||
|
|
||||||
if (!(await hasActiveSession())) return;
|
if (!(await hasActiveSession())) return;
|
||||||
throw redirect({ to: getPostAuthRoute() });
|
throw redirect({ to: getPostAuthRoute() });
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function requirePasswordChangeFlow(): Promise<void> {
|
export async function requirePasswordChangeFlow(): Promise<void> {
|
||||||
|
const status = await getAuthStatus();
|
||||||
|
if (status?.auth_disabled) {
|
||||||
|
throw redirect({ to: getPostAuthRoute() });
|
||||||
|
}
|
||||||
|
|
||||||
const requiresPasswordChange = await checkPasswordChangeRequired();
|
const requiresPasswordChange = await checkPasswordChangeRequired();
|
||||||
|
|
||||||
if (requiresPasswordChange) return;
|
if (requiresPasswordChange) return;
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,6 @@ import type {
|
||||||
} from "../types/api";
|
} from "../types/api";
|
||||||
import type {
|
import type {
|
||||||
TrainingMetricsResponse,
|
TrainingMetricsResponse,
|
||||||
TrainingProgressPayload,
|
|
||||||
TrainingStatusResponse,
|
TrainingStatusResponse,
|
||||||
} from "../types/runtime";
|
} from "../types/runtime";
|
||||||
|
|
||||||
|
|
@ -70,118 +69,4 @@ export async function getTrainingMetrics(): Promise<TrainingMetricsResponse> {
|
||||||
return parseJson<TrainingMetricsResponse>(response);
|
return parseJson<TrainingMetricsResponse>(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProgressEventName = "progress" | "heartbeat" | "complete" | "error";
|
|
||||||
|
|
||||||
interface ParsedSseEvent {
|
|
||||||
event: ProgressEventName;
|
|
||||||
payload: TrainingProgressPayload;
|
|
||||||
id: number | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseSseEvent(rawEvent: string): ParsedSseEvent | null {
|
|
||||||
const lines = rawEvent.split(/\r?\n/);
|
|
||||||
let eventName: ProgressEventName = "progress";
|
|
||||||
let id: number | null = null;
|
|
||||||
const dataLines: string[] = [];
|
|
||||||
|
|
||||||
for (const line of lines) {
|
|
||||||
if (!line) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (line.startsWith("event:")) {
|
|
||||||
const value = line.slice(6).trim();
|
|
||||||
if (
|
|
||||||
value === "progress" ||
|
|
||||||
value === "heartbeat" ||
|
|
||||||
value === "complete" ||
|
|
||||||
value === "error"
|
|
||||||
) {
|
|
||||||
eventName = value;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (line.startsWith("id:")) {
|
|
||||||
const value = Number(line.slice(3).trim());
|
|
||||||
id = Number.isFinite(value) ? value : null;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (line.startsWith("data:")) {
|
|
||||||
dataLines.push(line.slice(5).trimStart());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dataLines.length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const parsed = JSON.parse(dataLines.join("\n")) as TrainingProgressPayload;
|
|
||||||
return { event: eventName, payload: parsed, id };
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function streamTrainingProgress(options: {
|
|
||||||
signal: AbortSignal;
|
|
||||||
lastEventId?: number | null;
|
|
||||||
onOpen?: () => void;
|
|
||||||
onEvent: (event: ParsedSseEvent) => void;
|
|
||||||
}): Promise<void> {
|
|
||||||
const headers = new Headers();
|
|
||||||
if (typeof options.lastEventId === "number") {
|
|
||||||
headers.set("Last-Event-ID", String(options.lastEventId));
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await authFetch("/api/train/progress", {
|
|
||||||
method: "GET",
|
|
||||||
headers,
|
|
||||||
signal: options.signal,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(await readError(response));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!response.body) {
|
|
||||||
throw new Error("Progress stream unavailable");
|
|
||||||
}
|
|
||||||
|
|
||||||
options.onOpen?.();
|
|
||||||
|
|
||||||
const reader = response.body.getReader();
|
|
||||||
const decoder = new TextDecoder();
|
|
||||||
let buffer = "";
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
const { value, done } = await reader.read();
|
|
||||||
if (done) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
buffer += decoder.decode(value, { stream: true });
|
|
||||||
|
|
||||||
let separatorIndex = buffer.search(/\r?\n\r?\n/);
|
|
||||||
while (separatorIndex >= 0) {
|
|
||||||
const rawEvent = buffer.slice(0, separatorIndex);
|
|
||||||
const separatorLength = buffer[separatorIndex] === "\r" ? 4 : 2;
|
|
||||||
buffer = buffer.slice(separatorIndex + separatorLength);
|
|
||||||
|
|
||||||
if (rawEvent.startsWith("retry:")) {
|
|
||||||
separatorIndex = buffer.search(/\r?\n\r?\n/);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const event = parseSseEvent(rawEvent);
|
|
||||||
if (event) {
|
|
||||||
options.onEvent(event);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
if (!isAbortError(error)) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
separatorIndex = buffer.search(/\r?\n\r?\n/);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export { isAbortError };
|
export { isAbortError };
|
||||||
|
|
|
||||||
|
|
@ -7,44 +7,18 @@ import {
|
||||||
getTrainingMetrics,
|
getTrainingMetrics,
|
||||||
getTrainingStatus,
|
getTrainingStatus,
|
||||||
isAbortError,
|
isAbortError,
|
||||||
streamTrainingProgress,
|
|
||||||
} from "../api/train-api";
|
} from "../api/train-api";
|
||||||
import { useTrainingRuntimeStore } from "../stores/training-runtime-store";
|
import { useTrainingRuntimeStore } from "../stores/training-runtime-store";
|
||||||
import type { TrainingRuntimeStore } from "../types/runtime";
|
|
||||||
|
|
||||||
const STATUS_POLL_INTERVAL_MS = 3000;
|
const STATUS_POLL_INTERVAL_MS = 2000;
|
||||||
const METRICS_POLL_INTERVAL_MS = 5000;
|
const METRICS_POLL_INTERVAL_MS = 3000;
|
||||||
const STREAM_RECONNECT_DELAY_MS = 1500;
|
|
||||||
|
|
||||||
function shouldUseLiveSync(state: TrainingRuntimeStore): boolean {
|
|
||||||
return state.isTrainingRunning || state.phase === "training";
|
|
||||||
}
|
|
||||||
|
|
||||||
export function useTrainingRuntimeLifecycle(): void {
|
export function useTrainingRuntimeLifecycle(): void {
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
let disposed = false;
|
let disposed = false;
|
||||||
let openingStream = false;
|
|
||||||
let streamController: AbortController | null = null;
|
|
||||||
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
|
||||||
|
|
||||||
const runtimeStore = useTrainingRuntimeStore;
|
const runtimeStore = useTrainingRuntimeStore;
|
||||||
|
|
||||||
const clearReconnect = () => {
|
|
||||||
if (reconnectTimer) {
|
|
||||||
clearTimeout(reconnectTimer);
|
|
||||||
reconnectTimer = null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const stopStream = () => {
|
|
||||||
clearReconnect();
|
|
||||||
if (streamController) {
|
|
||||||
streamController.abort();
|
|
||||||
streamController = null;
|
|
||||||
}
|
|
||||||
runtimeStore.getState().setSseConnected(false);
|
|
||||||
};
|
|
||||||
|
|
||||||
const pollMetrics = async () => {
|
const pollMetrics = async () => {
|
||||||
if (!hasAuthToken()) return;
|
if (!hasAuthToken()) return;
|
||||||
const gen = runtimeStore.getState().resetGeneration;
|
const gen = runtimeStore.getState().resetGeneration;
|
||||||
|
|
@ -56,7 +30,7 @@ export function useTrainingRuntimeLifecycle(): void {
|
||||||
runtimeStore.getState().applyMetrics(metrics);
|
runtimeStore.getState().applyMetrics(metrics);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (!isAbortError(error) && !disposed && hasAuthToken()) {
|
if (!isAbortError(error) && !disposed && hasAuthToken()) {
|
||||||
runtimeStore.getState().setSseConnected(false);
|
// silent — next poll will retry
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -69,83 +43,10 @@ export function useTrainingRuntimeLifecycle(): void {
|
||||||
if (disposed || runtimeStore.getState().resetGeneration !== gen) {
|
if (disposed || runtimeStore.getState().resetGeneration !== gen) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
runtimeStore.getState().applyStatus(status);
|
runtimeStore.getState().applyStatus(status);
|
||||||
|
|
||||||
const nextState = runtimeStore.getState();
|
|
||||||
if (shouldUseLiveSync(nextState)) {
|
|
||||||
void ensureStream();
|
|
||||||
} else {
|
|
||||||
stopStream();
|
|
||||||
}
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (!isAbortError(error) && !disposed && hasAuthToken()) {
|
if (!isAbortError(error) && !disposed && hasAuthToken()) {
|
||||||
runtimeStore.getState().setSseConnected(false);
|
// silent — next poll will retry
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const ensureStream = async () => {
|
|
||||||
const state = runtimeStore.getState();
|
|
||||||
if (
|
|
||||||
disposed ||
|
|
||||||
openingStream ||
|
|
||||||
streamController ||
|
|
||||||
!shouldUseLiveSync(state)
|
|
||||||
) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
clearReconnect();
|
|
||||||
openingStream = true;
|
|
||||||
const controller = new AbortController();
|
|
||||||
streamController = controller;
|
|
||||||
|
|
||||||
try {
|
|
||||||
await streamTrainingProgress({
|
|
||||||
signal: controller.signal,
|
|
||||||
lastEventId: state.lastEventId,
|
|
||||||
onOpen: () => {
|
|
||||||
runtimeStore.getState().setSseConnected(true);
|
|
||||||
},
|
|
||||||
onEvent: (event) => {
|
|
||||||
const liveStore = runtimeStore.getState();
|
|
||||||
if (typeof event.id === "number") {
|
|
||||||
liveStore.setLastEventId(event.id);
|
|
||||||
}
|
|
||||||
|
|
||||||
liveStore.applyProgress(event.payload, event.id ?? undefined);
|
|
||||||
|
|
||||||
if (event.event === "complete") {
|
|
||||||
void pollStatus();
|
|
||||||
void pollMetrics();
|
|
||||||
stopStream();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (event.event === "error") {
|
|
||||||
liveStore.setRuntimeError("Training stream error");
|
|
||||||
stopStream();
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
if (!disposed && !controller.signal.aborted && !isAbortError(error)) {
|
|
||||||
runtimeStore.getState().setSseConnected(false);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
openingStream = false;
|
|
||||||
if (streamController === controller) {
|
|
||||||
streamController = null;
|
|
||||||
}
|
|
||||||
runtimeStore.getState().setSseConnected(false);
|
|
||||||
|
|
||||||
if (!disposed && !controller.signal.aborted) {
|
|
||||||
const liveState = runtimeStore.getState();
|
|
||||||
if (shouldUseLiveSync(liveState)) {
|
|
||||||
reconnectTimer = setTimeout(() => {
|
|
||||||
void ensureStream();
|
|
||||||
}, STREAM_RECONNECT_DELAY_MS);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -170,7 +71,7 @@ export function useTrainingRuntimeLifecycle(): void {
|
||||||
|
|
||||||
const metricsTimer = setInterval(() => {
|
const metricsTimer = setInterval(() => {
|
||||||
const state = runtimeStore.getState();
|
const state = runtimeStore.getState();
|
||||||
if (shouldUseLiveSync(state) || state.currentStep > 0) {
|
if (state.isTrainingRunning || state.phase === "training" || state.currentStep > 0) {
|
||||||
void pollMetrics();
|
void pollMetrics();
|
||||||
}
|
}
|
||||||
}, METRICS_POLL_INTERVAL_MS);
|
}, METRICS_POLL_INTERVAL_MS);
|
||||||
|
|
@ -179,7 +80,6 @@ export function useTrainingRuntimeLifecycle(): void {
|
||||||
disposed = true;
|
disposed = true;
|
||||||
clearInterval(statusTimer);
|
clearInterval(statusTimer);
|
||||||
clearInterval(metricsTimer);
|
clearInterval(metricsTimer);
|
||||||
stopStream();
|
|
||||||
};
|
};
|
||||||
}, []);
|
}, []);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -325,19 +325,34 @@ rm -rf "$LLAMA_CPP_DIR"
|
||||||
|
|
||||||
# Detect GPU compute capability and limit CUDA architectures
|
# Detect GPU compute capability and limit CUDA architectures
|
||||||
# Without this, cmake builds for ALL default archs (very slow)
|
# Without this, cmake builds for ALL default archs (very slow)
|
||||||
|
# CUDA_ARCHS can be pre-set via env var (e.g. Docker ARG) to
|
||||||
|
# include targets not detectable at build time (no GPU access).
|
||||||
|
# We always include 86 (A10) and merge with detected GPUs.
|
||||||
|
_add_arch() {
|
||||||
|
local _a="$1"
|
||||||
|
case ";$CUDA_ARCHS;" in
|
||||||
|
*";$_a;"*) ;;
|
||||||
|
*) CUDA_ARCHS="${CUDA_ARCHS:+$CUDA_ARCHS;}$_a" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
CUDA_ARCHS=""
|
CUDA_ARCHS=""
|
||||||
|
# Merge any pre-set CUDA_ARCHS from environment
|
||||||
|
if [ -n "${CUDA_ARCHS_EXTRA:-}" ]; then
|
||||||
|
IFS=';' read -ra _env_archs <<< "$CUDA_ARCHS_EXTRA"
|
||||||
|
for _ea in "${_env_archs[@]}"; do
|
||||||
|
_ea=$(echo "$_ea" | tr -d '[:space:]')
|
||||||
|
[ -n "$_ea" ] && _add_arch "$_ea"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
# Always include sm_86 (A10)
|
||||||
|
_add_arch "86"
|
||||||
if command -v nvidia-smi &>/dev/null; then
|
if command -v nvidia-smi &>/dev/null; then
|
||||||
# Read all GPUs, deduplicate (handles mixed-GPU hosts)
|
# Read all GPUs, deduplicate (handles mixed-GPU hosts)
|
||||||
_raw_caps=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null || true)
|
_raw_caps=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null || true)
|
||||||
while IFS= read -r _cap; do
|
while IFS= read -r _cap; do
|
||||||
_cap=$(echo "$_cap" | tr -d '[:space:]')
|
_cap=$(echo "$_cap" | tr -d '[:space:]')
|
||||||
if [[ "$_cap" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
if [[ "$_cap" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
||||||
_arch="${BASH_REMATCH[1]}${BASH_REMATCH[2]}"
|
_add_arch "${BASH_REMATCH[1]}${BASH_REMATCH[2]}"
|
||||||
# Append if not already present
|
|
||||||
case ";$CUDA_ARCHS;" in
|
|
||||||
*";$_arch;"*) ;;
|
|
||||||
*) CUDA_ARCHS="${CUDA_ARCHS:+$CUDA_ARCHS;}$_arch" ;;
|
|
||||||
esac
|
|
||||||
fi
|
fi
|
||||||
done <<< "$_raw_caps"
|
done <<< "$_raw_caps"
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue