Studio: further profile stat corrections

Follow-up to the previous pass, each with a test that fails without it.

- Cancelling a run also sets resume_blocked, so filtering on that flag
  alone dropped the steps and tokens of cancelled work while still
  counting the run and its duration. Only a run a later run resumed
  from is excluded now: the resume claim leaves output_dir intact,
  cancelling clears it, which tells the two apart.
- A fixed getTimezoneOffset() was applied to every historical message,
  so winter records read an hour out when the panel is opened on summer
  time, moving messages near midnight onto the wrong day. The endpoint
  now takes an IANA timezone and converts each timestamp with its own
  offset, keeping the fixed offset as the fallback.
- A fork with no new turn was skipped before its thread was registered,
  so it did not appear in the chat total until its first message. The
  thread is now counted before the cloned rows are suppressed.
- Routed and aliased responses record the model that actually answered
  in responseDetails.responseModelId, while contextUsage.modelId stays
  the requested checkpoint. Most used models now prefers the former.
- Renamed runs showed the model label instead of the name the user
  chose. The name leads and the model moves beside the dataset.
This commit is contained in:
Unsloth 2026-07-28 23:46:45 -07:00
commit 9ae3090b98
5 changed files with 245 additions and 38 deletions

View file

@ -31,20 +31,25 @@ logger = get_logger(__name__)
async def get_profile_stats(
days: int = Query(MAX_DAILY_DAYS, ge = 1, le = MAX_DAILY_DAYS),
tz_offset_minutes: int = Query(0, ge = -MAX_TZ_OFFSET_MINUTES, le = MAX_TZ_OFFSET_MINUTES),
tz: str = Query("", max_length = 64),
current_subject: str = Depends(get_current_subject),
) -> dict[str, Any]:
"""Usage stats for the signed-in user's local history.
``tz_offset_minutes`` is the caller's ``Date.getTimezoneOffset()``. Days and
hours are bucketed with it so a remote browser does not read the server's
calendar.
Days and hours are bucketed in the caller's timezone so a remote browser
does not read the server's calendar. ``tz`` is an IANA name, which carries
each date's own daylight-saving offset; ``tz_offset_minutes`` is the
``Date.getTimezoneOffset()`` fallback for hosts with no tzdata.
"""
try:
# A cold pass parses every message's metadata JSON: ~90 ms at 10k
# messages, ~1.2 s at 260k. Off the event loop so it cannot stall token
# streaming when Settings is opened mid-generation.
return await asyncio.to_thread(
compute_profile_stats, days = days, tz_offset_minutes = tz_offset_minutes
compute_profile_stats,
days = days,
tz_offset_minutes = tz_offset_minutes,
tz_name = tz,
)
except Exception as exc:
raise log_and_http_error(

View file

@ -20,6 +20,7 @@ import json
import threading
import time
from datetime import date, datetime, timedelta, timezone
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing import Any, Optional
from loggers import get_logger
@ -68,16 +69,31 @@ def _iso(day: date) -> str:
return day.isoformat()
def _local_stamp(created_at_ms: int, tz_offset_minutes: int) -> Optional[datetime]:
"""Wall-clock time in the caller's timezone, not the server's.
def _clean_str(value: Any) -> str:
return value.strip() if isinstance(value, str) else ""
``tz_offset_minutes`` follows the browser's ``getTimezoneOffset()``: minutes
to add to local time to reach UTC, so UTC-5 sends 300.
def _resolve_zone(tz_name: str, tz_offset_minutes: int):
"""The caller's zone, preferring an IANA name over a single offset.
A fixed offset is only correct for the half of the year the caller happens
to be in, so a winter message read during summer lands an hour out and can
cross midnight. An IANA name carries each date's own offset. The offset
stays as the fallback for callers that send no name, or hosts with no tzdata.
"""
if tz_name:
try:
return ZoneInfo(tz_name)
except (ValueError, ZoneInfoNotFoundError, OSError):
logger.debug("unknown timezone %r, falling back to fixed offset", tz_name)
return timezone(timedelta(minutes = -tz_offset_minutes))
def _local_stamp(created_at_ms: int, zone) -> Optional[datetime]:
"""Wall-clock time in the caller's timezone, not the server's."""
if created_at_ms <= 0:
return None
utc = datetime.fromtimestamp(created_at_ms / 1000, tz = timezone.utc)
return (utc - timedelta(minutes = tz_offset_minutes)).replace(tzinfo = None)
return datetime.fromtimestamp(created_at_ms / 1000, tz = zone).replace(tzinfo = None)
def _streaks(days: set[date], today: date) -> dict[str, Any]:
@ -160,7 +176,7 @@ class _MessageFold:
bucket["threads"].add(thread_id)
def _fold_messages(conn, tz_offset_minutes: int = 0) -> _MessageFold:
def _fold_messages(conn, zone) -> _MessageFold:
fold = _MessageFold()
rows = conn.execute(
"""
@ -202,6 +218,10 @@ def _fold_messages(conn, tz_offset_minutes: int = 0) -> _MessageFold:
thread_messages = 0
previous_created = None
# A fork is its own visible conversation, so it counts towards the chat
# total from the moment it exists, before any new turn is added.
fold.threads.add(thread_id)
# Forking clones the whole ancestry into the new thread, keeping each
# copy's original timestamp. Counting those again would double every
# metric for the branched-from conversation, so skip anything older
@ -209,7 +229,6 @@ def _fold_messages(conn, tz_offset_minutes: int = 0) -> _MessageFold:
if row["forked_from_thread_id"] and created_at < _as_int(row["thread_created_at"]):
continue
fold.threads.add(thread_id)
fold.messages += 1
thread_messages += 1
@ -219,7 +238,7 @@ def _fold_messages(conn, tz_offset_minutes: int = 0) -> _MessageFold:
thread_seconds += gap
previous_created = created_at
stamp = _local_stamp(created_at, tz_offset_minutes)
stamp = _local_stamp(created_at, zone)
role = row["role"]
if role == "user":
fold.user_messages += 1
@ -267,11 +286,16 @@ def _fold_messages(conn, tz_offset_minutes: int = 0) -> _MessageFold:
fold.tool_calls += _as_int(timing.get("toolCallCount"))
message_tokens = total_tokens
# Only the checkpoint recorded on the turn itself. The thread's
# model_id tracks whatever is selected now, so using it as a
# fallback credits older turns to the wrong model after a switch.
raw_model_id = usage.get("modelId")
model_id = raw_model_id.strip() if isinstance(raw_model_id, str) else ""
# responseDetails carries the model that actually answered, which
# differs from the requested checkpoint whenever a provider routes
# or resolves an alias. contextUsage.modelId is the request, so it
# is only the fallback. The thread's model_id is never used: it
# tracks the current selection, not the one that ran.
details = metadata.get("responseDetails")
details = details if isinstance(details, dict) else {}
model_id = _clean_str(details.get("responseModelId")) or _clean_str(
usage.get("modelId")
)
if model_id:
fold.note_model(model_id, message_tokens)
@ -318,6 +342,16 @@ def _daily_series(fold: _MessageFold, today: date, days: int) -> list[dict[str,
return series
def _superseded(prefix: str) -> str:
"""SQL for "a later run resumed from this one, so its counters live there".
``create_run``'s resume claim sets ``resume_blocked`` and leaves
``output_dir`` alone. Cancelling clears ``output_dir`` while setting the
same flag, so the flag alone cannot tell the two apart.
"""
return f"{prefix}resume_blocked = 1 AND {prefix}output_dir IS NOT NULL"
def _training_stats(conn) -> dict[str, Any]:
row = conn.execute(
"""
@ -333,22 +367,23 @@ def _training_stats(conn) -> dict[str, Any]:
# A resumed run continues its source's step and token counters from the
# checkpoint, so both absolute totals already include the source's work.
# resume_blocked marks a run that some later run resumed from; counting
# only the unresumed tails avoids adding the same progress twice.
# Only a run superseded by a resume is dropped: create_run's claim sets
# resume_blocked while leaving output_dir intact, whereas cancelling clears
# output_dir, so a cancelled run keeps contributing the work it did do.
steps = conn.execute(
"SELECT COALESCE(SUM(final_step), 0) FROM training_runs WHERE resume_blocked = 0"
f"SELECT COALESCE(SUM(final_step), 0) FROM training_runs WHERE NOT ({_superseded('')})"
).fetchone()[0]
# num_tokens is state.num_input_tokens_seen, a running total logged at each
# step, so summing the samples multiplies the real figure. Take each run's
# final counter, the same value get_run_metrics reports.
tokens = conn.execute(
"""
f"""
SELECT COALESCE(SUM(run_tokens), 0) FROM (
SELECT MAX(m.num_tokens) AS run_tokens
FROM training_metrics m
JOIN training_runs r ON r.id = m.run_id
WHERE r.resume_blocked = 0
WHERE NOT ({_superseded("r.")})
GROUP BY m.run_id
)
"""
@ -356,7 +391,7 @@ def _training_stats(conn) -> dict[str, Any]:
recent = conn.execute(
"""
SELECT id, COALESCE(display_name, model_name) AS name, model_name, dataset_name,
SELECT id, display_name, model_name, dataset_name,
status, final_loss, final_step, duration_seconds, started_at
FROM training_runs
ORDER BY started_at DESC
@ -377,7 +412,9 @@ def _training_stats(conn) -> dict[str, Any]:
"recent": [
{
"id": item["id"],
"name": item["name"],
# A renamed run keeps the name the user gave it; otherwise fall
# back to the short model label rather than the full repo id.
"name": _clean_str(item["display_name"]) or _model_label(item["model_name"] or ""),
"modelLabel": _model_label(item["model_name"] or ""),
"datasetLabel": _model_label(item["dataset_name"] or ""),
"status": item["status"],
@ -401,15 +438,20 @@ def _fingerprint(conn) -> tuple:
return (message_row[0], message_row[1], run_row[0], run_row[1])
def compute_profile_stats(days: int = MAX_DAILY_DAYS, tz_offset_minutes: int = 0) -> dict[str, Any]:
def compute_profile_stats(
days: int = MAX_DAILY_DAYS,
tz_offset_minutes: int = 0,
tz_name: str = "",
) -> dict[str, Any]:
"""Aggregate every profile statistic in one pass, memoised per history state."""
days = max(1, min(int(days), MAX_DAILY_DAYS))
tz_offset_minutes = max(
-MAX_TZ_OFFSET_MINUTES, min(int(tz_offset_minutes), MAX_TZ_OFFSET_MINUTES)
)
zone = _resolve_zone(tz_name, tz_offset_minutes)
conn = get_connection()
try:
fingerprint = (_fingerprint(conn), days, tz_offset_minutes)
fingerprint = (_fingerprint(conn), days, tz_offset_minutes, tz_name)
now = time.monotonic()
with _cache_lock:
if (
@ -420,12 +462,12 @@ def compute_profile_stats(days: int = MAX_DAILY_DAYS, tz_offset_minutes: int = 0
return _cache["payload"]
started = time.perf_counter()
fold = _fold_messages(conn, tz_offset_minutes)
fold = _fold_messages(conn, zone)
training = _training_stats(conn)
# "Today" has to match the buckets above, or the newest column and the
# current streak drift by a day whenever the caller is elsewhere.
today = (_local_stamp(int(time.time() * 1000), tz_offset_minutes) or datetime.now()).date()
today = (_local_stamp(int(time.time() * 1000), zone) or datetime.now()).date()
streak = _streaks(set(fold.by_day.keys()), today)
daily = _daily_series(fold, today, days)

View file

@ -343,16 +343,20 @@ def test_resumed_runs_do_not_double_count_steps_or_tokens(stats_db):
"""A resume continues the source's counters, so only the tail is counted."""
conn = studio_db.get_connection()
try:
# 'stopped' at step 10, then claimed by the resume below.
# 'stopped' at step 10, then claimed by the resume below. The claim sets
# resume_blocked and leaves output_dir, which is how it is told apart
# from a cancelled run.
conn.execute(
"INSERT INTO training_runs (id, status, model_name, dataset_name, config_json, "
"started_at, total_steps, final_step, duration_seconds, resume_blocked) "
"VALUES ('src', 'stopped', 'm', 'd', '{}', '2026-01-01T10:00:00', 20, 10, 600, 1)",
"started_at, total_steps, final_step, duration_seconds, output_dir, resume_blocked) "
"VALUES ('src', 'stopped', 'm', 'd', '{}', '2026-01-01T10:00:00', 20, 10, 600, "
"'/runs/out', 1)",
)
conn.execute(
"INSERT INTO training_runs (id, status, model_name, dataset_name, config_json, "
"started_at, total_steps, final_step, duration_seconds, resume_blocked) "
"VALUES ('cont', 'completed', 'm', 'd', '{}', '2026-01-02T10:00:00', 20, 15, 300, 0)",
"started_at, total_steps, final_step, duration_seconds, output_dir, resume_blocked) "
"VALUES ('cont', 'completed', 'm', 'd', '{}', '2026-01-02T10:00:00', 20, 15, 300, "
"'/runs/out', 0)",
)
conn.executemany(
"INSERT INTO training_metrics (run_id, step, num_tokens) VALUES (?, ?, ?)",
@ -373,6 +377,151 @@ def test_resumed_runs_do_not_double_count_steps_or_tokens(stats_db):
assert training["runs"] == 2
def test_cancelled_runs_keep_the_work_they_did(stats_db):
"""Cancelling sets resume_blocked too, but nothing resumed from that run."""
conn = studio_db.get_connection()
try:
# mark_run_cancel_requested clears output_dir and sets resume_blocked.
conn.execute(
"INSERT INTO training_runs (id, status, model_name, dataset_name, config_json, "
"started_at, total_steps, final_step, duration_seconds, output_dir, resume_blocked) "
"VALUES ('cancelled', 'stopped', 'm', 'd', '{}', '2026-01-01T10:00:00', 20, 8, "
"400, NULL, 1)",
)
conn.executemany(
"INSERT INTO training_metrics (run_id, step, num_tokens) VALUES (?, ?, ?)",
[("cancelled", step, step * 100) for step in range(1, 9)],
)
conn.commit()
finally:
conn.close()
training = compute_profile_stats(days = 7)["training"]
assert training["runs"] == 1
assert training["steps"] == 8
assert training["tokens"] == 800
def test_forks_count_as_chats_before_their_first_new_turn(stats_db):
"""A fork is a visible thread the moment it exists."""
now = datetime.now().replace(hour = 12, minute = 0, second = 0, microsecond = 0)
conn = studio_db.get_connection()
try:
_seed_thread(conn, "orig", "m", [(now - timedelta(hours = 2), _metadata(100, 50))])
conn.execute(
"INSERT INTO chat_threads (id, title, model_type, model_id, created_at, "
"updated_at, forked_from_thread_id, forked_from_message_id) "
"VALUES ('branch', 'fork', 'base', 'm', ?, ?, 'orig', 'orig-a0')",
(_ms(now), _ms(now)),
)
conn.execute(
"INSERT INTO chat_messages (id, thread_id, role, content_json, metadata_json, "
"created_at) VALUES ('branch-a0', 'branch', 'assistant', '[]', ?, ?)",
(json.dumps(_metadata(100, 50)), _ms(now - timedelta(hours = 2))),
)
conn.commit()
finally:
conn.close()
stats = compute_profile_stats(days = 7)
# Two conversations, but the cloned turn is not counted twice.
assert stats["totals"]["threads"] == 2
assert stats["totals"]["messages"] == 2
assert stats["totals"]["totalTokens"] == 150
def test_tokens_follow_the_model_that_answered(stats_db):
"""A routed provider records the real producer in responseDetails."""
now = datetime.now()
metadata = _metadata(100, 50)
metadata["contextUsage"]["modelId"] = "openrouter/auto"
metadata["responseDetails"] = {"responseModelId": "anthropic/claude-sonnet-4"}
conn = studio_db.get_connection()
try:
_seed_thread(conn, "routed", "openrouter/auto", [(now, metadata)])
conn.commit()
finally:
conn.close()
stats = compute_profile_stats(days = 7)
assert stats["models"][0]["id"] == "anthropic/claude-sonnet-4"
assert stats["models"][0]["tokens"] == 150
def test_recent_run_name_prefers_the_users_rename(stats_db):
conn = studio_db.get_connection()
try:
conn.execute(
"INSERT INTO training_runs (id, status, model_name, dataset_name, config_json, "
"started_at, display_name) VALUES ('named', 'completed', 'unsloth/llama-3-8b', "
"'tatsu-lab/alpaca', '{}', '2026-01-02T10:00:00', 'Support triage v3')",
)
conn.execute(
"INSERT INTO training_runs (id, status, model_name, dataset_name, config_json, "
"started_at) VALUES ('plain', 'completed', 'unsloth/qwen3-4b', 'my/dataset', '{}', "
"'2026-01-01T10:00:00')",
)
conn.commit()
finally:
conn.close()
recent = {run["id"]: run for run in compute_profile_stats(days = 7)["training"]["recent"]}
assert recent["named"]["name"] == "Support triage v3"
assert recent["named"]["modelLabel"] == "llama-3-8b"
# Unnamed runs fall back to the short label, not the full repo id.
assert recent["plain"]["name"] == "qwen3-4b"
def test_historical_daylight_saving_offsets_are_respected(stats_db):
"""A fixed offset would put a winter message in the wrong hour."""
# 2026-01-15 02:30 UTC. New York is UTC-5 in January, so 21:30 on the 14th.
winter = datetime(2026, 1, 15, 2, 30, tzinfo = timezone.utc)
conn = studio_db.get_connection()
try:
conn.execute(
"INSERT INTO chat_threads (id, title, model_type, model_id, created_at, updated_at) "
"VALUES ('dst', 'dst', 'base', 'm', ?, ?)",
(int(winter.timestamp() * 1000), int(winter.timestamp() * 1000)),
)
conn.execute(
"INSERT INTO chat_messages (id, thread_id, role, content_json, metadata_json, "
"created_at) VALUES ('dst-a0', 'dst', 'assistant', '[]', ?, ?)",
(json.dumps(_metadata(10, 10)), int(winter.timestamp() * 1000)),
)
conn.commit()
finally:
conn.close()
# A browser on summer time sends offset 240 (UTC-4) with the zone name.
named = compute_profile_stats(days = 366, tz_offset_minutes = 240, tz_name = "America/New_York")
invalidate_profile_stats_cache()
offset_only = compute_profile_stats(days = 366, tz_offset_minutes = 240)
assert named["hourly"][21] == 1
assert {day["date"] for day in named["daily"] if day["messages"]} == {"2026-01-14"}
# The fixed offset lands an hour late, which is what the zone name fixes.
assert offset_only["hourly"][22] == 1
def test_unknown_timezone_falls_back_to_the_offset(stats_db):
now = datetime.now()
conn = studio_db.get_connection()
try:
_seed_thread(conn, "tzbad", "m", [(now, _metadata(10, 10))])
conn.commit()
finally:
conn.close()
stats = compute_profile_stats(days = 7, tz_offset_minutes = 0, tz_name = "Not/A/Zone")
assert stats["totals"]["totalTokens"] == 20
def test_days_and_hours_use_the_callers_timezone(stats_db):
"""A remote browser must not be bucketed against the server's calendar."""
# 01:30 UTC. In UTC that is one day; at UTC-4 it is 21:30 the day before.
@ -450,7 +599,11 @@ def test_route_does_not_block_the_event_loop(stats_db, monkeypatch):
from routes import profile_stats as route_module
def slow_compute(days = 366, tz_offset_minutes = 0):
def slow_compute(
days = 366,
tz_offset_minutes = 0,
tz_name = "",
):
time.sleep(0.5)
return {"totals": {"messages": 0}}

View file

@ -91,9 +91,12 @@ export async function loadProfileStats(
signal?: AbortSignal,
): Promise<ProfileStats> {
// Bucket days and hours in this browser's timezone, which is not the
// server's when Studio is reached over the network.
// server's when Studio is reached over the network. The IANA name is what
// gives each historical date its own daylight-saving offset; the current
// offset only covers callers whose host cannot resolve the name.
const query = new URLSearchParams({
tz_offset_minutes: String(new Date().getTimezoneOffset()),
tz: Intl.DateTimeFormat().resolvedOptions().timeZone ?? "",
});
const res = await authFetch(`/api/profile/stats?${query}`, { signal });
if (!res.ok) {

View file

@ -65,14 +65,18 @@ export function TrainingHighlightsCard({ stats }: { stats: ProfileStats }) {
className="flex items-center justify-between gap-3 py-2"
>
<div className="flex min-w-0 flex-col gap-0.5">
{/* A renamed run leads with the name the user chose, so the
model moves down beside the dataset to stay visible. */}
<span
className="min-w-0 truncate text-sm text-foreground"
title={run.name}
>
{run.modelLabel}
{run.name}
</span>
<span className="min-w-0 truncate text-ui-11 text-muted-foreground">
{run.datasetLabel}
{run.name === run.modelLabel
? run.datasetLabel
: `${run.modelLabel} · ${run.datasetLabel}`}
</span>
</div>
<div className="flex shrink-0 items-center gap-4 text-xs tabular-nums">