diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py index f9128d1715..1165b6bb0e 100644 --- a/studio/backend/core/rag/store.py +++ b/studio/backend/core/rag/store.py @@ -158,6 +158,16 @@ def list_documents(conn: sqlite3.Connection, scope: str) -> list[dict]: return [dict(r) for r in rows] +def list_all_documents(conn: sqlite3.Connection) -> list[dict]: + """Every uploaded document across all scopes (KBs, threads, projects).""" + rows = conn.execute( + "SELECT id, scope, kb_id, thread_id, project_id, filename, sha256, status, error, " + "num_chunks, stored_path, created_at " + "FROM documents ORDER BY created_at DESC" + ).fetchall() + return [dict(r) for r in rows] + + def get_document(conn: sqlite3.Connection, document_id: str) -> dict | None: row = conn.execute("SELECT * FROM documents WHERE id=?", (document_id,)).fetchone() return dict(row) if row else None diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index 7a27a58a52..24b6dfb36d 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -5,7 +5,7 @@ Chat history API routes backed by studio.db. """ -from typing import Any, Literal, Optional +from typing import Annotated, Any, Literal, Optional from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -19,13 +19,16 @@ from storage.studio_db import ( clear_chat_history, count_chat_threads, count_forks_for_message, + delete_chat_attachment, delete_chat_threads, delete_chat_project, ensure_chat_project_workspace, fork_chat_thread, + get_chat_attachment, get_chat_project, get_chat_thread, get_chat_message, + list_chat_attachments_page, list_chat_projects, list_chat_legacy_imports, list_chat_settings, @@ -279,6 +282,131 @@ async def delete_threads( return {"status": "deleted"} +@router.get("/attachments") +def list_attachments( + limit: Annotated[int, Query(ge = 1, le = 100)] = 50, + offset: Annotated[int, Query(ge = 0)] = 0, + current_subject: str = Depends(get_current_subject), +) -> dict: + """One bounded page of chat uploads for the settings Data tab.""" + attachments, next_offset = list_chat_attachments_page(limit = limit, offset = offset) + return {"attachments": attachments, "nextOffset": next_offset} + + +def _decode_attachment_base64(payload: str) -> bytes: + """Strict base64 decode of a stored payload. + + Normalizes first: strips whitespace, fixes padding, accepts the URL-safe + alphabet. validate=False would silently drop bad characters and serve + corrupted bytes instead of failing, so raise 422 on anything else. + """ + import base64 + + normalized = "".join(payload.split()) + altchars = b"-_" if ("-" in normalized or "_" in normalized) else None + normalized += "=" * (-len(normalized) % 4) + try: + return base64.b64decode(normalized, altchars = altchars, validate = True) + except Exception as exc: # noqa: BLE001 - corrupt stored payload + raise HTTPException(status_code = 422, detail = "Attachment data is corrupt") from exc + + +_AUDIO_FORMAT_MEDIA_TYPES = { + "mp3": "audio/mpeg", + "wav": "audio/wav", + "ogg": "audio/ogg", + "flac": "audio/flac", +} + + +def _safe_image_media_type(media_type: str) -> str: + """Clamp a data-URL media type to something inert to render. + + Imported chats store image parts verbatim, so the embedded type can be + text/html or image/svg+xml; echoing those would execute markup with the + app origin when opened. Anything not a plain raster type downloads as + bytes instead. + """ + lowered = media_type.strip().lower() + if lowered.startswith("image/") and lowered != "image/svg+xml": + return lowered + return "application/octet-stream" + + +@router.get("/attachments/{message_id}/{attachment_id}/file") +def get_attachment_file( + message_id: str, + attachment_id: str, + current_subject: str = Depends(get_current_subject), +): + """Serve one attachment's stored content: image or audio bytes, or + extracted text.""" + import urllib.parse + + from fastapi.responses import Response + + attachment = get_chat_attachment(message_id, attachment_id) + if attachment is None: + raise HTTPException(status_code = 404, detail = "Attachment not found") + + attachment_content_type = attachment.get("contentType") + texts: list[str] = [] + for part in attachment.get("content") or []: + if not isinstance(part, dict): + continue + image = part.get("image") + if isinstance(image, str) and image[:5].lower() == "data:": + header, _, payload = image.partition(",") + media_type = _safe_image_media_type( + header[5:].split(";", 1)[0] or "application/octet-stream" + ) + if "base64" not in header.lower(): + # RFC 2397 non-base64 form stores percent-encoded bytes. + data = urllib.parse.unquote_to_bytes(payload) + return Response(content = data, media_type = media_type) + data = _decode_attachment_base64(payload) + return Response(content = data, media_type = media_type) + # Audio parts: the attachment adapter stores {data, format} with raw + # base64; compare chats store a bare base64 string. + audio = part.get("audio") + if isinstance(audio, dict) or (isinstance(audio, str) and audio): + if isinstance(audio, dict): + payload = audio.get("data") + audio_format = audio.get("format") + else: + payload = audio.rsplit(",", 1)[-1] + audio_format = None + if isinstance(payload, str) and payload: + data = _decode_attachment_base64(payload) + media_type = ( + attachment_content_type + if isinstance(attachment_content_type, str) + and attachment_content_type.startswith("audio/") + else _AUDIO_FORMAT_MEDIA_TYPES.get( + str(audio_format or "").lower(), "application/octet-stream" + ) + ) + return Response(content = data, media_type = media_type) + text = part.get("text") + if isinstance(text, str) and text: + texts.append(text) + if texts: + return Response(content = "\n".join(texts), media_type = "text/plain; charset=utf-8") + raise HTTPException(status_code = 404, detail = "Attachment has no stored content") + + +@router.delete("/attachments/{message_id}/{attachment_id}") +def delete_attachment( + message_id: str, + attachment_id: str, + current_subject: str = Depends(get_current_subject), +) -> dict: + """Remove one attachment from its chat message.""" + if not delete_chat_attachment(message_id, attachment_id): + raise HTTPException(status_code = 404, detail = "Attachment not found") + return {"ok": True} + + @router.get("/projects", response_model = ChatProjectListResponse) async def list_projects( include_archived: bool = Query(False), current_subject: str = Depends(get_current_subject) @@ -409,7 +537,7 @@ async def get_thread_message( @router.put("/threads/{thread_id}/messages/{message_id}", response_model = ChatMessage) -async def save_thread_message( +def save_thread_message( thread_id: str, message_id: str, payload: ChatMessage, @@ -432,7 +560,7 @@ async def save_thread_message( @router.put("/threads/{thread_id}/messages", response_model = ChatMessageListResponse) -async def replace_thread_messages( +def replace_thread_messages( thread_id: str, payload: ChatMessageSyncRequest, current_subject: str = Depends(get_current_subject), diff --git a/studio/backend/routes/rag.py b/studio/backend/routes/rag.py index e20fea74a3..392a4e0d02 100644 --- a/studio/backend/routes/rag.py +++ b/studio/backend/routes/rag.py @@ -318,6 +318,39 @@ def list_project_documents(project_id: str, subject: str = Depends(get_current_s conn.close() +@router.get("/documents") +def list_all_uploaded_documents(subject: str = Depends(get_current_subject)) -> dict: + """Every uploaded file across chats, projects, and knowledge bases (settings + Data tab).""" + _require_rag() + conn = rag_db.get_connection() + try: + docs = store.list_all_documents(conn) + kb_names = {kb["id"]: kb["name"] for kb in store.list_kbs(conn)} + finally: + conn.close() + + from storage.studio_db import list_chat_projects + + project_names = {p["id"]: p["name"] for p in list_chat_projects(include_archived = True)} + + out = [] + for doc in docs: + view = _doc_view(doc) + stored_path = doc.get("stored_path") + size = None + if stored_path: + try: + size = os.path.getsize(stored_path) + except OSError: + size = None + view["sizeBytes"] = size + view["kbName"] = kb_names.get(doc.get("kb_id")) + view["projectName"] = project_names.get(doc.get("project_id")) + out.append(view) + return {"documents": out} + + @router.delete("/documents/{document_id}") def delete_document(document_id: str, subject: str = Depends(get_current_subject)) -> dict: _require_rag() @@ -424,8 +457,10 @@ _CONTENT_TYPES = { ".txt": "text/plain; charset=utf-8", ".md": "text/markdown; charset=utf-8", ".markdown": "text/markdown; charset=utf-8", - ".html": "text/html; charset=utf-8", - ".htm": "text/html; charset=utf-8", + # Served as plain text, never text/html: an uploaded HTML document rendered + # same-origin would execute its scripts with access to the app's storage. + ".html": "text/plain; charset=utf-8", + ".htm": "text/plain; charset=utf-8", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", } diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.py index 4e0c711b69..d889894d04 100644 --- a/studio/backend/storage/studio_db.py +++ b/studio/backend/storage/studio_db.py @@ -7,6 +7,7 @@ Like auth/storage.py (module-level functions, raw sqlite3, per-function connections) plus WAL mode and PRAGMA foreign_keys = ON for CASCADE deletes. """ +import hashlib import json import logging import os @@ -100,6 +101,7 @@ _schema_lock = threading.Lock() _schema_ready = False _SQLITE_IN_CHUNK_SIZE = 900 _PROJECT_WORKSPACE_SUBDIRS = ("sandbox",) +_CHAT_ATTACHMENT_INVENTORY_VERSION = 1 def _project_slug(name: str) -> str: @@ -313,6 +315,141 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: ) """ ) + tombstone_schema = """ + CREATE TABLE chat_attachment_tombstones ( + thread_id TEXT NOT NULL REFERENCES chat_threads(id) ON DELETE CASCADE, + message_id TEXT NOT NULL, + attachment_id TEXT NOT NULL, + deleted_at INTEGER NOT NULL, + PRIMARY KEY(thread_id, message_id, attachment_id) + ) WITHOUT ROWID + """ + tombstone_table = conn.execute( + """ + SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'chat_attachment_tombstones' + """ + ).fetchone() + if tombstone_table is None: + conn.execute(tombstone_schema) + else: + tombstone_columns = { + row[1] for row in conn.execute("PRAGMA table_info(chat_attachment_tombstones)") + } + tombstone_fk_targets = { + row[2] for row in conn.execute("PRAGMA foreign_key_list(chat_attachment_tombstones)") + } + if "thread_id" not in tombstone_columns or "chat_threads" not in tombstone_fk_targets: + # The first implementation cascaded through chat_messages, which + # erased deletion knowledge during pruneMissing. Rebuild once, + # retaining every tombstone whose owning thread still exists. + conn.execute("SAVEPOINT migrate_chat_attachment_tombstones") + try: + conn.execute( + "ALTER TABLE chat_attachment_tombstones " + "RENAME TO chat_attachment_tombstones_legacy" + ) + conn.execute(tombstone_schema) + if "thread_id" in tombstone_columns: + conn.execute( + """ + INSERT OR IGNORE INTO chat_attachment_tombstones + (thread_id, message_id, attachment_id, deleted_at) + SELECT legacy.thread_id, legacy.message_id, + legacy.attachment_id, legacy.deleted_at + FROM chat_attachment_tombstones_legacy legacy + JOIN chat_threads thread ON thread.id = legacy.thread_id + """ + ) + else: + conn.execute( + """ + INSERT OR IGNORE INTO chat_attachment_tombstones + (thread_id, message_id, attachment_id, deleted_at) + SELECT message.thread_id, legacy.message_id, + legacy.attachment_id, legacy.deleted_at + FROM chat_attachment_tombstones_legacy legacy + JOIN chat_messages message ON message.id = legacy.message_id + """ + ) + conn.execute("DROP TABLE chat_attachment_tombstones_legacy") + conn.execute("RELEASE SAVEPOINT migrate_chat_attachment_tombstones") + except Exception: + conn.execute("ROLLBACK TO SAVEPOINT migrate_chat_attachment_tombstones") + conn.execute("RELEASE SAVEPOINT migrate_chat_attachment_tombstones") + raise + conn.execute( + """ + CREATE TABLE IF NOT EXISTS chat_attachment_inventory ( + message_id TEXT NOT NULL REFERENCES chat_messages(id) ON DELETE CASCADE, + attachment_id TEXT NOT NULL, + name TEXT NOT NULL, + type TEXT, + content_type TEXT, + size_bytes INTEGER, + PRIMARY KEY(message_id, attachment_id) + ) WITHOUT ROWID + """ + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS chat_attachment_inventory_state ( + singleton INTEGER NOT NULL PRIMARY KEY CHECK(singleton = 1), + inventory_version INTEGER NOT NULL DEFAULT 0, + dirty INTEGER NOT NULL DEFAULT 1, + backfilled_at INTEGER NOT NULL + ) + """ + ) + inventory_state_columns = { + row[1] for row in conn.execute("PRAGMA table_info(chat_attachment_inventory_state)") + } + if "inventory_version" not in inventory_state_columns: + conn.execute( + "ALTER TABLE chat_attachment_inventory_state " + "ADD COLUMN inventory_version INTEGER NOT NULL DEFAULT 0" + ) + if "dirty" not in inventory_state_columns: + conn.execute( + "ALTER TABLE chat_attachment_inventory_state " + "ADD COLUMN dirty INTEGER NOT NULL DEFAULT 1" + ) + conn.execute( + """ + CREATE TRIGGER IF NOT EXISTS chat_attachment_inventory_dirty_insert + AFTER INSERT ON chat_messages + BEGIN + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, 0, 1, 0) + ON CONFLICT(singleton) DO UPDATE SET dirty = 1; + END + """ + ) + conn.execute( + """ + CREATE TRIGGER IF NOT EXISTS chat_attachment_inventory_dirty_update + AFTER UPDATE ON chat_messages + BEGIN + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, 0, 1, 0) + ON CONFLICT(singleton) DO UPDATE SET dirty = 1; + END + """ + ) + conn.execute( + """ + CREATE TRIGGER IF NOT EXISTS chat_attachment_inventory_dirty_delete + AFTER DELETE ON chat_messages + BEGIN + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, 0, 1, 0) + ON CONFLICT(singleton) DO UPDATE SET dirty = 1; + END + """ + ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_chat_threads_model_type_created_at ON chat_threads(model_type, created_at)" ) @@ -391,6 +528,21 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: conn.execute( "CREATE INDEX IF NOT EXISTS idx_prompt_lists_created_at ON prompt_lists(created_at)" ) + inventory_state = conn.execute( + """ + SELECT inventory_version, dirty + FROM chat_attachment_inventory_state + WHERE singleton = 1 + """ + ).fetchone() + if ( + inventory_state is None + or inventory_state["inventory_version"] != _CHAT_ATTACHMENT_INVENTORY_VERSION + or inventory_state["dirty"] + ): + _rebuild_chat_attachment_inventory(conn) + _mark_chat_attachment_inventory_clean(conn) + conn.commit() def _prompt_entry_from_row(row: sqlite3.Row) -> dict: @@ -1219,7 +1371,14 @@ def delete_chat_threads(ids: list[str]) -> None: return conn = get_connection() try: + conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) + conn.executemany( + "DELETE FROM chat_attachment_tombstones WHERE thread_id = ?", + [(id,) for id in ids], + ) conn.executemany("DELETE FROM chat_threads WHERE id = ?", [(id,) for id in ids]) + _mark_chat_attachment_inventory_clean(conn) conn.commit() finally: conn.close() @@ -1228,7 +1387,11 @@ def delete_chat_threads(ids: list[str]) -> None: def clear_chat_history() -> None: conn = get_connection() try: + conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) + conn.execute("DELETE FROM chat_attachment_tombstones") conn.execute("DELETE FROM chat_threads") + _mark_chat_attachment_inventory_clean(conn) conn.commit() finally: conn.close() @@ -1354,6 +1517,7 @@ def delete_chat_project(id: str, delete_files: bool = False) -> Optional[dict]: conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) row = conn.execute("SELECT * FROM chat_projects WHERE id = ?", (id,)).fetchone() if row is None: conn.rollback() @@ -1361,6 +1525,7 @@ def delete_chat_project(id: str, delete_files: bool = False) -> Optional[dict]: project = _chat_project_from_row(row) conn.execute("DELETE FROM chat_threads WHERE project_id = ?", (id,)) conn.execute("DELETE FROM chat_projects WHERE id = ?", (id,)) + _mark_chat_attachment_inventory_clean(conn) conn.commit() if delete_files: _delete_project_workspace(project) @@ -1483,15 +1648,285 @@ def _recompute_chat_thread_updated_at(conn: sqlite3.Connection, thread_id: str) ) +_CONTENT_PART_ID_PREFIX = "content-part-sha256-" +_URI_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") + + +def _is_locally_stored_blob(value: str) -> bool: + """True for data URIs or bare base64, never external/blob URI references.""" + candidate = value.lstrip() + if not candidate: + return False + if candidate[:5].lower() == "data:": + return True + if candidate.startswith(("//", "\\\\")): + return False + return _URI_SCHEME_RE.match(candidate) is None + + +def _managed_content_part_payload(part: dict) -> Optional[tuple[str, Any]]: + """Return the locally stored blob payload used to identify a content part.""" + image = part.get("image") + if isinstance(image, str) and image[:5].lower() == "data:": + return "image", image + + audio = part.get("audio") + if isinstance(audio, str) and _is_locally_stored_blob(audio): + return "audio", audio + if isinstance(audio, dict): + data = audio.get("data") + if isinstance(data, str) and _is_locally_stored_blob(data): + return "audio", audio + return None + + +def _content_part_id(part: dict) -> Optional[str]: + """Stable managed id derived from blob data, without mutating inference content.""" + payload = _managed_content_part_payload(part) + if payload is None: + return None + canonical = json.dumps( + payload, + ensure_ascii = False, + separators = (",", ":"), + sort_keys = True, + ).encode("utf-8") + return f"{_CONTENT_PART_ID_PREFIX}{hashlib.sha256(canonical).hexdigest()}" + + +def _chat_attachment_tombstones_for_messages( + conn: sqlite3.Connection, thread_id: str, message_ids: list[str] +) -> dict[str, set[str]]: + tombstones = {message_id: set() for message_id in message_ids} + unique_ids = list(dict.fromkeys(message_ids)) + for start in range(0, len(unique_ids), _SQLITE_IN_CHUNK_SIZE): + chunk = unique_ids[start : start + _SQLITE_IN_CHUNK_SIZE] + placeholders = ",".join("?" for _ in chunk) + rows = conn.execute( + f""" + SELECT message_id, attachment_id + FROM chat_attachment_tombstones + WHERE thread_id = ? AND message_id IN ({placeholders}) + """, + (thread_id, *chunk), + ).fetchall() + for row in rows: + tombstones[row["message_id"]].add(row["attachment_id"]) + return tombstones + + +def _reconcile_chat_message_uploads(message: dict, tombstones: set[str]) -> dict: + """Strip uploads previously deleted through the Data tab from a stale write.""" + if not tombstones: + return message + + reconciled = dict(message) + attachments = message.get("attachments") + if isinstance(attachments, list): + reconciled["attachments"] = [ + attachment + for attachment in attachments + if not (isinstance(attachment, dict) and str(attachment.get("id") or "") in tombstones) + ] + + content = message.get("content") + if isinstance(content, list): + reconciled["content"] = [ + part + for part in content + if not (isinstance(part, dict) and (_content_part_id(part) or "") in tombstones) + ] + return reconciled + + +def _chat_attachment_metadata_text(value, fallback: Optional[str] = None) -> Optional[str]: + """Keep untyped legacy/import metadata safe for SQLite binding.""" + if value is None: + return fallback + if isinstance(value, str): + return value or fallback + if isinstance(value, (bool, int, float)): + return str(value) + # Objects and arrays are not useful display metadata and sqlite3 rejects + # binding them directly. + return fallback + + +def _chat_attachment_inventory_entries( + attachments_json: Optional[str], + content_json: Optional[str], + tombstones: Optional[set[str]] = None, +) -> list[dict]: + tombstones = tombstones or set() + attachments = _json_loads(attachments_json, None) + if not isinstance(attachments, list): + attachments = [] + attachments = [ + attachment + for attachment in attachments + if isinstance(attachment, dict) and attachment.get("id") + ] + attachments.extend(_content_part_attachments(content_json)) + + entries: list[dict] = [] + seen: set[str] = set() + for attachment in attachments: + attachment_id = str(attachment["id"]) + if attachment_id in seen or attachment_id in tombstones: + continue + seen.add(attachment_id) + entries.append( + { + "id": attachment_id, + "name": _chat_attachment_metadata_text(attachment.get("name"), "attachment"), + "type": _chat_attachment_metadata_text(attachment.get("type")), + "contentType": _chat_attachment_metadata_text(attachment.get("contentType")), + "sizeBytes": _chat_attachment_size_bytes(attachment), + } + ) + return entries + + +def _replace_chat_attachment_inventory( + conn: sqlite3.Connection, + message_id: str, + attachments_json: Optional[str], + content_json: Optional[str], + tombstones: Optional[set[str]] = None, +) -> None: + conn.execute("DELETE FROM chat_attachment_inventory WHERE message_id = ?", (message_id,)) + entries = _chat_attachment_inventory_entries( + attachments_json, + content_json, + tombstones, + ) + conn.executemany( + """ + INSERT INTO chat_attachment_inventory + (message_id, attachment_id, name, type, content_type, size_bytes) + VALUES (?, ?, ?, ?, ?, ?) + """, + [ + ( + message_id, + entry["id"], + entry["name"], + entry["type"], + entry["contentType"], + entry["sizeBytes"], + ) + for entry in entries + ], + ) + + +def _mark_chat_attachment_inventory_clean(conn: sqlite3.Connection) -> None: + conn.execute( + """ + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, ?, 0, ?) + ON CONFLICT(singleton) DO UPDATE SET + inventory_version = excluded.inventory_version, + dirty = 0, + backfilled_at = excluded.backfilled_at + """, + ( + _CHAT_ATTACHMENT_INVENTORY_VERSION, + int(datetime.now(timezone.utc).timestamp() * 1000), + ), + ) + + +def _rebuild_chat_attachment_inventory(conn: sqlite3.Connection) -> None: + """Rebuild after schema upgrade or a write from an older Studio build.""" + conn.execute("DELETE FROM chat_attachment_inventory") + tombstones: dict[tuple[str, str], set[str]] = {} + for row in conn.execute( + "SELECT thread_id, message_id, attachment_id FROM chat_attachment_tombstones" + ).fetchall(): + tombstones.setdefault((row["thread_id"], row["message_id"]), set()).add( + row["attachment_id"] + ) + rows = conn.execute( + "SELECT id, thread_id, attachments_json, content_json FROM chat_messages" + ).fetchall() + for row in rows: + _replace_chat_attachment_inventory( + conn, + row["id"], + row["attachments_json"], + row["content_json"], + tombstones.get((row["thread_id"], row["id"]), set()), + ) + + +def _ensure_chat_attachment_inventory_current(conn: sqlite3.Connection) -> None: + state = conn.execute( + """ + SELECT inventory_version, dirty + FROM chat_attachment_inventory_state + WHERE singleton = 1 + """ + ).fetchone() + if ( + state is not None + and state["inventory_version"] == _CHAT_ATTACHMENT_INVENTORY_VERSION + and not state["dirty"] + ): + return + + owns_transaction = not conn.in_transaction + if owns_transaction: + conn.execute("BEGIN IMMEDIATE") + try: + state = conn.execute( + """ + SELECT inventory_version, dirty + FROM chat_attachment_inventory_state + WHERE singleton = 1 + """ + ).fetchone() + if ( + state is None + or state["inventory_version"] != _CHAT_ATTACHMENT_INVENTORY_VERSION + or state["dirty"] + ): + _rebuild_chat_attachment_inventory(conn) + _mark_chat_attachment_inventory_clean(conn) + if owns_transaction: + conn.commit() + except Exception: + if owns_transaction: + conn.rollback() + raise + + def upsert_chat_message(message: dict) -> dict: conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) _raise_if_chat_message_thread_conflicts( conn, message["threadId"], [message["id"]], ) + tombstones = _chat_attachment_tombstones_for_messages( + conn, + message["threadId"], + [message["id"]], + ) + reconciled = _reconcile_chat_message_uploads( + message, + tombstones.get(message["id"], set()), + ) + content_json = json.dumps(reconciled.get("content", [])) + attachments_json = ( + json.dumps(reconciled.get("attachments")) + if reconciled.get("attachments") is not None + else None + ) conn.execute( """ INSERT INTO chat_messages @@ -1507,23 +1942,32 @@ def upsert_chat_message(message: dict) -> dict: WHERE excluded.thread_id = chat_messages.thread_id """, ( - message["id"], - message["threadId"], - message.get("parentId"), - message["role"], - json.dumps(message.get("content", [])), - json.dumps(message.get("attachments")) - if message.get("attachments") is not None + reconciled["id"], + reconciled["threadId"], + reconciled.get("parentId"), + reconciled["role"], + content_json, + attachments_json, + json.dumps(reconciled.get("metadata")) + if reconciled.get("metadata") is not None else None, - json.dumps(message.get("metadata")) - if message.get("metadata") is not None - else None, - int(message["createdAt"]), + int(reconciled["createdAt"]), ), ) - _bump_chat_thread_updated_at(conn, message["threadId"], int(message["createdAt"])) + _replace_chat_attachment_inventory( + conn, + reconciled["id"], + attachments_json, + content_json, + ) + _bump_chat_thread_updated_at( + conn, + reconciled["threadId"], + int(reconciled["createdAt"]), + ) + _mark_chat_attachment_inventory_clean(conn) conn.commit() - return message + return reconciled except Exception: conn.rollback() raise @@ -1539,13 +1983,28 @@ def sync_chat_messages( conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) _raise_if_chat_message_thread_conflicts( conn, thread_id, [m["id"] for m in messages], ) - if prune_missing: - conn.execute("DELETE FROM chat_messages WHERE thread_id = ?", (thread_id,)) + tombstones = _chat_attachment_tombstones_for_messages( + conn, + thread_id, + [m["id"] for m in messages], + ) + reconciled_messages = [ + _reconcile_chat_message_uploads(m, tombstones.get(m["id"], set())) for m in messages + ] + serialized_messages = [ + ( + m, + json.dumps(m.get("content", [])), + json.dumps(m.get("attachments")) if m.get("attachments") is not None else None, + ) + for m in reconciled_messages + ] conn.executemany( """ INSERT INTO chat_messages @@ -1566,20 +2025,46 @@ def sync_chat_messages( thread_id, m.get("parentId"), m["role"], - json.dumps(m.get("content", [])), - json.dumps(m.get("attachments")) if m.get("attachments") is not None else None, + content_json, + attachments_json, json.dumps(m.get("metadata")) if m.get("metadata") is not None else None, int(m["createdAt"]), ) - for m in messages + for m, content_json, attachments_json in serialized_messages ], ) - if prune_missing: - _recompute_chat_thread_updated_at(conn, thread_id) - elif messages: - _bump_chat_thread_updated_at( - conn, thread_id, max(int(m["createdAt"]) for m in messages) + for m, content_json, attachments_json in serialized_messages: + _replace_chat_attachment_inventory( + conn, + m["id"], + attachments_json, + content_json, ) + if prune_missing: + retained_ids = {m["id"] for m in reconciled_messages} + existing_ids = { + row["id"] + for row in conn.execute( + "SELECT id FROM chat_messages WHERE thread_id = ?", + (thread_id,), + ).fetchall() + } + missing_ids = sorted(existing_ids - retained_ids) + for start in range(0, len(missing_ids), _SQLITE_IN_CHUNK_SIZE): + chunk = missing_ids[start : start + _SQLITE_IN_CHUNK_SIZE] + placeholders = ",".join("?" for _ in chunk) + conn.execute( + f"DELETE FROM chat_messages WHERE thread_id = ? AND id IN ({placeholders})", + (thread_id, *chunk), + ) + _recompute_chat_thread_updated_at(conn, thread_id) + elif reconciled_messages: + _bump_chat_thread_updated_at( + conn, + thread_id, + max(int(m["createdAt"]) for m in reconciled_messages), + ) + _mark_chat_attachment_inventory_clean(conn) conn.commit() return list_chat_messages(thread_id) except ChatMessageConflictError: @@ -1613,6 +2098,7 @@ def fork_chat_thread( conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) src = conn.execute( "SELECT * FROM chat_threads WHERE id = ?", (source_thread_id,) ).fetchone() @@ -1686,6 +2172,14 @@ def fork_chat_thread( for row in ancestry ], ) + for row in ancestry: + _replace_chat_attachment_inventory( + conn, + id_map[row["id"]], + row["attachments_json"], + row["content_json"], + ) + _mark_chat_attachment_inventory_clean(conn) conn.commit() thread_row = conn.execute( "SELECT * FROM chat_threads WHERE id = ?", (new_thread_id,) @@ -1744,6 +2238,279 @@ def get_chat_message(thread_id: str, message_id: str) -> Optional[dict]: conn.close() +def _blob_part_base64_len(part: dict) -> int: + """Base64 payload length of an image or audio content part, or 0.""" + image = part.get("image") + if isinstance(image, str) and image[:5].lower() == "data:": + return len(image.rsplit(",", 1)[-1]) + audio = part.get("audio") + if isinstance(audio, str) and _is_locally_stored_blob(audio): + return len(audio.rsplit(",", 1)[-1]) + if isinstance(audio, dict): + data = audio.get("data") + if isinstance(data, str) and _is_locally_stored_blob(data): + return len(data) + return 0 + + +def _chat_attachment_size_bytes(attachment: dict) -> Optional[int]: + """Approximate stored size of one attachment's content parts. + + Image and audio parts hold base64 payloads (decoded bytes ~= 3/4 of the + encoded length); text parts count their character length. None when there + is no sizable content (e.g. a stripped/legacy attachment). + """ + total = 0 + found = False + for part in attachment.get("content") or []: + if not isinstance(part, dict): + continue + blob_len = _blob_part_base64_len(part) + if blob_len > 0: + total += (blob_len * 3) // 4 + found = True + continue + text = part.get("text") + if isinstance(text, str) and text: + total += len(text.encode("utf-8", errors = "ignore")) + found = True + return total if found else None + + +def _content_part_attachments(content_json: Optional[str]) -> list[dict]: + """Managed local blobs stored in content_json, with stable payload ids. + + Exact duplicate blobs intentionally share one inventory id. Deleting that + id removes every identical copy, avoiding ambiguous index-based addressing. + """ + content = _json_loads(content_json, None) + if not isinstance(content, list): + return [] + out: list[dict] = [] + seen: set[str] = set() + for part in content: + if not isinstance(part, dict): + continue + attachment_id = _content_part_id(part) + payload = _managed_content_part_payload(part) + if attachment_id is None or payload is None or attachment_id in seen: + continue + seen.add(attachment_id) + kind, value = payload + content_type = None + if kind == "image" and isinstance(value, str): + content_type = value[5:].split(";", 1)[0].split(",", 1)[0] or None + out.append( + { + "id": attachment_id, + "type": kind, + "name": "Chat image" if kind == "image" else "Chat audio", + "contentType": content_type, + "content": [part], + } + ) + return out + + +def list_chat_attachments_page( + limit: int = 50, offset: int = 0 +) -> tuple[list[dict], Optional[int]]: + """One bounded page from the normalized attachment inventory.""" + if not 1 <= limit <= 100: + raise ValueError("limit must be between 1 and 100") + if offset < 0: + raise ValueError("offset must be non-negative") + + conn = get_connection() + try: + _ensure_chat_attachment_inventory_current(conn) + rows = conn.execute( + """ + SELECT i.attachment_id, i.name, i.type, i.content_type, + i.size_bytes, m.id AS message_id, m.thread_id, + m.created_at, t.title AS thread_title, t.pair_id + FROM chat_attachment_inventory i + JOIN chat_messages m ON m.id = i.message_id + LEFT JOIN chat_threads t ON t.id = m.thread_id + ORDER BY m.created_at DESC, m.id ASC, i.attachment_id ASC + LIMIT ? OFFSET ? + """, + (limit + 1, offset), + ).fetchall() + finally: + conn.close() + + has_more = len(rows) > limit + page_rows = rows[:limit] + attachments = [ + { + "id": row["attachment_id"], + "messageId": row["message_id"], + "threadId": row["thread_id"], + "pairId": row["pair_id"], + "threadTitle": row["thread_title"], + "name": row["name"], + "type": row["type"], + "contentType": row["content_type"], + "sizeBytes": row["size_bytes"], + "createdAt": row["created_at"], + } + for row in page_rows + ] + return attachments, offset + limit if has_more else None + + +def list_chat_attachments() -> list[dict]: + """Compatibility helper returning the full normalized inventory.""" + attachments: list[dict] = [] + offset = 0 + while True: + page, next_offset = list_chat_attachments_page(limit = 100, offset = offset) + attachments.extend(page) + if next_offset is None: + return attachments + offset = next_offset + + +def get_chat_attachment(message_id: str, attachment_id: str) -> Optional[dict]: + """One attachment record (full content) from a message, or None.""" + conn = get_connection() + try: + row = conn.execute( + """ + SELECT message.attachments_json, message.content_json, + EXISTS( + SELECT 1 FROM chat_attachment_tombstones tombstone + WHERE tombstone.thread_id = message.thread_id + AND tombstone.message_id = message.id + AND tombstone.attachment_id = ? + ) AS tombstoned + FROM chat_messages message + WHERE message.id = ? + """, + (attachment_id, message_id), + ).fetchone() + finally: + conn.close() + if row is None or row["tombstoned"]: + return None + attachments = _json_loads(row["attachments_json"], None) + if isinstance(attachments, list): + for attachment in attachments: + if isinstance(attachment, dict) and str(attachment.get("id") or "") == attachment_id: + return attachment + if attachment_id.startswith(_CONTENT_PART_ID_PREFIX): + for attachment in _content_part_attachments(row["content_json"]): + if attachment["id"] == attachment_id: + return attachment + return None + + +def _record_chat_attachment_tombstone( + conn: sqlite3.Connection, thread_id: str, message_id: str, attachment_id: str +) -> None: + conn.execute( + """ + INSERT INTO chat_attachment_tombstones + (thread_id, message_id, attachment_id, deleted_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(thread_id, message_id, attachment_id) DO UPDATE SET + deleted_at = excluded.deleted_at + """, + ( + thread_id, + message_id, + attachment_id, + int(datetime.now(timezone.utc).timestamp() * 1000), + ), + ) + + +def delete_chat_attachment(message_id: str, attachment_id: str) -> bool: + """Remove one stored upload from a message. + + The tombstone is retained while the thread exists, so pruning and later + recreating the same message id cannot restore the deleted upload. If an + ordinary attachment id collides with a content-blob id, both are deleted as + one managed item. + """ + conn = get_connection() + try: + conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) + row = conn.execute( + """ + SELECT thread_id, attachments_json, content_json + FROM chat_messages WHERE id = ? + """, + (message_id,), + ).fetchone() + if row is None: + conn.rollback() + return False + + attachments = _json_loads(row["attachments_json"], None) + updated_attachments_json = row["attachments_json"] + deleted_attachment = False + if isinstance(attachments, list): + remaining_attachments = [ + attachment + for attachment in attachments + if not ( + isinstance(attachment, dict) + and str(attachment.get("id") or "") == attachment_id + ) + ] + deleted_attachment = len(remaining_attachments) != len(attachments) + if deleted_attachment: + updated_attachments_json = json.dumps(remaining_attachments) + + content = _json_loads(row["content_json"], None) + updated_content_json = row["content_json"] + deleted_content = False + if attachment_id.startswith(_CONTENT_PART_ID_PREFIX) and isinstance(content, list): + remaining_content = [ + part + for part in content + if not (isinstance(part, dict) and _content_part_id(part) == attachment_id) + ] + deleted_content = len(remaining_content) != len(content) + if deleted_content: + updated_content_json = json.dumps(remaining_content) + + if not deleted_attachment and not deleted_content: + conn.rollback() + return False + conn.execute( + """ + UPDATE chat_messages + SET attachments_json = ?, content_json = ? + WHERE id = ? + """, + (updated_attachments_json, updated_content_json, message_id), + ) + _record_chat_attachment_tombstone( + conn, + row["thread_id"], + message_id, + attachment_id, + ) + _replace_chat_attachment_inventory( + conn, + message_id, + updated_attachments_json, + updated_content_json, + ) + _mark_chat_attachment_inventory_clean(conn) + conn.commit() + return True + except Exception: + conn.rollback() + raise + finally: + conn.close() + + def list_chat_messages_for_threads(thread_ids: list[str]) -> list[dict]: if not thread_ids: return [] diff --git a/studio/backend/tests/test_chat_attachments.py b/studio/backend/tests/test_chat_attachments.py new file mode 100644 index 0000000000..459587ca9e --- /dev/null +++ b/studio/backend/tests/test_chat_attachments.py @@ -0,0 +1,634 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import base64 +import json +import os +import sqlite3 +import sys + +import pytest +from fastapi import HTTPException + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +from routes import chat_history +from storage import studio_db +from utils.paths import studio_db_path + +PNG_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URL = "data:image/png;base64," + base64.b64encode(PNG_BYTES).decode("ascii") + + +def _reset_studio_db(tmp_path, monkeypatch): + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setenv("UNSLOTH_STUDIO_PROJECTS_HOME", str(tmp_path / "Projects")) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + +def _thread( + thread_id: str = "thread-1", + title: str = "Test Chat", + pair_id: str | None = None, +) -> dict: + return { + "id": thread_id, + "title": title, + "modelType": "base", + "modelId": "test-model", + "pairId": pair_id, + "archived": False, + "createdAt": 1_700_000_000_000, + } + + +def _message( + message_id: str, + created_at: int = 1_700_000_000_000, + attachments = None, + thread_id: str = "thread-1", +) -> dict: + message = { + "id": message_id, + "threadId": thread_id, + "parentId": None, + "role": "user", + "content": [{"type": "text", "text": "hello"}], + "createdAt": created_at, + } + if attachments is not None: + message["attachments"] = attachments + return message + + +def _image_attachment(attachment_id: str = "att-1", name: str = "photo.png") -> dict: + return { + "id": attachment_id, + "type": "image", + "name": name, + "contentType": "image/png", + "content": [{"type": "image", "image": PNG_DATA_URL}], + "status": {"type": "complete"}, + } + + +def _seed( + tmp_path, + monkeypatch, + attachments, + message_id: str = "msg-1", +): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message(_message(message_id, attachments = attachments)) + + +def _set_raw_attachments_json(message_id: str, raw: str) -> None: + conn = sqlite3.connect(studio_db_path()) + try: + conn.execute( + "UPDATE chat_messages SET attachments_json = ? WHERE id = ?", + (raw, message_id), + ) + conn.commit() + finally: + conn.close() + + +def _raw_attachments_json(message_id: str): + conn = sqlite3.connect(studio_db_path()) + try: + row = conn.execute( + "SELECT attachments_json FROM chat_messages WHERE id = ?", + (message_id,), + ).fetchone() + return row[0] if row is not None else None + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# Storage: list_chat_attachments +# --------------------------------------------------------------------------- + + +def test_list_chat_attachments_empty_db(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + assert studio_db.list_chat_attachments() == [] + + +def test_list_chat_attachments_round_trip(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + records = studio_db.list_chat_attachments() + assert len(records) == 1 + record = records[0] + assert record["id"] == "att-1" + assert record["messageId"] == "msg-1" + assert record["threadId"] == "thread-1" + assert record["threadTitle"] == "Test Chat" + assert record["name"] == "photo.png" + assert record["type"] == "image" + assert record["contentType"] == "image/png" + assert record["createdAt"] == 1_700_000_000_000 + # Base64 length estimate is within padding error of the decoded size. + assert abs(record["sizeBytes"] - len(PNG_BYTES)) <= 2 + + +def test_list_chat_attachments_counts_text_utf8(tmp_path, monkeypatch): + text = "héllo wörld é世界" + attachment = { + "id": "att-txt", + "type": "document", + "name": "notes.txt", + "content": [{"type": "text", "text": text}], + } + _seed(tmp_path, monkeypatch, [attachment]) + records = studio_db.list_chat_attachments() + assert records[0]["sizeBytes"] == len(text.encode("utf-8")) + + +def test_list_chat_attachments_no_content_size_is_none(tmp_path, monkeypatch): + attachment = {"id": "att-empty", "name": "ghost.bin", "content": []} + _seed(tmp_path, monkeypatch, [attachment]) + records = studio_db.list_chat_attachments() + assert records[0]["sizeBytes"] is None + assert records[0]["name"] == "ghost.bin" + + +def test_list_chat_attachments_defaults_missing_name(tmp_path, monkeypatch): + attachment = {"id": "att-noname", "content": []} + _seed(tmp_path, monkeypatch, [attachment]) + assert studio_db.list_chat_attachments()[0]["name"] == "attachment" + + +def test_list_chat_attachments_sanitizes_structured_metadata(tmp_path, monkeypatch): + attachment = { + "id": "att-weird", + "name": {"nested": "name"}, + "type": ["image"], + "contentType": {"mime": "image/png"}, + "content": [], + } + _seed(tmp_path, monkeypatch, [attachment]) + record = studio_db.list_chat_attachments()[0] + assert record["name"] == "attachment" + assert record["type"] is None + assert record["contentType"] is None + + +def test_list_chat_attachments_skips_malformed_rows(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + for i, raw in enumerate( + [ + "not json at all", + '{"id": "att-obj"}', + "null", + "[]", + '[{"noid": true}, "just a string", 42]', + '[{"id": ""}]', + ] + ): + message_id = f"msg-bad-{i}" + studio_db.upsert_chat_message(_message(message_id)) + _set_raw_attachments_json(message_id, raw) + studio_db.upsert_chat_message(_message("msg-good", attachments = [_image_attachment("att-ok")])) + records = studio_db.list_chat_attachments() + assert [r["id"] for r in records] == ["att-ok"] + + +def test_list_chat_attachments_orders_newest_first(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message( + _message("msg-old", 1_700_000_000_000, [_image_attachment("att-old")]) + ) + studio_db.upsert_chat_message( + _message("msg-new", 1_700_000_100_000, [_image_attachment("att-new")]) + ) + assert [r["id"] for r in studio_db.list_chat_attachments()] == ["att-new", "att-old"] + + +def test_list_chat_attachments_survives_missing_thread_row(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message(_message("msg-1", attachments = [_image_attachment()])) + conn = sqlite3.connect(studio_db_path()) + try: + conn.execute("DELETE FROM chat_threads WHERE id = 'thread-1'") + conn.commit() + finally: + conn.close() + records = studio_db.list_chat_attachments() + assert len(records) == 1 + assert records[0]["threadTitle"] is None + + +def test_list_chat_attachments_includes_compare_pair_id(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread(pair_id = "pair-1")) + studio_db.upsert_chat_message(_message("msg-compare", attachments = [_image_attachment()])) + record = studio_db.list_chat_attachments()[0] + assert record["threadId"] == "thread-1" + assert record["pairId"] == "pair-1" + + +def test_list_chat_attachments_gone_after_thread_delete(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + studio_db.delete_chat_threads(["thread-1"]) + assert studio_db.list_chat_attachments() == [] + + +# --------------------------------------------------------------------------- +# Storage: get_chat_attachment / delete_chat_attachment +# --------------------------------------------------------------------------- + + +def test_get_chat_attachment_found_and_missing(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + attachment = studio_db.get_chat_attachment("msg-1", "att-1") + assert attachment is not None + assert attachment["content"][0]["image"] == PNG_DATA_URL + assert studio_db.get_chat_attachment("msg-1", "att-missing") is None + assert studio_db.get_chat_attachment("msg-missing", "att-1") is None + + +def test_delete_chat_attachment_keeps_others(tmp_path, monkeypatch): + _seed( + tmp_path, + monkeypatch, + [_image_attachment("att-1"), _image_attachment("att-2", "other.png")], + ) + assert studio_db.delete_chat_attachment("msg-1", "att-1") is True + assert studio_db.get_chat_attachment("msg-1", "att-1") is None + assert studio_db.get_chat_attachment("msg-1", "att-2") is not None + assert [r["id"] for r in studio_db.list_chat_attachments()] == ["att-2"] + + +def test_delete_last_chat_attachment_stores_empty_list(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + assert studio_db.delete_chat_attachment("msg-1", "att-1") is True + # '[]' rather than NULL: a NULL attachments field reads back as missing + # and triggers the legacy IndexedDB backfill, resurrecting the deleted + # attachment on the next chat load. + assert _raw_attachments_json("msg-1") == "[]" + assert studio_db.list_chat_attachments() == [] + # The message itself must survive with its content intact. + message = studio_db.get_chat_message("thread-1", "msg-1") + assert message is not None + assert message["content"] == [{"type": "text", "text": "hello"}] + assert message["attachments"] == [] + + +def test_delete_chat_attachment_missing_targets(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + assert studio_db.delete_chat_attachment("msg-missing", "att-1") is False + assert studio_db.delete_chat_attachment("msg-1", "att-missing") is False + _set_raw_attachments_json("msg-1", "not json") + assert studio_db.delete_chat_attachment("msg-1", "att-1") is False + + +# --------------------------------------------------------------------------- +# Routes: /attachments endpoints (real storage, direct calls) +# --------------------------------------------------------------------------- + + +def test_list_attachments_route(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + result = chat_history.list_attachments(current_subject = "unsloth") + assert [a["id"] for a in result["attachments"]] == ["att-1"] + + +def test_attachment_file_serves_image_bytes(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == PNG_BYTES + assert response.media_type == "image/png" + + +def test_attachment_file_tolerates_whitespace_in_base64(tmp_path, monkeypatch): + encoded = base64.b64encode(PNG_BYTES).decode("ascii") + wrapped = "\n".join(encoded[i : i + 8] for i in range(0, len(encoded), 8)) + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64," + wrapped}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == PNG_BYTES + + +def test_attachment_file_corrupt_base64_is_422(tmp_path, monkeypatch): + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64,%%%"}] + _seed(tmp_path, monkeypatch, [attachment]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 422 + + +def test_attachment_file_accepts_urlsafe_base64(tmp_path, monkeypatch): + data = bytes(range(251, 256)) * 3 # encodes to characters remapped by urlsafe + payload = base64.urlsafe_b64encode(data).decode("ascii") + assert "-" in payload or "_" in payload + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == data + + +def test_attachment_file_accepts_missing_padding(tmp_path, monkeypatch): + payload = base64.b64encode(PNG_BYTES).decode("ascii").rstrip("=") + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == PNG_BYTES + + +def test_attachment_file_serves_percent_encoded_data_url(tmp_path, monkeypatch): + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:text/plain,hello%20world"}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == b"hello world" + # Non-image data URL types are clamped so markup never renders same-origin. + assert response.media_type == "application/octet-stream" + + +def test_attachment_file_serves_text_parts(tmp_path, monkeypatch): + attachment = { + "id": "att-txt", + "type": "document", + "name": "notes.txt", + "content": [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second"}, + ], + } + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-txt", current_subject = "unsloth") + assert response.body.decode("utf-8") == "first\nsecond" + assert response.media_type.startswith("text/plain") + + +def test_attachment_file_no_content_is_404(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [{"id": "att-empty", "name": "ghost", "content": []}]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-empty", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +def test_attachment_file_missing_message_is_404(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("nope", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +def test_attachment_file_non_data_url_image_is_404(tmp_path, monkeypatch): + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "https://example.com/a.png"}] + _seed(tmp_path, monkeypatch, [attachment]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +def test_attachment_file_defaults_media_type(tmp_path, monkeypatch): + payload = base64.b64encode(b"raw-bytes").decode("ascii") + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == b"raw-bytes" + assert response.media_type == "application/octet-stream" + + +def test_attachment_file_svg_media_type(tmp_path, monkeypatch): + svg = b"" + payload = base64.b64encode(svg).decode("ascii") + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/svg+xml;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == svg + # SVG can carry scripts, so it downloads as bytes instead of rendering. + assert response.media_type == "application/octet-stream" + + +def test_delete_attachment_route_then_404(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + result = chat_history.delete_attachment("msg-1", "att-1", current_subject = "unsloth") + assert result == {"ok": True} + with pytest.raises(HTTPException) as excinfo: + chat_history.delete_attachment("msg-1", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +# --------------------------------------------------------------------------- +# Audio attachments (adapter {data, format} and compare-chat bare base64) +# --------------------------------------------------------------------------- + +WAV_BYTES = b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00" +WAV_B64 = base64.b64encode(WAV_BYTES).decode("ascii") + + +def _audio_attachment(attachment_id: str = "att-audio") -> dict: + return { + "id": attachment_id, + "type": "file", + "name": "clip.wav", + "contentType": "audio/wav", + "content": [{"type": "audio", "audio": {"data": WAV_B64, "format": "wav"}}], + "status": {"type": "complete"}, + } + + +def test_audio_attachment_lists_with_size(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_audio_attachment()]) + records = studio_db.list_chat_attachments() + assert len(records) == 1 + assert records[0]["id"] == "att-audio" + assert abs(records[0]["sizeBytes"] - len(WAV_BYTES)) <= 2 + + +def test_audio_attachment_file_serves_bytes(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_audio_attachment()]) + response = chat_history.get_attachment_file("msg-1", "att-audio", current_subject = "unsloth") + assert response.body == WAV_BYTES + assert response.media_type == "audio/wav" + + +def test_audio_attachment_media_type_from_format(tmp_path, monkeypatch): + attachment = _audio_attachment() + attachment["contentType"] = None + attachment["content"] = [{"type": "audio", "audio": {"data": WAV_B64, "format": "mp3"}}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-audio", current_subject = "unsloth") + assert response.media_type == "audio/mpeg" + + +def test_audio_attachment_corrupt_payload_is_422(tmp_path, monkeypatch): + attachment = _audio_attachment() + attachment["content"] = [{"type": "audio", "audio": {"data": "%%%", "format": "wav"}}] + _seed(tmp_path, monkeypatch, [attachment]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-audio", current_subject = "unsloth") + assert excinfo.value.status_code == 422 + + +# --------------------------------------------------------------------------- +# Compare-chat uploads stored as message content parts +# --------------------------------------------------------------------------- + + +def _compare_message(message_id: str = "msg-cmp") -> dict: + return { + "id": message_id, + "threadId": "thread-1", + "parentId": None, + "role": "user", + "content": [ + {"type": "image", "image": PNG_DATA_URL}, + {"type": "audio", "audio": WAV_B64}, + {"type": "text", "text": "compare these"}, + ], + "createdAt": 1_700_000_000_000, + } + + +def _seed_compare(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message(_compare_message()) + + +_CONTENT_PART_PREFIX = "content-part-sha256-" + + +def _content_part_id_for(message_id: str, kind: str) -> str: + """Resolve the stable content-hash id for a message's stored blob. + + Content-part ids are SHA-256 hashes of the blob payload, not array + indices, so tests look them up from the listing instead of hardcoding an + index that would shift when an earlier part is deleted. + """ + for record in studio_db.list_chat_attachments(): + if record["messageId"] == message_id and record["type"] == kind: + return record["id"] + raise AssertionError(f"no {kind} content-part upload for {message_id}") + + +def test_content_part_uploads_are_listed(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + records = studio_db.list_chat_attachments() + # Ids are stable content hashes, not array indices. + assert all(r["id"].startswith(_CONTENT_PART_PREFIX) for r in records) + assert {r["type"] for r in records} == {"image", "audio"} + image = next(r for r in records if r["type"] == "image") + assert image["contentType"] == "image/png" + assert abs(image["sizeBytes"] - len(PNG_BYTES)) <= 2 + audio = next(r for r in records if r["type"] == "audio") + assert audio["type"] == "audio" + + +def test_content_part_file_serves_image_bytes(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + image_id = _content_part_id_for("msg-cmp", "image") + response = chat_history.get_attachment_file("msg-cmp", image_id, current_subject = "unsloth") + assert response.body == PNG_BYTES + assert response.media_type == "image/png" + + +def test_content_part_delete_keeps_text(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + image_id = _content_part_id_for("msg-cmp", "image") + assert studio_db.delete_chat_attachment("msg-cmp", image_id) is True + message = studio_db.get_chat_message("thread-1", "msg-cmp") + types = [p["type"] for p in message["content"]] + assert types == ["audio", "text"] + # The surviving audio blob keeps its own stable hash id after the delete. + remaining = studio_db.list_chat_attachments() + assert [r["type"] for r in remaining] == ["audio"] + assert remaining[0]["id"].startswith(_CONTENT_PART_PREFIX) + assert remaining[0]["id"] != image_id + + +def test_content_part_delete_rejects_non_blob(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + # The text part is not a stored upload, so it never gets an id: only the + # image and audio blobs are addressable. + assert len(studio_db.list_chat_attachments()) == 2 + # A well-formed but unknown content-hash id, and malformed ids, all no-op. + assert studio_db.delete_chat_attachment("msg-cmp", _CONTENT_PART_PREFIX + "0" * 64) is False + assert studio_db.delete_chat_attachment("msg-cmp", "content-part-99") is False + assert studio_db.delete_chat_attachment("msg-cmp", "content-part-x") is False + + +def test_text_only_messages_not_listed_as_uploads(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + # The word "image" inside text must not create phantom upload rows. + message = _message("msg-txt") + message["content"] = [{"type": "text", "text": 'discussing an "image" and "audio" here'}] + studio_db.upsert_chat_message(message) + assert studio_db.list_chat_attachments() == [] + + +def test_remote_image_urls_are_not_listed_as_uploads(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + message = _message("msg-remote") + message["content"] = [ + {"type": "image", "image": "https://example.com/cat.png"}, + {"type": "text", "text": "look at this"}, + ] + studio_db.upsert_chat_message(message) + # No stored bytes: nothing to list, open, or delete. + assert studio_db.list_chat_attachments() == [] + assert studio_db.get_chat_attachment("msg-remote", "content-part-0") is None + assert studio_db.delete_chat_attachment("msg-remote", "content-part-0") is False + stored = studio_db.get_chat_message("thread-1", "msg-remote") + assert [p["type"] for p in stored["content"]] == ["image", "text"] + + +def test_html_data_url_serves_as_octet_stream(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + html_b64 = base64.b64encode(b"").decode() + message = _message("msg-html") + message["content"] = [ + {"type": "image", "image": f"data:text/html;base64,{html_b64}"}, + ] + studio_db.upsert_chat_message(message) + attachment_id = _content_part_id_for("msg-html", "image") + response = chat_history.get_attachment_file( + "msg-html", attachment_id, current_subject = "unsloth" + ) + # Never echo a script-capable media type back under the app origin. + assert response.media_type == "application/octet-stream" + assert response.body == b"" + + +def test_svg_data_url_serves_as_octet_stream(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + svg_b64 = base64.b64encode(b"").decode() + message = _message("msg-svg") + message["content"] = [ + {"type": "image", "image": f"data:image/svg+xml;base64,{svg_b64}"}, + ] + studio_db.upsert_chat_message(message) + attachment_id = _content_part_id_for("msg-svg", "image") + response = chat_history.get_attachment_file("msg-svg", attachment_id, current_subject = "unsloth") + assert response.media_type == "application/octet-stream" + + +def test_png_data_url_keeps_its_media_type(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + image_id = _content_part_id_for("msg-cmp", "image") + response = chat_history.get_attachment_file("msg-cmp", image_id, current_subject = "unsloth") + assert response.media_type == "image/png" diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index b6e218793a..b8601b00f6 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -969,10 +969,10 @@ export function AppSidebar() { ))} - {/* Bulk export and import live in Settings -> Chat -> Data. */} + {/* Bulk export and import live in Settings -> Data. */} - useSettingsDialogStore.getState().openDialog("chat") + useSettingsDialogStore.getState().openDialog("data") } > Export all chats… diff --git a/studio/frontend/src/components/assistant-ui/attachment.tsx b/studio/frontend/src/components/assistant-ui/attachment.tsx index 98ebe5ab5f..b26840cc02 100644 --- a/studio/frontend/src/components/assistant-ui/attachment.tsx +++ b/studio/frontend/src/components/assistant-ui/attachment.tsx @@ -7,6 +7,7 @@ import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { Dialog, + DialogClose, DialogContent, DialogTitle, DialogTrigger, @@ -27,12 +28,7 @@ import { import { AudioWave01Icon, File02Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { PlusIcon, XIcon } from "lucide-react"; -import { - type FC, - type PropsWithChildren, - useEffect, - useState, -} from "react"; +import { type FC, type PropsWithChildren, useEffect, useState } from "react"; import { useShallow } from "zustand/shallow"; const useFileSrc = (file: File | undefined): string | undefined => { @@ -83,7 +79,7 @@ const AttachmentPreview: FC = ({ src }) => { src={src} alt="Preview" className={cn( - "block h-auto max-h-[80vh] w-auto max-w-full object-contain", + "block h-auto max-h-[90dvh] w-auto max-w-[92vw] object-contain", isLoaded ? "aui-attachment-preview-image-loaded" : "aui-attachment-preview-image-loading invisible", @@ -108,12 +104,23 @@ const AttachmentPreviewDialog: FC = ({ children }) => { > {children} - + {/* Chrome-free lightbox: the image floats on the dimmed backdrop with + no dialog panel, and the close button sits in the screen corner. */} + Image Attachment Preview -
- + {/* Clicking the backdrop (anywhere off the image) closes the preview. */} + + @@ -1273,6 +1386,8 @@ export function HubModelPicker({ // Live model id from the runtime store (backend-mirrored active_model), not the dropdown // highlight which can be a staged pick. Disables the update action for it. const loadedModelId = useChatRuntimeStore((s) => s.params.checkpoint); + // Loaded GGUF quant of the active model; marks the matching pinned row. + const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant); // Last-loaded timestamps power the "Recent" sort (vs "Downloaded" = file date). const loadTimes = useModelLoadTimes(value); // Fade the list's top edge once scrolled, and its bottom edge while more @@ -1396,6 +1511,7 @@ export function HubModelPicker({ [expandQuantizations], ); + const [pinnedCollapsed, setPinnedCollapsed] = useState(false); const [downloadedCollapsed, setDownloadedCollapsed] = useState(false); const [otherModelsCollapsed, setOtherModelsCollapsed] = useState(false); const [customFoldersCollapsed, setCustomFoldersCollapsed] = useState(false); @@ -1964,6 +2080,110 @@ export function HubModelPicker({ // logic must use this (not visibleCachedModels) or the picker can go blank. const visibleCachedModelRows = chatOnly ? [] : visibleCachedModels; + // Pinned entries surface in their own section above the Unsloth heading. + // GGUF quants pin individually and their repo stays listed below; non-GGUF + // repos pin whole and leave the Unsloth / Other models groups. + const pinnedIds = usePinnedModelsStore((s) => s.pinned); + const togglePinned = usePinnedModelsStore((s) => s.togglePinned); + const pinnedSet = useMemo(() => new Set(pinnedIds), [pinnedIds]); + + // Candidate pins whose repo still exists in the managed cache. Per-quant + // validation below is required because deleting one variant can leave a + // sibling quant (and therefore the repo row) cached. + const pinnedQuantCandidates = useMemo(() => { + // The existence check ignores the text query (but keeps the format filter) + // so a pinned quant stays findable by its quant name even when the repo id + // does not match the query; querying visibleCachedGguf here would drop the + // repo before the later `${repoId} ${quant}` predicate could surface it. + const cached = new Set( + sortedCachedGguf + .filter((c) => matchesFormatFilter(c.repo_id, true, formatFilter)) + .map((c) => c.repo_id), + ); + return pinnedQuantEntries(pinnedIds).filter((entry) => + cached.has(entry.repoId), + ); + }, [pinnedIds, sortedCachedGguf, formatFilter]); + const pinnedQuantValidationKey = useMemo(() => { + const cacheByRepo = new Map( + sortedCachedGguf.map((repo) => [repo.repo_id, repo]), + ); + return pinnedQuantCandidates + .map((entry) => { + const cached = cacheByRepo.get(entry.repoId); + return `${pinKey(entry.repoId, entry.quant)}@${cached?.size_bytes ?? 0}:${cached?.last_modified ?? 0}`; + }) + .join("\u0000"); + }, [pinnedQuantCandidates, sortedCachedGguf]); + const [pinnedQuantValidation, setPinnedQuantValidation] = useState<{ + key: string; + downloaded: ReadonlySet; + }>({ key: "", downloaded: new Set() }); + + useEffect(() => { + let cancelled = false; + const repoIds = Array.from( + new Set(pinnedQuantCandidates.map((entry) => entry.repoId)), + ); + if (repoIds.length === 0) return; + + void Promise.all( + repoIds.map(async (repoId) => { + try { + const response = await listGgufVariants( + repoId, + hfToken || undefined, + ); + return normalizeGgufVariantsResponse(response).variants + .filter((variant) => variant.downloaded === true) + .map((variant) => pinKey(repoId, variant.quant)); + } catch { + // If the backend cannot verify a quant, hiding the direct-load row + // is safer than claiming a missing file is downloaded. + return []; + } + }), + ).then((groups) => { + if (!cancelled) { + setPinnedQuantValidation({ + key: pinnedQuantValidationKey, + downloaded: new Set(groups.flat()), + }); + } + }); + + return () => { + cancelled = true; + }; + }, [hfToken, pinnedQuantCandidates, pinnedQuantValidationKey]); + const downloadedPinnedQuantKeys = useMemo>( + () => + pinnedQuantValidation.key === pinnedQuantValidationKey + ? pinnedQuantValidation.downloaded + : new Set(), + [pinnedQuantValidation, pinnedQuantValidationKey], + ); + + // Verified downloaded quants, in pin order and filtered by repo id or quant. + const pinnedQuants = useMemo(() => { + const q = normalizeForSearch(debouncedQuery.trim()); + return pinnedQuantCandidates.filter( + (entry) => + downloadedPinnedQuantKeys.has(pinKey(entry.repoId, entry.quant)) && + (!q || + normalizeForSearch(`${entry.repoId} ${entry.quant}`).includes(q)), + ); + }, [ + debouncedQuery, + downloadedPinnedQuantKeys, + pinnedQuantCandidates, + ]); + + const pinnedCachedModelRows = useMemo( + () => visibleCachedModelRows.filter((c) => pinnedSet.has(pinKey(c.repo_id))), + [visibleCachedModelRows, pinnedSet], + ); + // Split downloaded models so non-Unsloth repos get their own "Other models" // section above Fine-tuned. const unslothCachedGguf = useMemo( @@ -1975,12 +2195,18 @@ export function HubModelPicker({ [visibleCachedGguf], ); const unslothCachedModelRows = useMemo( - () => visibleCachedModelRows.filter((c) => isUnslothRepoId(c.repo_id)), - [visibleCachedModelRows], + () => + visibleCachedModelRows.filter( + (c) => isUnslothRepoId(c.repo_id) && !pinnedSet.has(pinKey(c.repo_id)), + ), + [visibleCachedModelRows, pinnedSet], ); const otherCachedModelRows = useMemo( - () => visibleCachedModelRows.filter((c) => !isUnslothRepoId(c.repo_id)), - [visibleCachedModelRows], + () => + visibleCachedModelRows.filter( + (c) => !isUnslothRepoId(c.repo_id) && !pinnedSet.has(pinKey(c.repo_id)), + ), + [visibleCachedModelRows, pinnedSet], ); // Param counts come straight off the unsloth listings the picker already @@ -2076,6 +2302,25 @@ export function HubModelPicker({ const hubOptionKeys = useMemo(() => { const keys: string[] = []; + // Pinned rows sit above the Unsloth heading on the On Device tab. + if ( + section === "downloaded" && + cachedReady && + !pinnedCollapsed && + (pinnedQuants.length > 0 || pinnedCachedModelRows.length > 0) + ) { + keys.push( + ...pinnedQuants.map((entry) => + makeModelOptionKey("pinned-quant", pinKey(entry.repoId, entry.quant)), + ), + ); + keys.push( + ...pinnedCachedModelRows.map((model) => + makeModelOptionKey("downloaded-model", model.repo_id), + ), + ); + } + // Downloaded (Unsloth) rows (query-filtered) on the On Device tab only. if ( section === "downloaded" && @@ -2167,6 +2412,9 @@ export function HubModelPicker({ chatOnly, sortedCustomFolderModels, customFoldersCollapsed, + pinnedQuants, + pinnedCachedModelRows, + pinnedCollapsed, downloadedCollapsed, fineTunedRows, fineTunedCollapsed, @@ -2475,6 +2723,151 @@ export function HubModelPicker({ selected && "bg-[#ececec] dark:bg-[var(--sidebar-accent)]", ); + // Pin toggle at a row's right edge: hidden until the row is hovered (or the + // button is focused), always visible while pinned so pinned rows read as such. + // `small` matches the compact quant-row action sizing; it also skips the + // hide-until-hover classes since small pins render inside a hover-gated group. + const renderPinAction = ( + repoId: string, + quant?: string, + opts?: { className?: string; small?: boolean }, + ) => { + const pinned = pinnedSet.has(pinKey(repoId, quant)); + const target = quant ? `${repoId} ${quant}` : repoId; + return ( + + + + + + {pinned + ? quant + ? "Unpin quant" + : "Unpin model" + : quant + ? "Pin quant to the top" + : "Pin model to the top"} + + + ); + }; + + // A pinned quant: repo name with the quant as a grey chip. One click loads + // that quant directly, no expansion needed. + const renderPinnedQuantRow = (entry: { repoId: string; quant: string }) => { + const optionKey = makeModelOptionKey( + "pinned-quant", + pinKey(entry.repoId, entry.quant), + ); + const { owner, name } = splitRepoLabel(entry.repoId); + const isSelected = value === entry.repoId && activeGgufVariant === entry.quant; + const isLoaded = + modelIdsMatchForPicker(loadedModelId, entry.repoId) && + !ggufVariantsMatchForPicker(activeGgufVariant, null) && + ggufVariantsMatchForPicker(activeGgufVariant, entry.quant); + return ( +
+ + + {renderPinAction(entry.repoId, entry.quant, { small: true })} + + + This will remove{" "} + + {entry.repoId} ({entry.quant}) + {" "} + from disk. You can re-download it later. + + } + successMessage={`Deleted ${entry.repoId} ${entry.quant}`} + buttonClassName="p-1" + iconClassName="size-3" + disabled={deleteDisabled} + onConfirm={async () => { + await deleteCachedModel(entry.repoId, entry.quant); + refreshCachedLists(); + // The file is gone, so drop its pin too. + togglePinned(entry.repoId, entry.quant); + }} + /> + +
+ ); + }; + // Shared row renderers so Downloaded (Unsloth) and Other models render alike. const renderDownloadedGgufRow = (c: (typeof visibleCachedGguf)[number]) => { const optionKey = makeModelOptionKey("downloaded-gguf", c.repo_id); @@ -2489,6 +2882,12 @@ export function HubModelPicker({ meta="GGUF" showVision={c.has_vision ?? visionByRepo[c.repo_id]} selected={isSelected} + loaded={isRuntimeLoadedModel( + loadedModelId, + activeGgufVariant, + c.repo_id, + "required", + )} optionProps={hubModelList.getOptionProps(optionKey, isSelected)} onClick={() => toggleGgufExpanded(c.repo_id)} onArrowDownIntoChildren={ @@ -2506,6 +2905,7 @@ export function HubModelPicker({ reportVision(c.repo_id, v)} onSelect={onSelect} hfToken={hfToken || undefined} @@ -2523,6 +2923,7 @@ export function HubModelPicker({ await deleteCachedModel(c.repo_id, quant); refreshCachedLists(); }, + deleteDisabled, }} /> )} @@ -2547,6 +2948,12 @@ export function HubModelPicker({ c.size_bytes, )}`} selected={isSelected} + loaded={isRuntimeLoadedModel( + loadedModelId, + activeGgufVariant, + c.repo_id, + "none", + )} optionProps={hubModelList.getOptionProps( optionKey, isSelected, @@ -2562,6 +2969,7 @@ export function HubModelPicker({ className={downloadedRowButtonClassName} />
+ {renderPinAction(c.repo_id)} deleteCachedModel(c.repo_id)} + disabled={deleteDisabled} + onConfirm={async () => { + await deleteCachedModel(c.repo_id); + if (pinnedSet.has(pinKey(c.repo_id))) { + togglePinned(c.repo_id); + } + }} onDeleted={refreshCachedLists} /> @@ -2749,12 +3163,36 @@ export function HubModelPicker({ ) : null} + {/* Pinned quants and models sit above the Unsloth heading so + favorites are always first. Filtered by the query like the + sections below. */} + {showDownloaded && + (pinnedQuants.length > 0 || + pinnedCachedModelRows.length > 0) ? ( + <> + } + collapsed={pinnedCollapsed} + onToggle={() => setPinnedCollapsed((v) => !v)} + > + Pinned + + {!pinnedCollapsed && pinnedQuants.map(renderPinnedQuantRow)} + {!pinnedCollapsed && + pinnedCachedModelRows.map(renderDownloadedModelRow)} + + ) : null} + {/* Downloaded (Unsloth) stays visible (filtered) while searching. */} {showDownloaded && (unslothCachedGguf.length > 0 || unslothCachedModelRows.length > 0) ? ( <> 0 || + pinnedCachedModelRows.length > 0 + } collapsed={downloadedCollapsed} onToggle={() => setDownloadedCollapsed((v) => !v)} action={ @@ -2896,6 +3334,8 @@ export function HubModelPicker({ )} @@ -3497,6 +3974,12 @@ export function HubModelPicker({ : (vram?.detail ?? extractParamLabel(id)) } selected={value === id} + loaded={isRuntimeLoadedModel( + loadedModelId, + activeGgufVariant, + id, + isKnownGgufRepo(id) ? "required" : "none", + )} optionProps={hubModelList.getOptionProps( optionKey, value === id, @@ -3546,6 +4029,7 @@ export function HubModelPicker({ await deleteCachedModel(id, quant); refreshCachedLists(); }, + deleteDisabled, }} /> )} @@ -3586,6 +4070,12 @@ export function HubModelPicker({ .join(" · ") } selected={value === id} + loaded={isRuntimeLoadedModel( + loadedModelId, + activeGgufVariant, + id, + isSearchGguf ? "required" : "none", + )} optionProps={hubModelList.getOptionProps( optionKey, value === id, @@ -3637,6 +4127,7 @@ export function HubModelPicker({ await deleteCachedModel(id, quant); refreshCachedLists(); }, + deleteDisabled, }} /> )} @@ -3687,6 +4178,8 @@ export function HubModelPicker({ function FineTunedRows({ adapters, value, + loadedModelId, + activeGgufVariant, onSelect, onModelsChange, deleteDisabled = false, @@ -3697,6 +4190,8 @@ function FineTunedRows({ }: { adapters: LoraModelOption[]; value?: string; + loadedModelId?: string; + activeGgufVariant?: string | null; onSelect: (id: string, meta: ModelSelectorChangeMeta) => void; onModelsChange?: (deletedModel?: DeletedModelRef) => void; deleteDisabled?: boolean; @@ -3753,6 +4248,12 @@ function FineTunedRows({ label={adapter.name} meta={meta} selected={value === adapter.id} + loaded={isRuntimeLoadedModel( + loadedModelId, + activeGgufVariant, + adapter.id, + isLocalGgufDir || isExportedGguf ? "required" : "none", + )} optionProps={loraModelList.getOptionProps( optionKey, value === adapter.id, diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pinned-models.ts b/studio/frontend/src/components/assistant-ui/model-selector/pinned-models.ts new file mode 100644 index 0000000000..4835c4c0cf --- /dev/null +++ b/studio/frontend/src/components/assistant-ui/model-selector/pinned-models.ts @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Pinned models for the model selector's On Device list, persisted in +// localStorage so pins survive reloads. GGUF quants pin individually +// (repoId + quant); non-GGUF repos pin as a whole. Pinned entries surface +// in a "Pinned" section above the Unsloth/Downloaded group. + +import { create } from "zustand"; + +const KEY = "unsloth_pinned_models"; + +// Entries are stored as strings: "repoId" pins a whole (non-GGUF) repo, +// "repoId::quant" pins one GGUF quant. Neither part contains "::". +export function pinKey(repoId: string, quant?: string): string { + return quant ? `${repoId}::${quant}` : repoId; +} + +export interface PinnedQuantEntry { + repoId: string; + quant: string; +} + +/** The pinned GGUF quants, in pin order. Plain repo pins are excluded. */ +export function pinnedQuantEntries(pinned: string[]): PinnedQuantEntry[] { + const out: PinnedQuantEntry[] = []; + for (const key of pinned) { + const sep = key.indexOf("::"); + if (sep <= 0) continue; + const repoId = key.slice(0, sep); + const quant = key.slice(sep + 2); + if (repoId && quant) out.push({ repoId, quant }); + } + return out; +} + +function readPinned(): string[] { + try { + const raw = JSON.parse(localStorage.getItem(KEY) ?? "[]"); + return Array.isArray(raw) + ? raw.filter((v): v is string => typeof v === "string") + : []; + } catch { + return []; + } +} + +function writePinned(pinned: string[]): void { + try { + localStorage.setItem(KEY, JSON.stringify(pinned)); + } catch { + // Ignore unavailable storage; pins stay session-only. + } +} + +interface PinnedModelsState { + pinned: string[]; + togglePinned: (repoId: string, quant?: string) => void; +} + +export const usePinnedModelsStore = create((set) => ({ + pinned: readPinned(), + togglePinned: (repoId, quant) => + set((state) => { + const key = pinKey(repoId, quant); + const next = state.pinned.includes(key) + ? state.pinned.filter((id) => id !== key) + : [...state.pinned, key]; + writePinned(next); + return { pinned: next }; + }), +})); diff --git a/studio/frontend/src/components/ui/tooltip.tsx b/studio/frontend/src/components/ui/tooltip.tsx index cbc1a09a8f..91745af5ec 100644 --- a/studio/frontend/src/components/ui/tooltip.tsx +++ b/studio/frontend/src/components/ui/tooltip.tsx @@ -67,9 +67,14 @@ function TooltipTrigger({ const handleClick = useCallback( (e: React.MouseEvent) => { + // Run the composed handler first: when this trigger wraps another Radix + // trigger (e.g. DialogTrigger around an attachment tile), that trigger's + // action is skipped if the event is already default-prevented. + onClick?.(e); + // preventDefault keeps Radix Tooltip's internal close-on-click from + // undoing the tap-toggle below (its composed handler checks it). e.preventDefault(); toggle?.(); - onClick?.(e); }, [toggle, onClick], ); diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 0f6af38033..3ba1ad6fe4 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -2,7 +2,11 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { authFetch } from "@/features/auth"; +// These helpers are deliberately API-layer-only and are not part of their +// features' React-facing public barrels. +// eslint-disable-next-line no-restricted-imports import { hubTokenHeader } from "@/features/hub/lib/hub-token-header"; +// eslint-disable-next-line no-restricted-imports import { consumeNativePathToken } from "@/features/native-intents/api"; import { formatFastApiDetail } from "@/lib/format-fastapi-error"; import type { @@ -437,6 +441,73 @@ export async function listChatThreads( return Array.isArray(data.threads) ? data.threads : []; } +/** One chat message attachment, as listed for the settings uploaded-files view. */ +export interface ChatAttachmentRecord { + id: string; + messageId: string; + threadId: string; + pairId?: string | null; + threadTitle?: string | null; + name: string; + type?: string | null; + contentType?: string | null; + sizeBytes?: number | null; + createdAt?: number | null; +} + +export interface ChatAttachmentPage { + attachments: ChatAttachmentRecord[]; + nextOffset: number | null; +} + +export async function listChatAttachments( + offset = 0, + limit = 50, +): Promise { + const params = new URLSearchParams({ + limit: String(limit), + offset: String(offset), + }); + const response = await authFetch(`/api/chat/attachments?${params}`); + const data = await parseJsonOrThrow<{ + attachments: ChatAttachmentRecord[]; + nextOffset: number | null; + }>(response); + return { + attachments: Array.isArray(data.attachments) ? data.attachments : [], + nextOffset: + typeof data.nextOffset === "number" && Number.isFinite(data.nextOffset) + ? data.nextOffset + : null, + }; +} + +/** Stored attachment content (image bytes or extracted text) as a Blob. */ +export async function fetchChatAttachmentBlob( + messageId: string, + attachmentId: string, +): Promise { + const response = await authFetch( + `/api/chat/attachments/${encodeURIComponent(messageId)}/${encodeURIComponent(attachmentId)}/file`, + ); + if (!response.ok) { + const body = await response.json().catch(() => null); + throw new Error(parseErrorText(response.status, body)); + } + return response.blob(); +} + +export async function deleteChatAttachment( + messageId: string, + attachmentId: string, +): Promise { + const response = await authFetch( + `/api/chat/attachments/${encodeURIComponent(messageId)}/${encodeURIComponent(attachmentId)}`, + { method: "DELETE" }, + ); + await parseJsonOrThrow<{ ok: boolean }>(response); +} + export async function getChatThread( threadId: string, ): Promise { @@ -960,7 +1031,8 @@ export async function* streamChatCompletions( parsed.type === "reasoning_summary" ) { yield { - _reasoningDurationMs: (parsed as { duration_ms?: number }).duration_ms, + _reasoningDurationMs: (parsed as { duration_ms?: number }) + .duration_ms, } as unknown as OpenAIChatChunk; separatorIndex = buffer.search(/\r?\n\r?\n/); continue; diff --git a/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts b/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts index bfb3eeb14c..a08bd5fa54 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts @@ -217,6 +217,40 @@ export async function archiveChatItem( notifyChatHistoryUpdated(); } +export async function archiveAllChatItems( + activeId?: string, + onSelect?: (view: { mode: "single"; newThreadNonce: string }) => void, +): Promise { + const threads = await listStoredChatThreads({ includeArchived: true }); + // Boolean() mirrors groupThreads: legacy records may have archived + // undefined/null, which must count as "not archived". + const toArchive = threads.filter((t) => !t.archived); + if (toArchive.length === 0) return 0; + + for (const t of toArchive) cancelIfRunning(t.id); + + await Promise.all( + toArchive.map((t) => updateStoredChatThread(t.id, { archived: true })), + ); + + // Reset only when this action archived the active single thread or compare + // pair. An already-archived chat opened from the archive is not in + // toArchive and must stay open. + const archivedActive = + activeId !== undefined && + toArchive.some( + (thread) => thread.id === activeId || thread.pairId === activeId, + ); + if (archivedActive) { + useChatRuntimeStore.getState().setActiveThreadId(null); + onSelect?.({ mode: "single", newThreadNonce: crypto.randomUUID() }); + } + + notifyChatHistoryUpdated(); + // Report sidebar items, not raw threads: a compare pair reads as one chat. + return groupThreads(toArchive).length; +} + export async function unarchiveChatItem(item: SidebarItem): Promise { const threadIds: string[] = item.type === "single" diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts index a8b5fc23ad..b0059b57b1 100644 --- a/studio/frontend/src/features/chat/index.ts +++ b/studio/frontend/src/features/chat/index.ts @@ -3,10 +3,15 @@ export { ChatPage, validateChatSearch, type ChatSearch } from "./chat-page"; export { + deleteChatAttachment, + fetchChatAttachmentBlob, getInferenceStatus, + listChatAttachments, listGgufVariants, listLocalModels, loadModel, + type ChatAttachmentPage, + type ChatAttachmentRecord, type LocalModelInfo, } from "./api/chat-api"; export type { GgufVariantDetail } from "./types/api"; @@ -17,6 +22,10 @@ export { type Preset, } from "./chat-settings-sheet"; export { useChatRuntimeStore } from "./stores/chat-runtime-store"; +export { + CHAT_RAG_CAPTION_KEY, + CHAT_RAG_OCR_KEY, +} from "./stores/chat-runtime-store"; export { preferFullToolOutput, toolOutputKey, @@ -46,12 +55,16 @@ export { setTrainingCompareHandoff } from "./lib/training-compare-handoff"; export type { ProjectRecord } from "./types"; export { clearAllChats, countAllChats } from "./utils/clear-all-chats"; export { listStoredChatThreads } from "./utils/chat-history-storage"; +export { emitChatAttachmentDeleted } from "./utils/chat-attachment-events"; export { ArtifactCard } from "./artifacts/artifact-card"; export { useChatArtifactsStore, useSelectedChatArtifact, } from "./artifacts/store"; -export { downloadChatExport } from "./utils/export-chat-history"; +export { + downloadChatExport, + downloadArchivedChatExport, +} from "./utils/export-chat-history"; export { clearNewChatDraft, composerDraftKey, @@ -60,10 +73,14 @@ export { } from "./utils/composer-draft"; export { EXPORT_FORMATS_LIST, + buildFineTuneJsonl, bulkExportConversationsByScope, + exportFineTuneJsonl, importConversationsFromFile, + type FineTuneFormat, } from "./prompt-storage/prompt-storage-dialog"; export { + archiveAllChatItems, archiveChatItem, deleteChatItem, renameChatItem, diff --git a/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx b/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx index 33fc6286c0..68f24a7b08 100644 --- a/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx +++ b/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx @@ -186,7 +186,16 @@ function contentBlocksToText(content: unknown): string { // predate the user's next message); the parent chain is timestamp-independent. type _Msg = { id: string; parentId?: string | null; createdAt?: number }; -function orderByParentChain(messages: T[]): T[] { +function orderByParentChain( + messages: T[], + options: { + /** Append messages off the selected chain (abandoned branches) at the + * end. Full exports keep everything; fine-tune conversion must not, + * since alternate replies would merge into one conversation. */ + includeSiblings?: boolean; + } = {}, +): T[] { + const { includeSiblings = true } = options; const byId = new Map(messages.map((m) => [m.id, m])); const childrenOf = new Map(); for (const m of messages) { @@ -207,7 +216,9 @@ function orderByParentChain(messages: T[]): T[] { byId.delete(next.id); } - for (const [, m] of byId) result.push(m); + if (includeSiblings) { + for (const [, m] of byId) result.push(m); + } return result; } @@ -543,6 +554,214 @@ export async function exportProjectConversations( ); } +// ── Fine-tuning export ───────────────────────────────────────────────────── +// One JSONL line per conversation: {"messages": [{"role", "content"}]} with +// string-only content in system/user/assistant turns. Unsloth's training tab +// detects this as ChatML natively (no column mapping, no standardization) and +// it works with train-on-completions masking, which only trains on assistant +// turns. Reasoning, tool calls, and images are dropped: clean SFT targets. + +export type FineTuneMessage = { + role: "system" | "user" | "assistant"; + content: string; +}; + +const FINE_TUNE_ROLES = new Set(["system", "user", "assistant"]); + +/** Plain text of a message: text blocks plus text-type attachment parts. */ +function messageToPlainText(msg: { + content: unknown; + attachments?: unknown; +}): string { + const parts: string[] = []; + const collect = (blocks: unknown) => { + // Legacy and imported histories can store content as a plain string. + if (typeof blocks === "string") { + if (blocks.trim()) parts.push(blocks); + return; + } + if (!Array.isArray(blocks)) return; + for (const b of blocks) { + if (!b || typeof b !== "object") { + continue; + } + const block = b as Record; + if (block.type === "text" && typeof block.text === "string" && block.text) { + parts.push(block.text); + } + } + }; + collect(msg.content); + if (Array.isArray(msg.attachments)) { + for (const attachment of msg.attachments as Array<{ content?: unknown }>) { + collect(attachment?.content); + } + } + return parts.join("\n\n").trim(); +} + +/** Merge consecutive same-role turns so chat templates format cleanly. */ +function mergeSameRoleTurns(turns: FineTuneMessage[]): FineTuneMessage[] { + const merged: FineTuneMessage[] = []; + for (const turn of turns) { + const last = merged[merged.length - 1]; + if (last && last.role === turn.role) { + last.content += `\n\n${turn.content}`; + } else { + merged.push({ ...turn }); + } + } + return merged; +} + +/** Conversation turns for fine-tuning, or null when the thread has no + * usable user + assistant exchange. Consecutive same-role turns merge, + * assistant turns before the first user turn drop (an assistant target + * with no prompt teaches nothing), and trailing non-assistant turns drop + * so chat templates format cleanly. */ +function messagesToFineTuneTurns( + messages: Array<{ role: unknown; content: unknown; attachments?: unknown }>, +): FineTuneMessage[] | null { + const raw: FineTuneMessage[] = []; + for (const msg of messages) { + const role = msg.role as FineTuneMessage["role"]; + if (!FINE_TUNE_ROLES.has(role)) continue; + const content = messageToPlainText(msg); + if (!content) continue; + raw.push({ role, content }); + } + const firstUser = raw.findIndex((t) => t.role === "user"); + if (firstUser === -1) return null; + const turns = mergeSameRoleTurns( + raw.filter((t, i) => i >= firstUser || t.role === "system"), + ); + while (turns.length > 0 && turns[turns.length - 1].role !== "assistant") { + turns.pop(); + } + const hasUser = turns.some((t) => t.role === "user"); + const hasAssistant = turns.some((t) => t.role === "assistant"); + return hasUser && hasAssistant ? turns : null; +} + +export type FineTuneExportResult = { + lines: string[]; + conversations: number; + skipped: number; +}; + +/** Dataset shapes the Train tab detects without column mapping. */ +export type FineTuneFormat = "openai" | "sharegpt" | "alpaca"; + +const SHAREGPT_FROM: Record = { + system: "system", + user: "human", + assistant: "gpt", +}; + +/** JSONL lines for one conversation in the chosen format. Alpaca is + * single-turn, so each user to assistant pair becomes its own record with + * the system prompt and earlier exchange carried in the input field. */ +function turnsToFineTuneLines( + turns: FineTuneMessage[], + format: FineTuneFormat, +): string[] { + if (format === "sharegpt") { + return [ + JSON.stringify({ + conversations: turns.map((t) => ({ + from: SHAREGPT_FROM[t.role], + value: t.content, + })), + }), + ]; + } + if (format === "alpaca") { + const lines: string[] = []; + const context: string[] = []; + let system = ""; + let pendingUser: string | null = null; + for (const t of turns) { + if (t.role === "system") { + system = system ? `${system}\n\n${t.content}` : t.content; + continue; + } + if (t.role === "user") { + pendingUser = t.content; + continue; + } + if (pendingUser === null) continue; + const inputParts = []; + if (system) inputParts.push(system); + if (context.length > 0) inputParts.push(context.join("\n")); + lines.push( + JSON.stringify({ + instruction: pendingUser, + input: inputParts.join("\n\n"), + output: t.content, + }), + ); + context.push(`User: ${pendingUser}`, `Assistant: ${t.content}`); + pendingUser = null; + } + return lines; + } + return [JSON.stringify({ messages: turns })]; +} + +/** Every non-archived chat (Recents and Projects) as training-ready JSONL. */ +export async function buildFineTuneJsonl( + format: FineTuneFormat = "openai", +): Promise { + const threads = await listStoredChatThreads({ includeArchived: false }); + const ids = [...new Set(threads.map((t) => t.id))]; + const lines: string[] = []; + let conversations = 0; + let skipped = 0; + for (const id of ids) { + const raw = await listStoredChatMessages(id); + const hasParentIds = raw.some( + (m) => (m as { parentId?: unknown }).parentId != null, + ); + // Chain only: retries/regenerations leave sibling branches, and mixing + // alternate replies into one conversation corrupts the training targets. + const ordered = hasParentIds + ? (orderByParentChain(raw, { includeSiblings: false }) as typeof raw) + : raw; + const turns = messagesToFineTuneTurns(ordered); + const converted = turns ? turnsToFineTuneLines(turns, format) : []; + if (converted.length === 0) { + skipped += 1; + continue; + } + conversations += 1; + lines.push(...converted); + } + return { lines, conversations, skipped }; +} + +/** Download the fine-tuning JSONL; returns the conversation count. */ +export async function exportFineTuneJsonl( + format: FineTuneFormat = "openai", +): Promise { + const { lines, conversations, skipped } = await buildFineTuneJsonl(format); + if (conversations === 0) { + toast.info("No chats with a user and assistant exchange to export."); + return 0; + } + const suffix = format === "openai" ? "" : `-${format}`; + downloadBlob( + lines.join("\n"), + `chat-finetune${suffix}-${exportTs()}.jsonl`, + "application/x-ndjson", + ); + if (skipped > 0) { + toast.success( + `Exported ${conversations} conversation${conversations === 1 ? "" : "s"} (${skipped} without a full exchange skipped).`, + ); + } + return conversations; +} + // role:"tool" results are absorbed into the preceding assistant tool-call // part's `result` field rather than becoming separate records. function oaiMessagesToRecords( diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index 019d0bfd8a..4a8740ac9b 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -56,6 +56,11 @@ import { AudioAttachmentAdapter } from "./audio-attachment-adapter"; import { useChatRuntimeStore } from "./stores/chat-runtime-store"; import { ToolPaneScopeContext, toolPaneScope } from "./tool-output-scope"; import type { MessageRecord, ModelType, ThreadRecord } from "./types"; +import { + chatContentPartAttachmentIdFromSignature, + chatContentPartAttachmentSignature, + onChatAttachmentDeleted, +} from "./utils/chat-attachment-events"; import { deleteStoredChatThreads, ensureStoredChatThread, @@ -890,6 +895,168 @@ function useStudioRuntimeAdapters( ): StudioRuntimeAdapters { const aui = useAui(); + // Mirror Data-tab attachment deletions into the loaded thread. The in-memory + // repository otherwise keeps the attachment, and a later repo-to-storage sync + // (e.g. deleting a message in the thread) would write it back. + useEffect(() => { + let active = true; + let pendingDeletion = Promise.resolve(); + const unsubscribe = onChatAttachmentDeleted((event) => { + pendingDeletion = pendingDeletion.then(async () => { + if (!active) return; + const { messageId, attachmentId } = event; + try { + const thread = aui.thread(); + if (attachmentId.startsWith("content-part-sha256-")) { + for (let attempt = 0; attempt < 3 && active; attempt += 1) { + const exported = thread.export(); + const target = exported.messages.find( + (item) => item.message.id === messageId, + ); + if (!target || !Array.isArray(target.message.content)) return; + const content = target.message.content; + + const signatures = content.map((part) => + chatContentPartAttachmentSignature(part), + ); + const ids = await Promise.all( + signatures.map((signature) => + signature === null + ? null + : chatContentPartAttachmentIdFromSignature(signature), + ), + ); + const targetAttachments = ( + target.message as { + attachments?: readonly { id: string }[]; + } + ).attachments; + const hasTargetAttachment = + Array.isArray(targetAttachments) && + targetAttachments.some( + (attachment) => attachment.id === attachmentId, + ); + if ( + (!ids.includes(attachmentId) && !hasTargetAttachment) || + !active + ) { + return; + } + + // Preserve any messages added or streamed while WebCrypto ran. + // Retry if the target's managed content itself changed. + const latest = thread.export(); + const latestTarget = latest.messages.find( + (item) => item.message.id === messageId, + ); + const latestContent = latestTarget?.message.content; + if (!Array.isArray(latestContent)) return; + const latestSignatures = latestContent.map((part) => + chatContentPartAttachmentSignature(part), + ); + if ( + signatures.length !== latestSignatures.length || + signatures.some( + (signature, index) => signature !== latestSignatures[index], + ) + ) { + continue; + } + + const messages = latest.messages.map((item) => { + if (item.message.id !== messageId) return item; + const attachments = ( + item.message as { + attachments?: readonly { id: string }[]; + } + ).attachments; + return { + ...item, + message: { + ...item.message, + content: latestContent.filter( + (_, index) => ids[index] !== attachmentId, + ), + ...(Array.isArray(attachments) + ? { + attachments: attachments.filter( + (attachment) => + attachment.id !== attachmentId, + ), + } + : {}), + } as typeof item.message, + }; + }); + if (active) thread.import({ ...latest, messages }); + return; + } + return; + } + + const exported = thread.export(); + let changed = false; + const messages = exported.messages.map((item) => { + if (item.message.id !== messageId) return item; + const message = item.message; + const attachments = ( + message as { attachments?: readonly { id: string }[] } + ).attachments; + if ( + Array.isArray(attachments) && + attachments.some( + (attachment) => attachment.id === attachmentId, + ) + ) { + changed = true; + return { + ...item, + message: { + ...message, + attachments: attachments.filter( + (attachment) => attachment.id !== attachmentId, + ), + } as typeof message, + }; + } + if (/^content-part-[0-9]+$/.test(attachmentId)) { + // Legacy synthetic id for a blob stored as a message content part. + const idx = Number(attachmentId.slice("content-part-".length)); + const content = message.content; + if ( + !Array.isArray(content) || + !Number.isInteger(idx) || + idx < 0 || + idx >= content.length + ) { + return item; + } + const part = content[idx] as { type?: string }; + if (part?.type !== "image" && part?.type !== "audio") return item; + changed = true; + return { + ...item, + message: { + ...message, + content: content.filter((_, i) => i !== idx), + } as typeof message, + }; + } + return item; + }); + if (changed && active) thread.import({ ...exported, messages }); + } catch { + // No active thread mounted: storage already holds the truth. + } + }); + return pendingDeletion; + }); + return () => { + active = false; + unsubscribe(); + }; + }, [aui]); + const history = useMemo( () => ({ async load() { diff --git a/studio/frontend/src/features/chat/utils/archived-chat-export.ts b/studio/frontend/src/features/chat/utils/archived-chat-export.ts new file mode 100644 index 0000000000..834dfdcf5f --- /dev/null +++ b/studio/frontend/src/features/chat/utils/archived-chat-export.ts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Minimal views over the `unknown[]` export fields we filter on. +type ExportThreadView = { + id?: string; + archived?: boolean; + projectId?: string | null; +}; +type ExportMessageView = { threadId?: string }; +type ExportProjectView = { id?: string }; + +// Full chat-export backup shape, kept structural so the pure filter below +// stays decoupled from the storage layer that produces it. +export interface ChatExportData { + exportedAt?: string; + version?: number; + threadCount: number; + projects?: unknown[]; + threads: unknown[]; + messages: unknown[]; +} + +// Restrict a full chat export to archived threads, their messages and the +// projects those threads belong to. Pure: never mutates the input, and keeps +// the original thread/message objects so the backup re-imports unchanged. +export function filterArchivedChatExport( + full: T, +): { data: T; archivedCount: number } { + const archivedThreads = (full.threads as ExportThreadView[]).filter( + (thread) => thread.archived === true, + ); + const archivedThreadIds = new Set( + archivedThreads + .map((thread) => thread.id) + .filter((id): id is string => typeof id === "string"), + ); + const messages = (full.messages as ExportMessageView[]).filter( + (message) => + typeof message.threadId === "string" && + archivedThreadIds.has(message.threadId), + ); + const referencedProjectIds = new Set( + archivedThreads + .map((thread) => thread.projectId) + .filter((id): id is string => typeof id === "string"), + ); + const projects = (full.projects as ExportProjectView[] | undefined)?.filter( + (project) => + typeof project.id === "string" && referencedProjectIds.has(project.id), + ); + return { + data: { + ...full, + threadCount: archivedThreads.length, + projects: projects ?? [], + threads: archivedThreads as unknown[], + messages: messages as unknown[], + }, + archivedCount: archivedThreads.length, + }; +} diff --git a/studio/frontend/src/features/chat/utils/chat-attachment-events.ts b/studio/frontend/src/features/chat/utils/chat-attachment-events.ts new file mode 100644 index 0000000000..dbde157890 --- /dev/null +++ b/studio/frontend/src/features/chat/utils/chat-attachment-events.ts @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * Notifies loaded chat runtimes when the Data tab deletes a stored attachment. + * Without this, the active thread's in-memory repository still holds the + * attachment, and any later repo-to-storage sync (e.g. deleting a message in + * that thread) writes it back, undoing the deletion. + */ + +import forge from "node-forge"; + +export type ChatAttachmentDeletedEvent = { + messageId: string; + attachmentId: string; +}; + +const CONTENT_PART_ID_PREFIX = "content-part-sha256-"; +const URI_SCHEME_RE = /^[A-Za-z][A-Za-z0-9+.-]*:/; + +function isLocallyStoredBlob(value: string): boolean { + const candidate = value.trimStart(); + if (!candidate) return false; + if (candidate.slice(0, 5).toLowerCase() === "data:") return true; + if (candidate.startsWith("//") || candidate.startsWith("\\\\")) { + return false; + } + return !URI_SCHEME_RE.test(candidate); +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value + .map((item) => (item === undefined ? "null" : stableJson(item))) + .join(",")}]`; + } + if (value && typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .filter((key) => record[key] !== undefined) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +/** Canonical payload used to detect whether an async hash still describes the + * current message content. */ +export function chatContentPartAttachmentSignature( + part: unknown, +): string | null { + if (!part || typeof part !== "object") return null; + const record = part as Record; + let payload: ["image" | "audio", unknown] | null = null; + if ( + typeof record.image === "string" && + record.image.slice(0, 5).toLowerCase() === "data:" + ) { + payload = ["image", record.image]; + } else if ( + typeof record.audio === "string" && + isLocallyStoredBlob(record.audio) + ) { + payload = ["audio", record.audio]; + } else if (record.audio && typeof record.audio === "object") { + const data = (record.audio as Record).data; + if (typeof data === "string" && isLocallyStoredBlob(data)) { + payload = ["audio", record.audio]; + } + } + if (!payload) return null; + + return stableJson(payload); +} + +/** Mirrors the backend's stable content-part identity without adding private + * metadata to the message payload sent to inference. */ +export async function chatContentPartAttachmentIdFromSignature( + signature: string, +): Promise { + let hex: string | null = null; + const subtle = globalThis.crypto?.subtle; + if (subtle) { + try { + const digest = await subtle.digest( + "SHA-256", + new TextEncoder().encode(signature), + ); + hex = Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + } catch { + // Fall through to the pure-JS implementation below. Some embedded + // browsers expose crypto.subtle but reject it outside a secure context. + } + } + if (hex === null) { + const digest = forge.md.sha256.create(); + digest.update(signature, "utf8"); + hex = digest.digest().toHex(); + } + return `${CONTENT_PART_ID_PREFIX}${hex}`; +} + +type Listener = (event: ChatAttachmentDeletedEvent) => void | Promise; + +const listeners = new Set(); + +export function onChatAttachmentDeleted(listener: Listener): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +export function emitChatAttachmentDeleted( + event: ChatAttachmentDeletedEvent, +): void { + for (const listener of [...listeners]) { + void listener(event); + } +} diff --git a/studio/frontend/src/features/chat/utils/download-json.ts b/studio/frontend/src/features/chat/utils/download-json.ts new file mode 100644 index 0000000000..0c5ce00ff9 --- /dev/null +++ b/studio/frontend/src/features/chat/utils/download-json.ts @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Save `data` as a pretty-printed JSON file via a temporary object URL. Uses +// only the standard Blob/anchor download path so it works in every browser. +export function triggerJsonDownload(data: unknown, filename: string): void { + const blob = new Blob([JSON.stringify(data, null, 2)], { + type: "application/json", + }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); +} diff --git a/studio/frontend/src/features/chat/utils/export-chat-history.ts b/studio/frontend/src/features/chat/utils/export-chat-history.ts index 5faf4dc08a..b4bc64d053 100644 --- a/studio/frontend/src/features/chat/utils/export-chat-history.ts +++ b/studio/frontend/src/features/chat/utils/export-chat-history.ts @@ -1,21 +1,34 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +import { filterArchivedChatExport } from "./archived-chat-export"; import { buildStoredChatExport } from "./chat-history-storage"; +import { triggerJsonDownload } from "./download-json"; export const buildChatExport = buildStoredChatExport; +function dateStamp(): string { + // Date only (no colons) so the filename is valid on every OS. + return new Date().toISOString().slice(0, 10); +} + export async function downloadChatExport(): Promise { const data = await buildChatExport(); - const blob = new Blob([JSON.stringify(data, null, 2)], { - type: "application/json", - }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = `unsloth-chats-${new Date().toISOString().slice(0, 10)}.json`; - document.body.appendChild(a); - a.click(); - a.remove(); - URL.revokeObjectURL(url); + triggerJsonDownload(data, `unsloth-chats-${dateStamp()}.json`); +} + +// Full backup restricted to archived chats. Returns the archived thread count. +export async function buildArchivedChatExport() { + return filterArchivedChatExport(await buildChatExport()); +} + +// Download only the archived chats. Returns how many were exported; skips the +// download entirely when there are none. +export async function downloadArchivedChatExport(): Promise { + const { data, archivedCount } = await buildArchivedChatExport(); + if (archivedCount === 0) { + return 0; + } + triggerJsonDownload(data, `unsloth-archived-chats-${dateStamp()}.json`); + return archivedCount; } diff --git a/studio/frontend/src/features/rag/api/rag-api.ts b/studio/frontend/src/features/rag/api/rag-api.ts index 20800230b5..c0bad1a9d5 100644 --- a/studio/frontend/src/features/rag/api/rag-api.ts +++ b/studio/frontend/src/features/rag/api/rag-api.ts @@ -10,6 +10,7 @@ import type { KnowledgeBase, PreviewTarget, RagDocument, + UploadedDocument, } from "../types/rag"; const RAG_BASE = "/api/rag"; @@ -194,10 +195,25 @@ export function invalidateProjectSources(projectId: string): void { projectSourcesCache.delete(projectId); } -export function deleteDocument(documentId: string): Promise<{ ok: boolean }> { - return ragRequest(`/documents/${encodeURIComponent(documentId)}`, { - method: "DELETE", - }); +export async function listAllDocuments(): Promise { + const data = await ragRequest<{ documents: UploadedDocument[] }>( + "/documents", + ); + return data.documents ?? []; +} + +export async function deleteDocument( + documentId: string, + projectId?: string | null, +): Promise<{ ok: boolean }> { + const result = await ragRequest<{ ok: boolean }>( + `/documents/${encodeURIComponent(documentId)}`, + { + method: "DELETE", + }, + ); + if (projectId) invalidateProjectSources(projectId); + return result; } export function getJob(jobId: string): Promise { @@ -237,7 +253,8 @@ export async function* streamJobEvents( const dataLines: string[] = []; for (const line of rawEvent.split(/\r?\n/)) { - if (line.startsWith("data:")) dataLines.push(line.slice(5).trimStart()); + if (line.startsWith("data:")) + dataLines.push(line.slice(5).trimStart()); } if (dataLines.length > 0) { const dataText = dataLines.join("\n"); diff --git a/studio/frontend/src/features/rag/components/use-rag-documents.ts b/studio/frontend/src/features/rag/components/use-rag-documents.ts index 8e756b2782..8d6433d8c3 100644 --- a/studio/frontend/src/features/rag/components/use-rag-documents.ts +++ b/studio/frontend/src/features/rag/components/use-rag-documents.ts @@ -2,12 +2,11 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { useCallback, useEffect, useRef, useState } from "react"; -import { useChatRuntimeStore } from "@/features/chat"; - import { CHAT_RAG_CAPTION_KEY, CHAT_RAG_OCR_KEY, -} from "@/features/chat/stores/chat-runtime-store"; + useChatRuntimeStore, +} from "@/features/chat"; import { toast } from "@/lib/toast"; import { deleteDocument, @@ -60,9 +59,7 @@ export function useRagDocuments( if (ids.size === 0) return false; const docs = documentsRef.current.filter((d) => ids.has(d.id)); if (docs.length === 0) return false; // sig tracked but doc gone -> allow re-upload - return docs.some( - (d) => d.status !== "completed" || (d.numChunks ?? 0) > 0, - ); + return docs.some((d) => d.status !== "completed" || (d.numChunks ?? 0) > 0); }, []); // True while upload() runs, so the scope-change effect can tell a real switch // from lazy thread materialization mid-upload (which must not reset). @@ -80,9 +77,7 @@ export function useRagDocuments( const patchDoc = useCallback( (documentId: string, patch: Partial) => { setDocuments((rows) => - rows.map((row) => - row.id === documentId ? { ...row, ...patch } : row, - ), + rows.map((row) => (row.id === documentId ? { ...row, ...patch } : row)), ); }, [], @@ -176,49 +171,61 @@ export function useRagDocuments( [patchDoc], ); - const refresh = useCallback(async (opts?: { quiet?: boolean }) => { - if (!scope) return; - if (!opts?.quiet) setLoading(true); - try { - // Merge server truth with local progress so a refresh mid-index keeps a - // live "running %" chip. Failed docs hidden (toast warned at upload). - const rows = (await lister()).filter((row) => row.status !== "failed"); - setDocuments((prev) => { - const merged = rows.map((row) => { - const tracked = prev.find((p) => p.id === row.id); - return tracked && tracked.progress != null && row.status !== "completed" - ? { ...row, progress: tracked.progress } - : row; + const refresh = useCallback( + async (opts?: { quiet?: boolean }) => { + if (!scope) return; + if (!opts?.quiet) setLoading(true); + try { + // Merge server truth with local progress so a refresh mid-index keeps a + // live "running %" chip. Failed docs hidden (toast warned at upload). + const rows = (await lister()).filter((row) => row.status !== "failed"); + setDocuments((prev) => { + const merged = rows.map((row) => { + const tracked = prev.find((p) => p.id === row.id); + return tracked && + tracked.progress != null && + row.status !== "completed" + ? { ...row, progress: tracked.progress } + : row; + }); + // Keep optimistic chips (not yet listed) so a refresh racing an upload + // can't make them vanish. + const serverIds = new Set(rows.map((row) => row.id)); + const pendingLocal = prev.filter( + (row) => row.id.startsWith("pending_") && !serverIds.has(row.id), + ); + return [...merged, ...pendingLocal]; }); - // Keep optimistic chips (not yet listed) so a refresh racing an upload - // can't make them vanish. - const serverIds = new Set(rows.map((row) => row.id)); - const pendingLocal = prev.filter( - (row) => row.id.startsWith("pending_") && !serverIds.has(row.id), - ); - return [...merged, ...pendingLocal]; - }); - } catch (err) { - toast.error("Failed to load documents", { - description: err instanceof Error ? err.message : String(err), - }); - } finally { - if (!opts?.quiet) setLoading(false); - } - }, [scope, lister]); + } catch (err) { + toast.error("Failed to load documents", { + description: err instanceof Error ? err.message : String(err), + }); + } finally { + if (!opts?.quiet) setLoading(false); + } + }, + [scope, lister], + ); // A real switch (thread/KB swap) resets + reloads; first acquiring a scope just // loads. Skip both during materialization mid-upload (scope null -> new thread // while upload() runs) so we don't abort tracking or wipe optimistic chips. useEffect(() => { + const jobs = trackedJobs.current; const prev = prevScopeKeyRef.current; prevScopeKeyRef.current = scopeKey; if (prev !== null && prev !== scopeKey) { - for (const controller of trackedJobs.current.values()) controller.abort(); - trackedJobs.current.clear(); + for (const controller of jobs.values()) controller.abort(); + jobs.clear(); sigByDocId.current.clear(); + // Scope changes intentionally clear the old scope before fetching the new + // one. Keep this synchronous so React StrictMode's setup/cleanup replay + // cannot cancel the only refresh after prevScopeKeyRef has advanced. setDocuments([]); - if (scope) void refresh(); + if (scope) { + // eslint-disable-next-line react-hooks/set-state-in-effect + void refresh(); + } } else if (prev === null && scope && !uploadInFlightRef.current) { void refresh(); } @@ -226,8 +233,8 @@ export function useRagDocuments( // Preserve in-flight tracking when cleanup is the materialization flip, // not a real switch/unmount. if (uploadInFlightRef.current) return; - for (const controller of trackedJobs.current.values()) controller.abort(); - trackedJobs.current.clear(); + for (const controller of jobs.values()) controller.abort(); + jobs.clear(); }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [scopeKey]); @@ -260,21 +267,41 @@ export function useRagDocuments( // otherwise backend env defaults own the ingest policy. const state = useChatRuntimeStore.getState(); const hasLocal = (key: string) => - typeof window !== "undefined" && window.localStorage.getItem(key) !== null; - const ocr = hasLocal(CHAT_RAG_OCR_KEY) ? state.ragOcrScanned : undefined; + typeof window !== "undefined" && + window.localStorage.getItem(key) !== null; + const ocr = hasLocal(CHAT_RAG_OCR_KEY) + ? state.ragOcrScanned + : undefined; const caption = hasLocal(CHAT_RAG_CAPTION_KEY) ? state.ragCaptionFigures : undefined; const result = activeScope.type === "kb" - ? await uploadKnowledgeBaseDocument(activeScope.kbId, file, ocr, caption) + ? await uploadKnowledgeBaseDocument( + activeScope.kbId, + file, + ocr, + caption, + ) : activeScope.type === "project" - ? await uploadProjectDocument(activeScope.projectId, file, ocr, caption) - : await uploadThreadDocument(activeScope.threadId, file, ocr, caption); + ? await uploadProjectDocument( + activeScope.projectId, + file, + ocr, + caption, + ) + : await uploadThreadDocument( + activeScope.threadId, + file, + ocr, + caption, + ); sigByDocId.current.set(result.documentId, fileSignature(file)); if (seenIds.has(result.documentId)) { setDocuments((rows) => rows.filter((row) => row.id !== tempId)); - toast.info(`${result.filename || file.name} is already indexed - skipping`); + toast.info( + `${result.filename || file.name} is already indexed - skipping`, + ); return; } seenIds.add(result.documentId); @@ -341,7 +368,9 @@ export function useRagDocuments( ]); const resolved = - overrideScope instanceof Promise ? await overrideScope : overrideScope; + overrideScope instanceof Promise + ? await overrideScope + : overrideScope; const activeScope = resolved ?? scope; if (!activeScope) { // Materialization failed: drop the chips so they don't hang "pending". @@ -377,7 +406,10 @@ export function useRagDocuments( const prevSig = sigByDocId.current.get(documentId); sigByDocId.current.delete(documentId); try { - await deleteDocument(documentId); + await deleteDocument( + documentId, + scope?.type === "project" ? scope.projectId : undefined, + ); } catch (err) { setDocuments(prev); if (prevSig !== undefined) sigByDocId.current.set(documentId, prevSig); @@ -386,7 +418,7 @@ export function useRagDocuments( }); } }, - [documents], + [documents, scope], ); return { documents, loading, uploading, refresh, upload, remove }; diff --git a/studio/frontend/src/features/rag/index.ts b/studio/frontend/src/features/rag/index.ts index 9e35e345ee..b06c7e09cc 100644 --- a/studio/frontend/src/features/rag/index.ts +++ b/studio/frontend/src/features/rag/index.ts @@ -5,4 +5,9 @@ export { KnowledgeBaseComposerButton } from "./components/knowledge-base-compose export { KnowledgeBaseDialog } from "./components/knowledge-base-dialog"; export { RetrievalSettingsSection } from "./components/retrieval-settings-section"; export { ThreadDocumentsBar } from "./components/thread-documents-bar"; -export type { KnowledgeBase, RagDocument } from "./types/rag"; +export { + deleteDocument, + getDocumentFileUrl, + listAllDocuments, +} from "./api/rag-api"; +export type { KnowledgeBase, RagDocument, UploadedDocument } from "./types/rag"; diff --git a/studio/frontend/src/features/rag/types/rag.ts b/studio/frontend/src/features/rag/types/rag.ts index 1277500ae6..9922c740af 100644 --- a/studio/frontend/src/features/rag/types/rag.ts +++ b/studio/frontend/src/features/rag/types/rag.ts @@ -24,6 +24,13 @@ export interface RagDocument { createdAt?: string | null; } +/** RagDocument enriched for the global uploaded-files list (settings Data tab). */ +export interface UploadedDocument extends RagDocument { + sizeBytes?: number | null; + kbName?: string | null; + projectName?: string | null; +} + export interface DocumentUploadResult { documentId: string; jobId: string; diff --git a/studio/frontend/src/features/settings/components/archived-chats-dialog.tsx b/studio/frontend/src/features/settings/components/archived-chats-dialog.tsx index 836b4d25d4..c4932ccc5b 100644 --- a/studio/frontend/src/features/settings/components/archived-chats-dialog.tsx +++ b/studio/frontend/src/features/settings/components/archived-chats-dialog.tsx @@ -12,18 +12,12 @@ import { AlertDialogTitle, } from "@/components/ui/alert-dialog"; import { - Dialog, - DialogContent, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; -import { + type SidebarItem, deleteChatItem, unarchiveChatItem, useChatPreferencesStore, useChatRuntimeStore, useChatSidebarItems, - type SidebarItem, } from "@/features/chat"; import { toast } from "@/lib/toast"; import { ArchiveRestoreIcon, Delete02Icon } from "@hugeicons/core-free-icons"; @@ -40,13 +34,7 @@ function formatCreatedAt(ms: number): string { }); } -export function ArchivedChatsDialog({ - open, - onOpenChange, -}: { - open: boolean; - onOpenChange: (open: boolean) => void; -}) { +export function ArchivedChatsView() { const { archivedItems } = useChatSidebarItems({ requireMessages: false }); const navigate = useNavigate(); const closeSettings = useSettingsDialogStore((s) => s.closeDialog); @@ -74,7 +62,6 @@ export function ArchivedChatsDialog({ search: item.type === "single" ? { thread: item.id } : { compare: item.id }, }); - onOpenChange(false); closeSettings(); } @@ -114,72 +101,66 @@ export function ArchivedChatsDialog({ } return ( - - - - Archived chats - - - {archivedItems.length === 0 ? ( -

- No archived chats. -

- ) : ( -
-
- Name - Date created - -
- {archivedItems.map((item) => ( -
+ {archivedItems.length === 0 ? ( +

+ No archived chats. +

+ ) : ( +
+
+ Name + Date created + +
+ {archivedItems.map((item) => ( +
+ + + {formatCreatedAt(item.createdAt)} + + - - {formatCreatedAt(item.createdAt)} - - - - - -
- ))} -
- )} - + + +
+ ))} +
+ )} - + ); } diff --git a/studio/frontend/src/features/settings/components/finetune-recipe.ts b/studio/frontend/src/features/settings/components/finetune-recipe.ts new file mode 100644 index 0000000000..c98f422a35 --- /dev/null +++ b/studio/frontend/src/features/settings/components/finetune-recipe.ts @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Settings Data tab glue: turn chat history into a fine-tuning JSONL, stage +// it as a Data Recipe seed upload, and open a new recipe on that file. + +import { type FineTuneFormat, buildFineTuneJsonl } from "@/features/chat"; +import { saveRecipe } from "@/features/data-recipes/data/recipes-db"; +import { createEmptyRecipePayload } from "@/features/recipe-studio"; +import { inspectSeedUpload } from "@/features/recipe-studio/api"; +import { uploadTrainingDataset } from "@/features/training/api/datasets-api"; +import { useTrainingConfigStore } from "@/features/training/stores/training-config-store"; +import { toast } from "@/lib/toast"; + +/** btoa cannot handle code points above latin-1, so encode UTF-8 bytes. */ +function base64FromString(value: string): string { + const bytes = new TextEncoder().encode(value); + let binary = ""; + const CHUNK = 0x8000; + for (let i = 0; i < bytes.length; i += CHUNK) { + binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)); + } + return btoa(binary); +} + +/** Builds the JSONL, uploads it as a local recipe seed, and saves a new + * recipe whose seed block points at the file. Returns the recipe id, or + * null when there is nothing to export. */ +export async function createFineTuneRecipeFromChats( + format: FineTuneFormat = "openai", +): Promise { + const { lines, conversations } = await buildFineTuneJsonl(format); + if (conversations === 0) { + toast.info("No chats with a user and assistant exchange to export."); + return null; + } + + const dateLabel = new Date().toISOString().slice(0, 10); + const suffix = format === "openai" ? "" : `-${format}`; + const filename = `chat-finetune${suffix}-${dateLabel}.jsonl`; + const inspected = await inspectSeedUpload({ + filename, + // biome-ignore lint/style/useNamingConvention: api schema + content_base64: base64FromString(lines.join("\n")), + // biome-ignore lint/style/useNamingConvention: api schema + preview_size: 10, + }); + + const payload = createEmptyRecipePayload(); + payload.recipe.seed_config = { + source: { + // biome-ignore lint/style/useNamingConvention: api schema + seed_type: "local", + path: inspected.resolved_path, + }, + // biome-ignore lint/style/useNamingConvention: api schema + sampling_strategy: "ordered", + // biome-ignore lint/style/useNamingConvention: api schema + selection_strategy: null, + }; + payload.ui.nodes = [{ id: "seed", x: 0, y: 0, width: 400 }]; + payload.ui.seed_source_type = "local"; + payload.ui.seed_columns = inspected.columns; + payload.ui.seed_preview_rows = inspected.preview_rows ?? []; + payload.ui.local_file_name = filename; + + const record = await saveRecipe({ + name: `Chat fine-tuning ${dateLabel}`, + payload, + }); + return record.id; +} + +/** Builds the JSONL, uploads it as a training dataset, and selects it in the + * Train tab's config store so the Train page opens with it loaded. Returns + * false when there is nothing to export. */ +export async function loadFineTuneDatasetInTrainTab( + format: FineTuneFormat = "openai", +): Promise { + const { lines, conversations } = await buildFineTuneJsonl(format); + if (conversations === 0) { + toast.info("No chats with a user and assistant exchange to export."); + return false; + } + + const dateLabel = new Date().toISOString().slice(0, 10); + const suffix = format === "openai" ? "" : `-${format}`; + const file = new File( + [lines.join("\n")], + `chat-finetune${suffix}-${dateLabel}.jsonl`, + { type: "application/x-ndjson" }, + ); + const uploaded = await uploadTrainingDataset(file); + // Selecting also kicks off the dataset format check, so the Train tab + // shows the detected format as soon as it mounts. + useTrainingConfigStore.getState().selectLocalDataset(uploaded.stored_path); + return true; +} diff --git a/studio/frontend/src/features/settings/components/uploaded-files-dialog.tsx b/studio/frontend/src/features/settings/components/uploaded-files-dialog.tsx new file mode 100644 index 0000000000..3368f07ff2 --- /dev/null +++ b/studio/frontend/src/features/settings/components/uploaded-files-dialog.tsx @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Spinner } from "@/components/ui/spinner"; +import { + type ChatAttachmentRecord, + deleteChatAttachment, + emitChatAttachmentDeleted, + fetchChatAttachmentBlob, + listChatAttachments, +} from "@/features/chat"; +import { + deleteDocument, + getDocumentFileUrl, + listAllDocuments, + type UploadedDocument, +} from "@/features/rag"; +import { toast } from "@/lib/toast"; +import { + ArrowUpRight01Icon, + Delete02Icon, + File02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate } from "@tanstack/react-router"; +import { type ReactNode, useEffect, useRef, useState } from "react"; +import { useSettingsDialogStore } from "../stores/settings-dialog-store"; + +function formatUploadedAt(value: string | number | null | undefined): string { + if (value === null || value === undefined || value === "") return "-"; + // Chat attachments carry ms epoch numbers; RAG documents carry SQLite + // ISO-ish strings (no timezone). Unparseable strings fall through raw. + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) return String(value); + return parsed.toLocaleDateString(undefined, { + year: "numeric", + month: "long", + day: "numeric", + }); +} + +function formatSize(bytes: number | null | undefined): string { + if (bytes === null || bytes === undefined) return "-"; + if (bytes < 1024) return `${bytes} B`; + const units = ["KB", "MB", "GB"]; + let value = bytes; + let unit = "B"; + for (const next of units) { + if (value < 1024) break; + value /= 1024; + unit = next; + } + return `${value >= 10 ? Math.round(value) : value.toFixed(1)} ${unit}`; +} + +function ragLocationLabel(doc: UploadedDocument): string { + if (doc.kbId) return doc.kbName ? `KB · ${doc.kbName}` : "Knowledge base"; + if (doc.projectId) { + return doc.projectName ? `Project · ${doc.projectName}` : "Project"; + } + if (doc.threadId) return "Chat files (RAG)"; + return "-"; +} + +/** Short uppercase file-type label from the filename extension, falling back + * to the content-type subtype (e.g. "image/webp" gives WEBP). */ +function fileTypeLabel( + name: string, + contentType?: string | null, +): string | null { + const dot = name.lastIndexOf("."); + const ext = dot > 0 ? name.slice(dot + 1).trim() : ""; + if (ext && ext.length <= 5) return ext.toUpperCase(); + const subtype = contentType?.split("/")[1]?.split("+")[0]?.trim(); + return subtype && subtype.length <= 10 ? subtype.toUpperCase() : null; +} + +/** Lazy image thumbnail for a chat attachment; a file icon until it loads. + * The stored blob only downloads once the row scrolls into view, so a long + * history of screenshots does not fetch every image on open. */ +function ChatImageThumb({ + messageId, + attachmentId, +}: { + messageId: string; + attachmentId: string; +}) { + const [src, setSrc] = useState(null); + const [visible, setVisible] = useState(false); + const holderRef = useRef(null); + + useEffect(() => { + const el = holderRef.current; + if (!el) return; + if (typeof IntersectionObserver === "undefined") { + return; + } + const observer = new IntersectionObserver((entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + setVisible(true); + observer.disconnect(); + } + }); + observer.observe(el); + return () => observer.disconnect(); + }, []); + + useEffect(() => { + if (!visible) return; + let cancelled = false; + let url: string | null = null; + fetchChatAttachmentBlob(messageId, attachmentId) + .then((blob) => { + if (cancelled) return; + url = URL.createObjectURL(blob); + setSrc(url); + }) + .catch(() => { + // Keep the file icon on failure. + }); + return () => { + cancelled = true; + if (url) URL.revokeObjectURL(url); + }; + }, [visible, messageId, attachmentId]); + + if (!src) { + return ( + + + + ); + } + return ; +} + +function FileIconThumb() { + return ( + + ); +} + +/** One display row: a RAG document or a chat message attachment. */ +interface UploadedFileRow { + key: string; + source: "rag" | "chat"; + name: string; + location: string; + sizeBytes?: number | null; + createdAt?: string | number | null; + failed?: boolean; + /** Epoch ms for sorting; rows with unknown dates sort last. */ + sortTime: number; + typeLabel: string | null; + /** Image rows render a thumbnail; others show a file icon. */ + thumb: ReactNode; + /** Chat rows link back to their thread. */ + threadId?: string | null; + /** Compare-chat rows navigate by pair id instead of opening one pane alone. */ + pairId?: string | null; + open: () => Promise; + remove: () => Promise; + deleteDescription: string; +} + +function toSortTime(value: string | number | null | undefined): number { + if (value === null || value === undefined || value === "") return 0; + const parsed = new Date(value).getTime(); + return Number.isNaN(parsed) ? 0 : parsed; +} + +// Safari and Firefox block window.open after an await (the user gesture is +// gone), so open a blank tab synchronously and point it at the URL once +// resolved. A blocked synchronous open is surfaced instead of silently losing +// the file after the asynchronous URL lookup. +async function openResolvedUrl(resolve: () => Promise): Promise { + const win = window.open("", "_blank"); + if (!win) { + throw new Error( + "Your browser blocked the new tab. Allow popups and retry.", + ); + } + win.opener = null; + let url: string; + try { + url = await resolve(); + } catch (err) { + win.close(); + throw err; + } + win.location.replace(url); +} + +function ragRow(doc: UploadedDocument): UploadedFileRow { + return { + key: `rag-${doc.id}`, + source: "rag", + name: doc.filename, + location: ragLocationLabel(doc), + sizeBytes: doc.sizeBytes, + createdAt: doc.createdAt, + failed: doc.status === "failed", + sortTime: toSortTime(doc.createdAt), + typeLabel: fileTypeLabel(doc.filename), + // RAG uploads are documents (pdf, txt, md, docx, html), not images. + thumb: , + open: () => openResolvedUrl(() => getDocumentFileUrl(doc.id)), + remove: async () => { + await deleteDocument(doc.id, doc.projectId); + }, + deleteDescription: + "The file and its indexed content are removed. This cannot be undone.", + }; +} + +function chatAttachmentRow(att: ChatAttachmentRecord): UploadedFileRow { + const isImage = + att.type === "image" || Boolean(att.contentType?.startsWith("image/")); + return { + key: `chat-${att.messageId}-${att.id}`, + source: "chat", + name: att.name, + location: att.threadTitle ? `Chat · ${att.threadTitle}` : "Chat", + sizeBytes: att.sizeBytes, + createdAt: att.createdAt, + sortTime: toSortTime(att.createdAt), + typeLabel: fileTypeLabel(att.name, att.contentType), + threadId: att.threadId, + pairId: att.pairId, + thumb: isImage ? ( + + ) : ( + + ), + open: () => + openResolvedUrl(async () => { + const blob = await fetchChatAttachmentBlob(att.messageId, att.id); + const url = URL.createObjectURL(blob); + // Give the new tab time to load the blob before revoking. + setTimeout(() => URL.revokeObjectURL(url), 60_000); + return url; + }), + remove: async () => { + await deleteChatAttachment(att.messageId, att.id); + // Patch any loaded runtime copy so a later repo sync cannot write the + // deleted attachment back to storage. + emitChatAttachmentDeleted({ + messageId: att.messageId, + attachmentId: att.id, + }); + }, + deleteDescription: + "The attachment is removed from its chat message; the message text is kept. This cannot be undone.", + }; +} + +type SourceLoad = { + status: "loading" | "ready" | "error"; + data: T; + error: string | null; +}; + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error ? error.message : fallback; +} + +/** Inline settings page listing uploaded files from each available source. */ +export function UploadedFilesView() { + const [ragFiles, setRagFiles] = useState>({ + status: "loading", + data: [], + error: null, + }); + const [chatFiles, setChatFiles] = useState< + SourceLoad + >({ status: "loading", data: [], error: null }); + const [chatNextOffset, setChatNextOffset] = useState(null); + const [loadingMore, setLoadingMore] = useState(false); + const [confirmingDelete, setConfirmingDelete] = + useState(null); + const navigate = useNavigate(); + + const rows = [ + ...ragFiles.data.map(ragRow), + ...chatFiles.data.map(chatAttachmentRow), + ].sort((a, b) => b.sortTime - a.sortTime); + + // Jump to the chat thread the attachment lives in, closing the settings + // dialog so the thread is actually visible. + function goToChat(row: UploadedFileRow) { + if (!row.threadId) return; + useSettingsDialogStore.getState().closeDialog(); + if (row.pairId) { + void navigate({ to: "/chat", search: { compare: row.pairId } }); + } else { + void navigate({ to: "/chat", search: { thread: row.threadId } }); + } + } + + useEffect(() => { + let cancelled = false; + void listAllDocuments().then( + (data) => { + if (!cancelled) setRagFiles({ status: "ready", data, error: null }); + }, + (error: unknown) => { + if (!cancelled) { + setRagFiles({ + status: "error", + data: [], + error: errorMessage(error, "Failed to load RAG documents"), + }); + } + }, + ); + void listChatAttachments().then( + (page) => { + if (!cancelled) { + setChatFiles({ + status: "ready", + data: page.attachments, + error: null, + }); + setChatNextOffset(page.nextOffset); + } + }, + (error: unknown) => { + if (!cancelled) { + setChatFiles({ + status: "error", + data: [], + error: errorMessage(error, "Failed to load chat attachments"), + }); + } + }, + ); + return () => { + cancelled = true; + }; + }, []); + + function retryRagFiles() { + setRagFiles((current) => ({ ...current, status: "loading", error: null })); + void listAllDocuments().then( + (data) => setRagFiles({ status: "ready", data, error: null }), + (error: unknown) => + setRagFiles((current) => ({ + ...current, + status: "error", + error: errorMessage(error, "Failed to load RAG documents"), + })), + ); + } + + async function loadChatPage(offset: number, append: boolean) { + setLoadingMore(true); + setChatFiles((current) => ({ ...current, status: "loading", error: null })); + try { + const page = await listChatAttachments(offset); + setChatFiles((current) => ({ + status: "ready", + data: append + ? [ + ...current.data, + ...page.attachments.filter( + (incoming) => + !current.data.some( + (existing) => + existing.id === incoming.id && + existing.messageId === incoming.messageId, + ), + ), + ] + : page.attachments, + error: null, + })); + setChatNextOffset(page.nextOffset); + } catch (error) { + setChatFiles((current) => ({ + ...current, + status: "error", + error: errorMessage(error, "Failed to load chat attachments"), + })); + } finally { + setLoadingMore(false); + } + } + + function retryChatFiles() { + const append = chatFiles.data.length > 0 && chatNextOffset !== null; + void loadChatPage(append ? chatNextOffset : 0, append); + } + + async function handleOpen(row: UploadedFileRow) { + try { + await row.open(); + } catch (err) { + toast.error("Failed to open file", { + description: err instanceof Error ? err.message : undefined, + }); + } + } + + async function handleDelete(row: UploadedFileRow) { + // Offset pages and destructive mutations must not race: a deletion shifts + // the boundary used by an in-flight page request. + if (loadingMore) return; + try { + await row.remove(); + if (row.source === "rag") { + setRagFiles((current) => ({ + ...current, + data: current.data.filter((doc) => `rag-${doc.id}` !== row.key), + })); + } else { + setChatFiles((current) => ({ + ...current, + data: current.data.filter( + (attachment) => + `chat-${attachment.messageId}-${attachment.id}` !== row.key, + ), + })); + // Offset pagination is relative to the current server inventory. A + // deletion before the next page shifts every later row back by one. + setChatNextOffset((current) => + current === null ? null : Math.max(0, current - 1), + ); + } + toast.success("File deleted"); + } catch (err) { + toast.error("Failed to delete file", { + description: err instanceof Error ? err.message : undefined, + }); + } + } + + return ( +
+ {ragFiles.status === "error" ? ( +
+ RAG documents unavailable: {ragFiles.error} + +
+ ) : null} + {chatFiles.status === "error" ? ( +
+ Chat attachments unavailable: {chatFiles.error} + +
+ ) : null} + + {rows.length === 0 && + (ragFiles.status === "loading" || chatFiles.status === "loading") ? ( +
+ +
+ ) : rows.length === 0 && + ragFiles.status !== "error" && + chatFiles.status !== "error" ? ( +

+ No uploaded files. +

+ ) : rows.length > 0 ? ( +
+
+ Name + Location + Uploaded + +
+ {rows.map((row) => ( +
+ {/* Clicking the file jumps to its chat; files without one + open directly. The theme scales rounded-md up to a near + circle at this size, so the thumb pins a small radius. */} + + {row.threadId ? ( + + ) : ( + + {row.location} + + )} + + {formatUploadedAt(row.createdAt)} + + + + + +
+ ))} + {chatNextOffset !== null ? ( +
+ +
+ ) : null} +
+ ) : null} + + { + if (!o) setConfirmingDelete(null); + }} + > + + + Delete file + + Delete{" "} + + "{confirmingDelete?.name}" + + ? {confirmingDelete?.deleteDescription} + + + + Cancel + { + const row = confirmingDelete; + setConfirmingDelete(null); + if (row) void handleDelete(row); + }} + > + Delete + + + + +
+ ); +} diff --git a/studio/frontend/src/features/settings/settings-dialog.tsx b/studio/frontend/src/features/settings/settings-dialog.tsx index 08588be8fe..d98d1b8ac0 100644 --- a/studio/frontend/src/features/settings/settings-dialog.tsx +++ b/studio/frontend/src/features/settings/settings-dialog.tsx @@ -15,6 +15,7 @@ import { Cancel01Icon, CloudIcon, CpuIcon, + DatabaseSettingIcon, Globe02Icon, HelpCircleIcon, Message01Icon, @@ -43,6 +44,7 @@ import { ApiKeysTab } from "./tabs/api-keys-tab"; import { AppearanceTab } from "./tabs/appearance-tab"; import { ChatTab } from "./tabs/chat-tab"; import { ConnectionsTab } from "./tabs/connections-tab"; +import { DataTab } from "./tabs/data-tab"; import { GeneralTab } from "./tabs/general-tab"; import { ProfileTab } from "./tabs/profile-tab"; import { ResourcesTab } from "./tabs/resources-tab"; @@ -93,6 +95,12 @@ const TABS: TabDef[] = [ iconComponent: MicIcon, badgeKey: "common.new", }, + { + id: "data", + labelKey: "settings.tabs.data", + icon: DatabaseSettingIcon, + badgeKey: "common.new", + }, { id: "about", labelKey: "settings.tabs.about", icon: HelpCircleIcon }, ]; @@ -112,6 +120,8 @@ function renderTab(tab: SettingsTab) { return ; case "connections": return ; + case "data": + return ; case "api-keys": return ; case "about": @@ -210,6 +220,7 @@ export function SettingsDialog() { chat: null, voice: null, connections: null, + data: null, "api-keys": null, about: null, }); diff --git a/studio/frontend/src/features/settings/settings-search.ts b/studio/frontend/src/features/settings/settings-search.ts index 786ee11d0a..c19b27c732 100644 --- a/studio/frontend/src/features/settings/settings-search.ts +++ b/studio/frontend/src/features/settings/settings-search.ts @@ -85,12 +85,19 @@ export const SETTINGS_SEARCH_INDEX: Record = { "settings.chat.artifacts.title", "settings.chat.artifacts.collapseHtmlBlocks", "settings.chat.artifacts.allowNetworkAccess", - "settings.chat.data", + "settings.chat.modelDisclaimer", + ], + // Chat data management moved to the Data tab; keep these rows findable there. + data: [ + "settings.data.fineTuneExport", + "settings.data.archivedChats", + "settings.data.archiveAllChats", + "settings.data.confirmBeforeDeleting", + "settings.data.uploadedFiles", + "settings.chat.exportHistory", "settings.chat.exportConversations", "settings.chat.importChats", "settings.chat.clearAllChats", - "settings.chat.exportHistory", - "settings.chat.modelDisclaimer", ], "api-keys": [ "settings.apiKeys.title", diff --git a/studio/frontend/src/features/settings/stores/settings-dialog-store.ts b/studio/frontend/src/features/settings/stores/settings-dialog-store.ts index b9e9c75b14..51908a5ad0 100644 --- a/studio/frontend/src/features/settings/stores/settings-dialog-store.ts +++ b/studio/frontend/src/features/settings/stores/settings-dialog-store.ts @@ -11,6 +11,7 @@ export type SettingsTab = | "chat" | "voice" | "connections" + | "data" | "api-keys" | "about"; @@ -30,7 +31,7 @@ interface SettingsDialogState { // explicitly via onCloseAutoFocus. opener: HTMLElement | null; // Set when something asks to jump straight to the archived chats list (the - // archive toast). ChatTab consumes it to open the dialog, then clears it. + // archive toast). DataTab uses it as its initial subpage, then clears it. archivedChatsRequested: boolean; openDialog: (tab?: SettingsTab, options?: OpenDialogOptions) => void; openArchivedChats: () => void; @@ -66,6 +67,7 @@ function loadInitialTab(): SettingsTab { "chat", "voice", "connections", + "data", "api-keys", "about", ]; @@ -90,7 +92,7 @@ export const useSettingsDialogStore = create((set) => ({ openArchivedChats: () => set({ open: true, - activeTab: "chat", + activeTab: "data", scrollTarget: null, archivedChatsRequested: true, opener: captureOpener(), diff --git a/studio/frontend/src/features/settings/tabs/chat-tab.tsx b/studio/frontend/src/features/settings/tabs/chat-tab.tsx index d5c9f10a4f..3e419af78d 100644 --- a/studio/frontend/src/features/settings/tabs/chat-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/chat-tab.tsx @@ -1,43 +1,16 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { Button } from "@/components/ui/button"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuSeparator, - DropdownMenuSub, - DropdownMenuSubContent, - DropdownMenuSubTrigger, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; import { Switch } from "@/components/ui/switch"; import { - EXPORT_FORMATS_LIST, type PlusMenuItemId, - bulkExportConversationsByScope, - clearAllChats, - countAllChats, - downloadChatExport, - importConversationsFromFile, useChatPreferencesStore, useChatRuntimeStore, usePlusMenuPrefsStore, } from "@/features/chat"; import { useT } from "@/i18n"; -import { toast } from "@/lib/toast"; import { Bookmark02Icon, - Delete02Icon, Download01Icon, FileDatabaseIcon, Folder01Icon, @@ -45,19 +18,16 @@ import { PencilRulerIcon, Settings02Icon, ShieldBanIcon, - Upload01Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Columns2Icon, PlusIcon } from "lucide-react"; -import { useEffect, useRef, useState } from "react"; +import { useEffect } from "react"; import type { ReactNode } from "react"; -import { ArchivedChatsDialog } from "../components/archived-chats-dialog"; import { SettingsRow } from "../components/settings-row"; import { SettingsGroupDivider, SettingsSection, } from "../components/settings-section"; -import { useSettingsDialogStore } from "../stores/settings-dialog-store"; // Adjustable "+" menu items shown in settings, in display order. Icons mirror // the ones used in the composer + menu itself. @@ -155,24 +125,6 @@ export function ChatTab() { const t = useT(); const plusPins = usePlusMenuPrefsStore((state) => state.pins); const togglePlusPin = usePlusMenuPrefsStore((state) => state.togglePin); - const [confirmOpen, setConfirmOpen] = useState(false); - const [archivedOpen, setArchivedOpen] = useState(false); - const [count, setCount] = useState(null); - const archivedChatsRequested = useSettingsDialogStore( - (s) => s.archivedChatsRequested, - ); - const consumeArchivedChatsRequest = useSettingsDialogStore( - (s) => s.consumeArchivedChatsRequest, - ); - - // Open the archived list when the archive toast asked to jump here. - useEffect(() => { - if (!archivedChatsRequested) return; - setArchivedOpen(true); - consumeArchivedChatsRequest(); - }, [archivedChatsRequested, consumeArchivedChatsRequest]); - const [exporting, setExporting] = useState(false); - const [clearing, setClearing] = useState(false); const autoTitle = useChatRuntimeStore((state) => state.autoTitle); const setAutoTitle = useChatRuntimeStore((state) => state.setAutoTitle); const showCanvasMenuItem = useChatRuntimeStore( @@ -212,12 +164,6 @@ export function ChatTab() { const setShowAllQuantizations = useChatRuntimeStore( (state) => state.setShowAllQuantizations, ); - const confirmDeleteChats = useChatPreferencesStore( - (state) => state.confirmDeleteChats, - ); - const setConfirmDeleteChats = useChatPreferencesStore( - (state) => state.setConfirmDeleteChats, - ); const showModelDisclaimer = useChatPreferencesStore( (state) => state.showModelDisclaimer, ); @@ -232,95 +178,9 @@ export function ChatTab() { ); useEffect(() => { - void countAllChats().then(setCount); void hydratePersistedSettings(); }, [hydratePersistedSettings]); - const handleExport = async () => { - setExporting(true); - try { - await downloadChatExport(); - } finally { - setExporting(false); - } - }; - - const importInputRef = useRef(null); - const handleImport = async (file: File) => { - try { - const imported = await importConversationsFromFile(file, null); - if (imported === 0) { - toast.info(t("settings.chat.importNoConversations")); - } else { - toast.success( - imported === 1 - ? t("settings.chat.importedOneChat") - : t("settings.chat.importedChatCount", { count: imported }), - ); - setCount(await countAllChats().catch(() => count)); - } - } catch { - toast.error(t("settings.chat.importFailed")); - } - }; - - const handleClear = async () => { - setClearing(true); - try { - const result = await clearAllChats(); - const clearedCount = result.deletedThreadIds.length; - const hasFailedStore = - result.backend === "failed" || result.legacy === "failed"; - if (!hasFailedStore && result.failedThreadIds.length === 0) { - setCount(0); - setConfirmOpen(false); - toast.success( - clearedCount === 0 - ? t("settings.chat.clearedAllChats") - : clearedCount === 1 - ? t("settings.chat.clearedOneChat") - : t("settings.chat.clearedChatCount", { count: clearedCount }), - ); - return; - } - - const fallbackRemaining = - result.failedThreadIds.length > 0 - ? result.failedThreadIds.length - : (count ?? 0); - const remaining = await countAllChats().catch(() => fallbackRemaining); - setCount(remaining); - setConfirmOpen(false); - toast.warning(t("settings.chat.someChatsCouldNotBeCleared"), { - description: - result.failedThreadIds.length > 0 - ? clearedCount === 1 && result.failedThreadIds.length === 1 - ? t("settings.chat.oneChatClearedRemainOne") - : clearedCount === 1 - ? t("settings.chat.oneChatClearedRemain", { - remainingCount: result.failedThreadIds.length, - }) - : result.failedThreadIds.length === 1 - ? t("settings.chat.chatsClearedRemainOne", { clearedCount }) - : t("settings.chat.chatsClearedRemain", { - clearedCount, - remainingCount: result.failedThreadIds.length, - }) - : remaining === 1 - ? t("settings.chat.storageClearFailedOne") - : t("settings.chat.storageClearFailed", { count: remaining }), - }); - } catch (error) { - const remaining = await countAllChats().catch(() => count); - setCount(remaining); - toast.error(t("settings.chat.failedToClearChats"), { - description: error instanceof Error ? error.message : undefined, - }); - } finally { - setClearing(false); - } - }; - return (
@@ -347,7 +207,7 @@ export function ChatTab() { Q4_K_M - + downloaded 16 GB @@ -481,191 +341,6 @@ export function ChatTab() { /> - - - - - - - - - - - - - - - - - - - - - {( - [ - { scope: "recents", label: "exportScopeRecents" }, - { scope: "all", label: "exportScopeAll" }, - ] as const - ).map(({ scope, label }) => ( - - - - {t(`settings.chat.${label}`)} - - - {EXPORT_FORMATS_LIST.map(({ fmt, label: fmtLabel }) => ( - - void bulkExportConversationsByScope(scope, fmt, true) - } - > - {fmtLabel} {t("settings.chat.exportCombinedSuffix")} - - ))} - - {EXPORT_FORMATS_LIST.map(({ fmt, label: fmtLabel }) => ( - - void bulkExportConversationsByScope(scope, fmt, false) - } - > - {fmtLabel} {t("settings.chat.exportPerChatSuffix")} - - ))} - - - ))} - - - - - - - { - const file = e.target.files?.[0]; - e.target.value = ""; - if (file) void handleImport(file); - }} - /> - - - - - - - - - - - - - - {count === 1 - ? t("settings.chat.clearOneChatTitle") - : t("settings.chat.clearChatsTitle", { count: count ?? 0 })} - - - {t("settings.chat.clearChatsConfirmDescription")} - - - - - - - -
); } diff --git a/studio/frontend/src/features/settings/tabs/data-tab.tsx b/studio/frontend/src/features/settings/tabs/data-tab.tsx new file mode 100644 index 0000000000..dc9e707ea0 --- /dev/null +++ b/studio/frontend/src/features/settings/tabs/data-tab.tsx @@ -0,0 +1,727 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuSub, + DropdownMenuSubContent, + DropdownMenuSubTrigger, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; +import { usePlatformStore } from "@/config/env"; +import { + EXPORT_FORMATS_LIST, + type FineTuneFormat, + archiveAllChatItems, + bulkExportConversationsByScope, + clearAllChats, + countAllChats, + downloadArchivedChatExport, + downloadChatExport, + exportFineTuneJsonl, + importConversationsFromFile, + useChatPreferencesStore, + useChatRuntimeStore, + useChatSidebarItems, +} from "@/features/chat"; +import { useT } from "@/i18n"; +import { + ChevronDownStandardIcon, + ChevronRightStandardIcon, +} from "@/lib/chevron-icons"; +import { toast } from "@/lib/toast"; +import { + Archive02Icon, + ArrowLeft01Icon, + Delete02Icon, + Download01Icon, + Tick02Icon, + Upload01Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate, useRouterState } from "@tanstack/react-router"; +import { useEffect, useRef, useState } from "react"; +import { ArchivedChatsView } from "../components/archived-chats-dialog"; +import { + createFineTuneRecipeFromChats, + loadFineTuneDatasetInTrainTab, +} from "../components/finetune-recipe"; +import { SettingsRow } from "../components/settings-row"; +import { SettingsSection } from "../components/settings-section"; +import { UploadedFilesView } from "../components/uploaded-files-dialog"; +import { useSettingsDialogStore } from "../stores/settings-dialog-store"; + +export function DataTab() { + const t = useT(); + const navigate = useNavigate(); + const archivedChatsRequested = useSettingsDialogStore( + (s) => s.archivedChatsRequested, + ); + const consumeArchivedChatsRequest = useSettingsDialogStore( + (s) => s.consumeArchivedChatsRequest, + ); + const [confirmOpen, setConfirmOpen] = useState(false); + const [archiveConfirmOpen, setArchiveConfirmOpen] = useState(false); + // Subpages swap the Data tab body instead of opening nested dialogs. + const [subpage, setSubpage] = useState<"main" | "archived" | "files">( + archivedChatsRequested ? "archived" : "main", + ); + const [count, setCount] = useState(null); + const [exporting, setExporting] = useState(false); + const [archivedExporting, setArchivedExporting] = useState(false); + // Gates the archived subpage Export button. + const { archivedItems } = useChatSidebarItems({ requireMessages: false }); + const [clearing, setClearing] = useState(false); + const [archiving, setArchiving] = useState(false); + const [fineTuneExporting, setFineTuneExporting] = useState(false); + const [openingRecipe, setOpeningRecipe] = useState(false); + const [loadingTraining, setLoadingTraining] = useState(false); + // Chat-only hosts redirect /studio back to /chat, so loading a dataset in + // the Train tab would upload it and then strand the user; gate the action + // the same way the sidebar gates Train. + const chatOnly = usePlatformStore((s) => s.isChatOnly()); + const [fineTuneAction, setFineTuneAction] = useState< + "train" | "recipes" | "export" + >(chatOnly ? "export" : "train"); + // Chat Completions (OpenAI messages) is the only export format we ship. + const fineTuneFormat: FineTuneFormat = "openai"; + + // The MLX self-heal can flip chat-only while the dialog is open. + useEffect(() => { + if (chatOnly) { + setFineTuneAction((a) => (a === "train" ? "export" : a)); + } + }, [chatOnly]); + // Requests can arrive after Data is already mounted (for example from the + // archive-all toast), so always switch before consuming the flag. + useEffect(() => { + if (!archivedChatsRequested) return; + let cancelled = false; + queueMicrotask(() => { + if (cancelled) return; + setSubpage("archived"); + consumeArchivedChatsRequest(); + }); + return () => { + cancelled = true; + }; + }, [archivedChatsRequested, consumeArchivedChatsRequest]); + + const confirmDeleteChats = useChatPreferencesStore( + (state) => state.confirmDeleteChats, + ); + const setConfirmDeleteChats = useChatPreferencesStore( + (state) => state.setConfirmDeleteChats, + ); + + const storeThreadId = useChatRuntimeStore((s) => s.activeThreadId); + // Open chat id from the route (single thread or compare pair), mirroring + // ArchivedChatsView: compare panes only live in the search params. + const openChatId = useRouterState({ + select: (s) => { + if (!s.location.pathname.startsWith("/chat")) return undefined; + const search = s.location.search as Record; + return search.thread ?? search.compare ?? storeThreadId ?? undefined; + }, + }); + + useEffect(() => { + void countAllChats().then(setCount); + }, []); + + const handleExport = async () => { + setExporting(true); + try { + await downloadChatExport(); + } finally { + setExporting(false); + } + }; + + const handleExportArchived = async () => { + setArchivedExporting(true); + try { + const exported = await downloadArchivedChatExport(); + toast.success( + exported === 0 + ? t("settings.data.noArchivedChatsToExport") + : exported === 1 + ? t("settings.data.exportedOneArchivedChat") + : t("settings.data.exportedArchivedChatCount", { count: exported }), + ); + } catch (error) { + toast.error(t("settings.data.failedToExportArchivedChats"), { + description: error instanceof Error ? error.message : undefined, + }); + } finally { + setArchivedExporting(false); + } + }; + + const importInputRef = useRef(null); + const handleImport = async (file: File) => { + try { + const imported = await importConversationsFromFile(file, null); + if (imported === 0) { + toast.info(t("settings.chat.importNoConversations")); + } else { + toast.success( + imported === 1 + ? t("settings.chat.importedOneChat") + : t("settings.chat.importedChatCount", { count: imported }), + ); + setCount(await countAllChats().catch(() => count)); + } + } catch { + toast.error(t("settings.chat.importFailed")); + } + }; + + const handleArchiveAll = async () => { + setArchiving(true); + try { + const archived = await archiveAllChatItems(openChatId, (view) => { + navigate({ to: "/chat", search: { new: view.newThreadNonce } }); + }); + setArchiveConfirmOpen(false); + toast.success( + archived === 0 + ? t("settings.data.noChatsToArchive") + : archived === 1 + ? t("settings.data.archivedOneChat") + : t("settings.data.archivedChatCount", { count: archived }), + ); + } catch (error) { + toast.error(t("settings.data.failedToArchiveChats"), { + description: error instanceof Error ? error.message : undefined, + }); + } finally { + setArchiving(false); + } + }; + + const handleFineTuneExport = async () => { + setFineTuneExporting(true); + try { + await exportFineTuneJsonl(fineTuneFormat); + } catch (error) { + toast.error(t("settings.data.fineTuneExportFailed"), { + description: error instanceof Error ? error.message : undefined, + }); + } finally { + setFineTuneExporting(false); + } + }; + + const handleOpenInRecipes = async () => { + setOpeningRecipe(true); + try { + const recipeId = await createFineTuneRecipeFromChats(fineTuneFormat); + if (!recipeId) return; + useSettingsDialogStore.getState().closeDialog(); + void navigate({ to: "/data-recipes/$recipeId", params: { recipeId } }); + } catch (error) { + toast.error(t("settings.data.fineTuneRecipeFailed"), { + description: error instanceof Error ? error.message : undefined, + }); + } finally { + setOpeningRecipe(false); + } + }; + + const handleUseInTraining = async () => { + setLoadingTraining(true); + try { + const loaded = await loadFineTuneDatasetInTrainTab(fineTuneFormat); + if (!loaded) return; + useSettingsDialogStore.getState().closeDialog(); + void navigate({ to: "/studio" }); + } catch (error) { + toast.error(t("settings.data.fineTuneTrainFailed"), { + description: error instanceof Error ? error.message : undefined, + }); + } finally { + setLoadingTraining(false); + } + }; + + const fineTuneActionLabels = { + train: t("settings.data.fineTuneTrainAction"), + recipes: t("settings.data.fineTuneOpenRecipesAction"), + export: t("settings.data.fineTuneExportAction"), + } as const; + const fineTuneBusy = loadingTraining || openingRecipe || fineTuneExporting; + const runFineTuneAction = () => { + if (fineTuneAction === "train") { + if (chatOnly) return; + void handleUseInTraining(); + } else if (fineTuneAction === "recipes") void handleOpenInRecipes(); + else void handleFineTuneExport(); + }; + + const handleClear = async () => { + setClearing(true); + try { + const result = await clearAllChats(); + const clearedCount = result.deletedThreadIds.length; + const hasFailedStore = + result.backend === "failed" || result.legacy === "failed"; + if (!hasFailedStore && result.failedThreadIds.length === 0) { + setCount(0); + setConfirmOpen(false); + toast.success( + clearedCount === 0 + ? t("settings.chat.clearedAllChats") + : clearedCount === 1 + ? t("settings.chat.clearedOneChat") + : t("settings.chat.clearedChatCount", { count: clearedCount }), + ); + return; + } + + const fallbackRemaining = + result.failedThreadIds.length > 0 + ? result.failedThreadIds.length + : (count ?? 0); + const remaining = await countAllChats().catch(() => fallbackRemaining); + setCount(remaining); + setConfirmOpen(false); + toast.warning(t("settings.chat.someChatsCouldNotBeCleared"), { + description: + result.failedThreadIds.length > 0 + ? clearedCount === 1 && result.failedThreadIds.length === 1 + ? t("settings.chat.oneChatClearedRemainOne") + : clearedCount === 1 + ? t("settings.chat.oneChatClearedRemain", { + remainingCount: result.failedThreadIds.length, + }) + : result.failedThreadIds.length === 1 + ? t("settings.chat.chatsClearedRemainOne", { clearedCount }) + : t("settings.chat.chatsClearedRemain", { + clearedCount, + remainingCount: result.failedThreadIds.length, + }) + : remaining === 1 + ? t("settings.chat.storageClearFailedOne") + : t("settings.chat.storageClearFailed", { count: remaining }), + }); + } catch (error) { + const remaining = await countAllChats().catch(() => count); + setCount(remaining); + toast.error(t("settings.chat.failedToClearChats"), { + description: error instanceof Error ? error.message : undefined, + }); + } finally { + setClearing(false); + } + }; + + if (subpage === "archived") { + return ( +
+
+ +

+ {t("settings.data.title")} +

+
+
+
+

+ {t("settings.data.archivedChats")} +

+

+ {t("settings.data.archivedChatsDescription")} +

+
+ {archivedItems.length > 0 && ( + + )} +
+ +
+ ); + } + + if (subpage === "files") { + return ( +
+
+ +

+ {t("settings.data.title")} +

+
+
+

+ {t("settings.data.uploadedFiles")} +

+

+ {t("settings.data.uploadedFilesDescription")} +

+
+ +
+ ); + } + + return ( +
+
+

+ {t("settings.data.title")} +

+

+ {t("settings.data.description")} +

+
+ +
+ +
+ + + {/* Fixed width so switching actions never resizes the row. */} + + + + {(["export", "train", "recipes"] as const).map((action) => ( + setFineTuneAction(action)} + > + + {fineTuneActionLabels[action]} + + {fineTuneAction === action ? ( + + ) : null} + + ))} + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + {( + [ + { scope: "recents", label: "exportScopeRecents" }, + { scope: "all", label: "exportScopeAll" }, + ] as const + ).map(({ scope, label }) => ( + + + + {t(`settings.chat.${label}`)} + + + {EXPORT_FORMATS_LIST.map(({ fmt, label: fmtLabel }) => ( + + void bulkExportConversationsByScope(scope, fmt, true) + } + > + {fmtLabel} {t("settings.chat.exportCombinedSuffix")} + + ))} + + {EXPORT_FORMATS_LIST.map(({ fmt, label: fmtLabel }) => ( + + void bulkExportConversationsByScope(scope, fmt, false) + } + > + {fmtLabel} {t("settings.chat.exportPerChatSuffix")} + + ))} + + + ))} + + + + + + + + + + + { + const file = e.target.files?.[0]; + e.target.value = ""; + if (file) void handleImport(file); + }} + /> + +
+ + + + + + + + + + + {t("settings.data.archiveAllChatsTitle")} + + {t("settings.data.archiveAllChatsConfirmDescription")} + + + + + + + + + + + + + + {count === 1 + ? t("settings.chat.clearOneChatTitle") + : t("settings.chat.clearChatsTitle", { count: count ?? 0 })} + + + {t("settings.chat.clearChatsConfirmDescription")} + + + + + + + + +
+ ); +} diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index cbddc9f0c2..5ee2805a33 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -99,6 +99,7 @@ export const en = { chat: "Chat", voice: "Voice", connections: "Connections", + data: "Data", apiKeys: "API", about: "About", }, @@ -511,7 +512,7 @@ export const en = { }, chat: { title: "Chat", - description: "Manage chat history stored on this device.", + description: "Customize how chat behaves on this device.", modelDisclaimer: "Show model disclaimer", modelDisclaimerDescription: 'Show "LLMs can make mistakes" under the chat box.', @@ -580,6 +581,53 @@ export const en = { "A storage clear failed; {count} chats may remain. Please retry.", failedToClearChats: "Failed to clear chats", }, + data: { + title: "Data", + description: + "Manage chat history and uploaded files stored on this device.", + archivedChats: "Archived chats", + archivedChatsDescription: "View and manage chats you have archived.", + manageAction: "Manage", + exportArchivedChats: "Export", + exportingArchivedChats: "Exporting...", + exportedOneArchivedChat: "Exported 1 archived chat", + exportedArchivedChatCount: "Exported {count} archived chats", + noArchivedChatsToExport: "No archived chats to export.", + failedToExportArchivedChats: "Failed to export archived chats", + archiveAllChats: "Archive all chats", + archiveAllChatsDescription: + "Move every chat in Recents and Projects to the archive.", + noChatsToArchive: "No chats to archive.", + archiveAllAction: "Archive all", + archivingAction: "Archiving...", + archiveAllChatsTitle: "Archive all chats?", + archiveAllChatsConfirmDescription: + "Moves every chat on this device to the archive. Archived chats stay available and can be unarchived at any time.", + archivedAllChats: "Archived all chats", + archivedOneChat: "Archived 1 chat", + archivedChatCount: "Archived {count} chats", + failedToArchiveChats: "Failed to archive chats", + confirmBeforeDeleting: "Confirm before deleting", + confirmBeforeDeletingDescription: + "Ask for confirmation before a chat is deleted. Turn off to delete instantly.", + filesSection: "Files", + uploadedFiles: "Uploaded files", + uploadedFilesDescription: + "View and manage files uploaded to chats, projects, and knowledge bases.", + fineTuneExport: "Use chats as training data", + fineTuneExportDescription: + "Create a fine-tuning JSONL dataset from your chats. Load it in Train, refine in Recipes, or export it.", + fineTuneExportAction: "Export JSONL", + fineTuneRunAction: "Run", + fineTuneExportingAction: "Exporting...", + fineTuneOpenRecipesAction: "Open in Recipes", + fineTuneOpeningRecipesAction: "Opening...", + fineTuneTrainAction: "Load in Train tab", + fineTuneTrainingAction: "Loading...", + fineTuneExportFailed: "Failed to export training data", + fineTuneRecipeFailed: "Failed to open chats in Recipes", + fineTuneTrainFailed: "Failed to load dataset in the Train tab", + }, connections: { title: "Connections", description: "Manage providers and external connections.",