From 6a1d3cfdc0dd1df5b5ac80a9998ecea4475a3fe0 Mon Sep 17 00:00:00 2001 From: danielhanchen <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:38:47 +0000 Subject: [PATCH 001/297] Do not embed the seeded bootstrap password in the served page --- studio/backend/main.py | 198 +----------------- .../tests/test_index_bootstrap_loopback.py | 123 ----------- .../tests/test_index_bootstrap_origin.py | 131 ------------ .../test_index_bootstrap_origin_extra.py | 189 ----------------- .../test_index_no_bootstrap_injection.py | 42 ++++ 5 files changed, 48 insertions(+), 635 deletions(-) delete mode 100644 studio/backend/tests/test_index_bootstrap_loopback.py delete mode 100644 studio/backend/tests/test_index_bootstrap_origin.py delete mode 100644 studio/backend/tests/test_index_bootstrap_origin_extra.py create mode 100644 studio/backend/tests/test_index_no_bootstrap_injection.py diff --git a/studio/backend/main.py b/studio/backend/main.py index 8062b7b073..07bf67945f 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -234,14 +234,12 @@ if _STUDIO_ROOT_RESOLVED != _LEGACY_STUDIO_ROOT: os.environ.setdefault("UNSLOTH_IS_PRESENT", "1") import hashlib -import ipaddress import mimetypes import re as _re import shutil import warnings from contextlib import asynccontextmanager from importlib.metadata import PackageNotFoundError, version as package_version -from urllib.parse import urlparse _STUDIO_INSTALL_ID_RE = _re.compile(r"^[0-9a-f]{64}$") @@ -1291,180 +1289,6 @@ def _strip_crossorigin(html_bytes: bytes) -> bytes: return html.encode("utf-8") -def _inject_bootstrap(html_bytes: bytes, app: FastAPI): - """Inject bootstrap credentials when password change is pending. - Returns ``(html_bytes, script_nonce_or_None)``; callers forward the nonce - via ``_CSP_SCRIPT_NONCE_HEADER`` so CSP allows the inline script. - """ - import json as _json - import secrets as _secrets - - if not storage.requires_password_change(storage.DEFAULT_ADMIN_USERNAME): - return html_bytes, None - - bootstrap_pw = getattr(app.state, "bootstrap_password", None) - if not bootstrap_pw: - return html_bytes, None - - payload = _json.dumps( - { - "username": storage.DEFAULT_ADMIN_USERNAME, - "password": bootstrap_pw, - } - ) - nonce = _secrets.token_urlsafe(16) - tag = f'' - html = html_bytes.decode("utf-8") - html = html.replace("", f"{tag}", 1) - return html.encode("utf-8"), nonce - - -_DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443} - - -def _canonical_origin(scheme: str, netloc: str) -> Optional[tuple[str, str, int]]: - """Canonicalise an Origin to ``(scheme, host, port)`` for equality. - Browsers strip default ports (RFC 6454 sec 6.1) and scheme/host are - case-insensitive (RFC 3986), so a bare string compare misclassifies - same-origin requests as cross-origin. Returns ``None`` on unparseable input - so callers fall to the safer cross-origin default. - """ - scheme = (scheme or "").strip().lower() - if not scheme or not netloc: - return None - # Strip userinfo (RFC 3986); Origin never carries credentials. - if "@" in netloc: - netloc = netloc.rsplit("@", 1)[1] - # IPv6 hosts use brackets (RFC 3986 sec 3.2.2): ``[::1]:8902``. Bare - # ``partition(":")`` mis-parses these, breaking ``unsloth studio -H ::1``. - if netloc.startswith("["): - close = netloc.find("]") - if close == -1: - return None - host = netloc[1:close] - rest = netloc[close + 1 :] - if rest.startswith(":"): - port_str = rest[1:] - elif rest == "": - port_str = "" - else: - return None - else: - host, _, port_str = netloc.partition(":") - host = host.strip().lower() - if not host: - return None - if port_str: - try: - port = int(port_str) - except ValueError: - return None - else: - port = _DEFAULT_PORTS.get(scheme, 0) - return (scheme, host, port) - - -def _is_loopback_ip(host: Optional[str]) -> bool: - """Return whether ``host`` is a loopback IP, including IPv4-mapped IPv6.""" - if not host or "%" in host: # a scope id (::1%eth0) is never a plain loopback - return False - try: - ip = ipaddress.ip_address(host) - except (TypeError, ValueError): - return False - mapped = getattr(ip, "ipv4_mapped", None) - return ip.is_loopback or (mapped is not None and mapped.is_loopback) - - -# A loopback peer carrying any of these is a proxy/tunnel relaying a remote -# client, so the peer is the proxy, not the caller: cloudflared sets -# cf-connecting-ip, reverse proxies set the rest (uvicorn only consumes -# x-forwarded-for, so the others survive to here). -_PROXIED_CLIENT_HEADERS = ( - "cf-connecting-ip", - "forwarded", - "x-forwarded-for", - "x-forwarded-host", - "x-real-ip", -) - - -def _host_header_is_loopback(host_header: Optional[str]) -> bool: - """Loopback/localhost check on the raw Host header. - - Reads the header directly so a malformed or absent Host cannot fall back to - ``request.url.hostname``'s (loopback) ASGI server address. - """ - if not host_header: - return False - host = host_header.strip() - if host.startswith("["): # [IPv6] or [IPv6]:port - end = host.find("]") - if end == -1 or (host[end + 1 :] and not host[end + 1 :].startswith(":")): - return False # unclosed bracket or junk after ] (e.g. [::1]evil) - host = host[1:end] - elif host.count(":") == 1: # host:port - host = host.split(":", 1)[0] - host = host.lower().rstrip(".") - return host == "localhost" or _is_loopback_ip(host) - - -def _is_local_bootstrap_request(request: Request) -> bool: - """Allow bootstrap injection only through a direct loopback authority.""" - client = request.client - if client is None or not _is_loopback_ip(client.host): - return False - if any(request.headers.get(h) is not None for h in _PROXIED_CLIENT_HEADERS): - return False - return _host_header_is_loopback(request.headers.get("host")) - - -def _is_same_origin_request(request: Request) -> bool: - """True when Origin is missing or matches request's scheme://host:port. - - Missing Origin counts as same-origin (top-level GETs omit it). Both sides - are canonicalised via :func:`_canonical_origin`; callers must emit - ``Vary: Origin``. - """ - origin = request.headers.get("origin") - if origin is None: - # Missing header: top-level same-document GETs omit Origin. - return True - # Empty string is not a valid serialised origin (RFC 6454 sec 6.1). - if not origin: - return False - # "null" token (sandboxed iframes, file:// pages) is never same-origin. - if origin == "null": - return False - # ``urlparse`` raises ``ValueError`` on malformed IPv6 brackets; swallow - # so a garbage Origin doesn't 500 the SPA handler. - try: - parsed = urlparse(origin) - except ValueError: - return False - origin_canon = _canonical_origin(parsed.scheme, parsed.netloc) - if origin_canon is None: - return False - try: - self_canon = _canonical_origin(request.url.scheme, request.url.netloc) - except ValueError: - return False - if self_canon is None: - return False - return origin_canon == self_canon - - -def _should_inject_bootstrap(request: Request) -> bool: - """Whether to embed the seeded bootstrap password in index.html.""" - if not _is_same_origin_request(request): - return False - if _IS_COLAB: - # Single-user notebook proxy: allow autofill, but never a public - # shareable tunnel (a Colab Cloudflare link sets cf-connecting-ip). - return request.headers.get("cf-connecting-ip") is None - return _is_local_bootstrap_request(request) - - def setup_frontend(app: FastAPI, build_path: Path): """Mount frontend static files (optional)""" if not build_path.exists(): @@ -1475,25 +1299,15 @@ def setup_frontend(app: FastAPI, build_path: Path): app.mount("/assets", StaticFiles(directory = assets_dir), name = "assets") def _build_index_response(request: Request) -> Response: - content = (build_path / "index.html").read_bytes() - content = _strip_crossorigin(content) - # Bootstrap pw goes only to a same-origin, direct-loopback client (or - # Colab's single-user notebook proxy): a wildcard bind must not serve it - # in-page to a LAN or proxied peer. Vary: Origin keeps caches honest. - if _should_inject_bootstrap(request): - content, nonce = _inject_bootstrap(content, app) - else: - nonce = None - headers = { - "Cache-Control": "no-cache, no-store, must-revalidate", - "Vary": "Origin", - } - if nonce: - headers[_CSP_SCRIPT_NONCE_HEADER] = nonce + # The seeded bootstrap password is never embedded in the served page: a + # loopback reverse proxy is indistinguishable from a genuine local client, + # so it cannot be scoped safely. It is printed at startup and saved to + # .bootstrap_password; the operator copies it from there on first login. + content = _strip_crossorigin((build_path / "index.html").read_bytes()) return Response( content = content, media_type = "text/html", - headers = headers, + headers = {"Cache-Control": "no-cache, no-store, must-revalidate"}, ) @app.get("/") diff --git a/studio/backend/tests/test_index_bootstrap_loopback.py b/studio/backend/tests/test_index_bootstrap_loopback.py deleted file mode 100644 index 87abace22c..0000000000 --- a/studio/backend/tests/test_index_bootstrap_loopback.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 - -"""Regression coverage for bootstrap password exposure to remote clients.""" - -from types import SimpleNamespace - - -def _request( - client_host, - request_host = "127.0.0.1", - headers = None, -): - """Build a minimal request; ``None`` models an unresolved peer / absent Host.""" - client = None if client_host is None else SimpleNamespace(host = client_host, port = 0) - hdrs = {} - if request_host is not None: - hdrs["host"] = request_host - hdrs.update(headers or {}) - return SimpleNamespace(client = client, headers = hdrs, url = SimpleNamespace(hostname = request_host)) - - -def test_loopback_peers_are_local(): - from main import _is_local_bootstrap_request - cases = ( - ("127.0.0.1", "127.0.0.1"), - ("::1", "::1"), - ("::ffff:127.0.0.1", "::ffff:127.0.0.1"), - ("127.0.0.1", "localhost"), - ) - for peer, host in cases: - assert _is_local_bootstrap_request(_request(peer, host)) is True, (peer, host) - - -def test_non_loopback_peers_are_remote(): - from main import _is_local_bootstrap_request - - # ::1%eth0 is a scope-id'd address, which ipaddress treats as loopback on - # 3.9+; it must not count as a direct local peer. - for host in ("192.168.1.10", "::ffff:192.168.1.10", "::1%eth0"): - assert _is_local_bootstrap_request(_request(host)) is False, host - - -def test_absent_or_unparseable_peer_fails_safe(): - from main import _is_local_bootstrap_request - for host in (None, "localhost"): - assert _is_local_bootstrap_request(_request(host)) is False, host - - -def test_cloudflare_tunnel_clients_are_remote_despite_loopback_peer(): - from main import _is_local_bootstrap_request - for client_ip in ("203.0.113.7", ""): - request = _request("127.0.0.1", headers = {"cf-connecting-ip": client_ip}) - assert _is_local_bootstrap_request(request) is False, client_ip - - -def test_dns_rebinding_host_is_remote_despite_loopback_peer(): - from main import _is_local_bootstrap_request - for host in ("attacker.example", "192.168.1.10", None): - assert _is_local_bootstrap_request(_request("127.0.0.1", host)) is False, host - - -def test_unparseable_request_host_fails_safe(): - """A Host that makes ``request.url.hostname`` raise must fall to remote.""" - from main import _is_local_bootstrap_request - - class _RaisingURL: - @property - def hostname(self): - raise ValueError("malformed host") - - request = SimpleNamespace( - client = SimpleNamespace(host = "127.0.0.1", port = 0), headers = {}, url = _RaisingURL() - ) - assert _is_local_bootstrap_request(request) is False - - -def test_reverse_proxy_forwarded_headers_are_remote(): - """A loopback proxy relaying a remote client (non-Cloudflare headers) is remote.""" - from main import _is_local_bootstrap_request - for header in ("forwarded", "x-forwarded-for", "x-forwarded-host", "x-real-ip"): - request = _request("127.0.0.1", "localhost", headers = {header: "203.0.113.7"}) - assert _is_local_bootstrap_request(request) is False, header - - -def test_malformed_or_absent_host_is_remote(): - """A malformed/absent/scope-id Host must not fall back to the loopback server address.""" - from main import _is_local_bootstrap_request - - # incl. bracket smuggling: [::1]evil / unclosed [::1 must not reduce to ::1 - for host in ( - "e_vil", - "[malformed", - "", - None, - "[::1%25eth0]:8888", - "[::1]attacker", - "[::1]evil.com", - "[::1", - "[::1]x", - ): - assert _is_local_bootstrap_request(_request("127.0.0.1", host)) is False, host - - -def test_colab_allows_notebook_proxy_but_not_shareable_tunnel(monkeypatch): - """Colab autofills its single-user proxy, but not a public Cloudflare link.""" - import main - - monkeypatch.setattr(main, "_IS_COLAB", True) - # In-notebook proxy: same-origin, no tunnel header, injects off-loopback too. - assert main._should_inject_bootstrap(_request("10.0.0.2", "colab.proxy")) is True - # Shareable Cloudflare link marks visitors with cf-connecting-ip; withhold. - tunnel = _request("127.0.0.1", "localhost", headers = {"cf-connecting-ip": "203.0.113.7"}) - assert main._should_inject_bootstrap(tunnel) is False - - -def test_non_colab_gate_requires_local_client(monkeypatch): - """Outside Colab the gate injects only for a direct loopback client.""" - import main - - monkeypatch.setattr(main, "_IS_COLAB", False) - assert main._should_inject_bootstrap(_request("127.0.0.1", "localhost")) is True - assert main._should_inject_bootstrap(_request("192.168.1.10", "localhost")) is False diff --git a/studio/backend/tests/test_index_bootstrap_origin.py b/studio/backend/tests/test_index_bootstrap_origin.py deleted file mode 100644 index b9d7f58867..0000000000 --- a/studio/backend/tests/test_index_bootstrap_origin.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 - -"""Regression coverage for the bootstrap-pw cross-origin leak (PR 5739). - -``_is_same_origin_request`` gates ``_inject_bootstrap`` so the seeded admin -password only ships to same-origin callers. -""" - -import os -from types import SimpleNamespace -from unittest.mock import MagicMock - -import pytest - - -def _build_request( - host: str, - origin: str | None, - scheme: str = "http", -) -> MagicMock: - request = MagicMock() - request.url.scheme = scheme - request.url.netloc = host - request.headers = {"origin": origin} if origin is not None else {} - return request - - -def test_is_same_origin_request_missing_origin_is_same_origin(monkeypatch): - from main import _is_same_origin_request - req = _build_request("127.0.0.1:8888", origin = None) - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_matching_origin_is_same_origin(): - from main import _is_same_origin_request - req = _build_request("127.0.0.1:8888", origin = "http://127.0.0.1:8888") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_evil_origin_is_cross_origin(): - from main import _is_same_origin_request - req = _build_request("127.0.0.1:8888", origin = "https://evil.example") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_scheme_mismatch_is_cross_origin(): - # https origin against an http listener is not same-origin. - from main import _is_same_origin_request - req = _build_request("127.0.0.1:8888", origin = "https://127.0.0.1:8888") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_port_mismatch_is_cross_origin(): - # Same host different port is not same-origin per the web platform. - from main import _is_same_origin_request - req = _build_request("127.0.0.1:8888", origin = "http://127.0.0.1:5173") - assert _is_same_origin_request(req) is False - - -# ── Canonicalisation: default-port stripping + case folding ───────── - - -def test_is_same_origin_request_https_default_port_stripped_on_origin(): - """RFC 6454 strips default ports on Origin; canonicalise both sides so this stays same-origin.""" - from main import _is_same_origin_request - - req = _build_request("example.com:443", origin = "https://example.com", scheme = "https") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_http_default_port_stripped_on_origin(): - from main import _is_same_origin_request - req = _build_request("example.com:80", origin = "http://example.com") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_default_port_present_on_origin(): - """Mirror case: Origin carries the default port, netloc doesn't. Same-origin.""" - from main import _is_same_origin_request - - req = _build_request("example.com", origin = "https://example.com:443", scheme = "https") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_host_case_insensitive(): - """Host portion is case-insensitive per RFC 3986.""" - from main import _is_same_origin_request - - req = _build_request("example.com", origin = "http://EXAMPLE.com") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_scheme_case_insensitive(): - """Scheme portion is case-insensitive per RFC 3986.""" - from main import _is_same_origin_request - - req = _build_request("example.com", origin = "HTTP://example.com") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_null_origin_is_cross_origin(): - """Sandboxed iframes / file:// pages send ``Origin: null``; cross-origin.""" - from main import _is_same_origin_request - - req = _build_request("example.com", origin = "null") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_unparseable_origin_is_cross_origin(): - """Hostless garbage falls to cross-origin so a malformed header can't leak the bootstrap.""" - from main import _is_same_origin_request - - req = _build_request("example.com", origin = "not-a-url") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_userinfo_in_netloc_ignored(): - """``user:pass@host:port`` netlocs (RFC 3986) must compare equal to the credentials-less Origin.""" - from main import _is_same_origin_request - - req = _build_request("user:pass@example.com:80", origin = "http://example.com") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_explicit_non_default_port_still_mismatch(): - """Canonicalisation does NOT collapse non-default ports to default.""" - from main import _is_same_origin_request - - req = _build_request("example.com", origin = "https://example.com:9999", scheme = "https") - assert _is_same_origin_request(req) is False diff --git a/studio/backend/tests/test_index_bootstrap_origin_extra.py b/studio/backend/tests/test_index_bootstrap_origin_extra.py deleted file mode 100644 index feda88c14c..0000000000 --- a/studio/backend/tests/test_index_bootstrap_origin_extra.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 - -"""Extra edge-case coverage for the bootstrap-pw cross-origin gate. -Companion to ``test_index_bootstrap_origin.py``: IPv6 netlocs, opaque origins -(``data:``, ``blob:``), comma-joined multi-Origin headers, and the -``localhost`` vs ``127.0.0.1`` distinct-origin rule. -""" - -from unittest.mock import MagicMock - - -def _build_request( - host: str, - origin, - scheme: str = "http", -) -> MagicMock: - request = MagicMock() - request.url.scheme = scheme - request.url.netloc = host - request.headers = {"origin": origin} if origin is not None else {} - return request - - -# ── IPv6 ──────────────────────────────────────────────────────────── - - -def test_is_same_origin_request_ipv6_loopback_same_origin(): - """Studio supports ``-H ::1`` binds; netloc is ``[::1]:8902``. Bare - ``partition(":")`` mis-parses the bracketed form and would refuse the - bootstrap on legitimate same-origin navigation. - """ - from main import _is_same_origin_request - - req = _build_request("[::1]:8902", origin = "http://[::1]:8902") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_ipv6_full_address_same_origin(): - from main import _is_same_origin_request - req = _build_request( - "[2001:db8::1]:8443", - origin = "https://[2001:db8::1]:8443", - scheme = "https", - ) - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_ipv6_default_port_stripped(): - """Browser drops :80 on ``http://[::1]``.""" - from main import _is_same_origin_request - - req = _build_request("[::1]:80", origin = "http://[::1]") - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_ipv6_case_insensitive(): - """Hex digits in IPv6 are case-insensitive per RFC 5952.""" - from main import _is_same_origin_request - - req = _build_request( - "[2001:DB8::1]:8443", - origin = "https://[2001:db8::1]:8443", - scheme = "https", - ) - assert _is_same_origin_request(req) is True - - -def test_is_same_origin_request_ipv6_different_host_cross_origin(): - from main import _is_same_origin_request - req = _build_request("[::1]:8902", origin = "http://[2001:db8::1]:8902") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_ipv6_port_mismatch_cross_origin(): - from main import _is_same_origin_request - req = _build_request("[::1]:8902", origin = "http://[::1]:9999") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_ipv6_userinfo_stripped(): - from main import _is_same_origin_request - req = _build_request("user:pass@[::1]:8902", origin = "http://[::1]:8902") - assert _is_same_origin_request(req) is True - - -# ── Opaque origins (data:, blob:) ─────────────────────────────────── - - -def test_is_same_origin_request_data_url_origin_is_cross_origin(): - """``data:`` URLs are opaque origins (HTML living standard); no host, never same-origin.""" - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "data:text/html,") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_blob_url_origin_is_cross_origin(): - """``blob:`` URLs carry the inner origin only in non-canonical form; the canonical comparison rejects them.""" - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "blob:http://127.0.0.1:8902/uuid") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_file_url_origin_is_cross_origin(): - """``file://`` pages usually send ``Origin: null``; older engines sent - ``Origin: file://``. Neither is same-origin vs an http listener. - """ - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "file://") - assert _is_same_origin_request(req) is False - - -# ── Multi-Origin header (comma-joined by Starlette) ──────────────── - - -def test_is_same_origin_request_comma_joined_origins_cross_origin(): - """Starlette joins repeated headers with ``, ``; the canonical parser can't - safely split this, so it falls to cross-origin. - """ - from main import _is_same_origin_request - - req = _build_request( - "127.0.0.1:8902", - origin = "http://127.0.0.1:8902, http://evil.example", - ) - assert _is_same_origin_request(req) is False - - -# ── localhost vs 127.0.0.1 (distinct origins per web platform) ────── - - -def test_is_same_origin_request_localhost_vs_127_is_cross_origin(): - """Browsers treat ``localhost`` and ``127.0.0.1`` as distinct origins; the - canonical comparison must not DNS-collapse them. - """ - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "http://localhost:8902") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_127_vs_localhost_is_cross_origin(): - from main import _is_same_origin_request - req = _build_request("localhost:8902", origin = "http://127.0.0.1:8902") - assert _is_same_origin_request(req) is False - - -# ── urlparse ValueError robustness ───────────────────────────────── - - -def test_is_same_origin_request_malformed_ipv6_bracket_is_cross_origin(): - """``urlparse`` raises ``ValueError('Invalid IPv6 URL')`` on unclosed - brackets (CVE-2024-11168 hardening). The gate must swallow it and fall to - cross-origin rather than 500 the SPA handler. - """ - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "http://[malformed") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_invalid_ipv6_address_is_cross_origin(): - """Bracketed but invalid IPv6 (e.g. ``[::g]``) also raises - ``ValueError`` inside ``urlparse``.""" - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "http://[::g]:8902") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_bracket_with_trailing_garbage_is_cross_origin(): - """Text after the closing bracket also raises inside ``urlparse``.""" - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "http://[2001:db8::1]extra:8902") - assert _is_same_origin_request(req) is False - - -def test_is_same_origin_request_empty_origin_header_is_cross_origin(): - """Explicit empty ``Origin:`` is not a valid serialised origin and must not - be conflated with a missing header; cross-origin, bootstrap withheld. - """ - from main import _is_same_origin_request - - req = _build_request("127.0.0.1:8902", origin = "") - assert _is_same_origin_request(req) is False diff --git a/studio/backend/tests/test_index_no_bootstrap_injection.py b/studio/backend/tests/test_index_no_bootstrap_injection.py new file mode 100644 index 0000000000..4fe0e97431 --- /dev/null +++ b/studio/backend/tests/test_index_no_bootstrap_injection.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""The seeded bootstrap password is never embedded in the served index.html. + +The seed is delivered only via the startup log and ``.bootstrap_password``; the +served page must never carry it, for any caller. +""" + +from fastapi import FastAPI +from starlette.testclient import TestClient + + +def _client(tmp_path, monkeypatch): + import main + + # Force the conditions that previously triggered injection so this proves the + # seed is withheld even with a pending password change and a seed present. + monkeypatch.setattr(main.storage, "requires_password_change", lambda *a, **k: True) + build = tmp_path / "build" + build.mkdir() + (build / "index.html").write_text("ok") + app = FastAPI() + app.state.bootstrap_password = "SEED-DO-NOT-LEAK" + assert main.setup_frontend(app, build) is True + return TestClient(app) + + +def test_index_never_contains_bootstrap_seed(tmp_path, monkeypatch): + client = _client(tmp_path, monkeypatch) + # root, SPA fallback, and a same-origin request all get a clean page. + for path, headers in ( + ("/", {}), + ("/some/spa/route", {}), + ("/", {"origin": "http://testserver"}), + ): + r = client.get(path, headers=headers) + assert r.status_code == 200, (path, r.status_code) + assert "SEED-DO-NOT-LEAK" not in r.text, path + assert "__UNSLOTH_BOOTSTRAP__" not in r.text, path + # no per-request injection means no Origin-varying and no script nonce + assert "x-internal-script-nonce" not in {k.lower() for k in r.headers} From 55ef7a1fe19621ca700a644afc57ffdaf89d3cc4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:43:04 +0000 Subject: [PATCH 002/297] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_index_no_bootstrap_injection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/backend/tests/test_index_no_bootstrap_injection.py b/studio/backend/tests/test_index_no_bootstrap_injection.py index 4fe0e97431..f1536cdd06 100644 --- a/studio/backend/tests/test_index_no_bootstrap_injection.py +++ b/studio/backend/tests/test_index_no_bootstrap_injection.py @@ -34,7 +34,7 @@ def test_index_never_contains_bootstrap_seed(tmp_path, monkeypatch): ("/some/spa/route", {}), ("/", {"origin": "http://testserver"}), ): - r = client.get(path, headers=headers) + r = client.get(path, headers = headers) assert r.status_code == 200, (path, r.status_code) assert "SEED-DO-NOT-LEAK" not in r.text, path assert "__UNSLOTH_BOOTSTRAP__" not in r.text, path From 3e03c401615a72d89eea22018b82b3222981fc3b Mon Sep 17 00:00:00 2001 From: danielhanchen <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:19:17 +0000 Subject: [PATCH 003/297] Print bootstrap password at startup for PR #7140 --- studio/backend/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/studio/backend/main.py b/studio/backend/main.py index 07bf67945f..20102c118e 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -566,7 +566,8 @@ async def lifespan(app: FastAPI): print("\n" + "=" * 60) print("DEFAULT ADMIN ACCOUNT CREATED") print(f" username: {storage.DEFAULT_ADMIN_USERNAME}") - print(f" password saved to: {bootstrap_path}") + print(f" password: {bootstrap_pw}") + print(f" (also saved to: {bootstrap_path})") print(" Open the Studio UI to sign in and change it.") print("=" * 60 + "\n") else: From 777e4abd7a8e68d3b84dde267ab430b4ff256e65 Mon Sep 17 00:00:00 2001 From: danielhanchen <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:39:35 +0000 Subject: [PATCH 004/297] Point to .bootstrap_password file instead of printing the seed for PR #7140 --- studio/backend/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/studio/backend/main.py b/studio/backend/main.py index 20102c118e..76037ac4d5 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -566,9 +566,8 @@ async def lifespan(app: FastAPI): print("\n" + "=" * 60) print("DEFAULT ADMIN ACCOUNT CREATED") print(f" username: {storage.DEFAULT_ADMIN_USERNAME}") - print(f" password: {bootstrap_pw}") - print(f" (also saved to: {bootstrap_path})") - print(" Open the Studio UI to sign in and change it.") + print(f" password saved to: {bootstrap_path}") + print(" open that file to read the password, then sign in and change it.") print("=" * 60 + "\n") else: app.state.bootstrap_password = storage.get_bootstrap_password() From 7145d4762f52e40bb0916a239a11fd408ae6f215 Mon Sep 17 00:00:00 2001 From: danielhanchen <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:42:16 +0000 Subject: [PATCH 005/297] Fix stale docstring comment to match file-based seed delivery for PR #7140 --- studio/backend/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/studio/backend/main.py b/studio/backend/main.py index 76037ac4d5..675124533e 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -1301,8 +1301,8 @@ def setup_frontend(app: FastAPI, build_path: Path): def _build_index_response(request: Request) -> Response: # The seeded bootstrap password is never embedded in the served page: a # loopback reverse proxy is indistinguishable from a genuine local client, - # so it cannot be scoped safely. It is printed at startup and saved to - # .bootstrap_password; the operator copies it from there on first login. + # so it cannot be scoped safely. It is saved to .bootstrap_password (0600); + # the operator reads it from there on first login. content = _strip_crossorigin((build_path / "index.html").read_bytes()) return Response( content = content, From 7e11dc6f3602b9e294835fc30f29af39cb678f8f Mon Sep 17 00:00:00 2001 From: danielhanchen <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:04:00 +0000 Subject: [PATCH 006/297] Fill current password in UI smoke tests after removing autofill for PR #7140 --- .github/workflows/studio-ui-smoke.yml | 10 +++++++--- tests/studio/playwright_chat_ime_i18n.py | 5 +++++ tests/studio/playwright_chat_ui.py | 4 ++++ tests/studio/playwright_extra_ui.py | 4 ++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 297a585430..51d6d6ee1d 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -255,17 +255,21 @@ jobs: jq -e '.status == "healthy"' /tmp/health3.json - name: Pass bootstrap pw for IME / i18n test - # IME smoke does the change-password against the bootstrap that - # Studio's frontend injects into the page, so it only needs the - # NEW password. + # IME smoke does its own change-password through the UI. The page no + # longer autofills the seed, so the test needs the current (bootstrap) + # password to fill the Current password field plus the NEW password. run: | + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) NEW="CIIme-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + echo "::add-mask::$OLD" echo "::add-mask::$NEW" + echo "STUDIO_IME_OLD_PW=$OLD" >> "$GITHUB_ENV" echo "STUDIO_IME_NEW_PW=$NEW" >> "$GITHUB_ENV" - name: Drive IME + multilingual paste regression with Playwright env: BASE_URL: http://127.0.0.1:18896 + STUDIO_OLD_PW: ${{ env.STUDIO_IME_OLD_PW }} STUDIO_NEW_PW: ${{ env.STUDIO_IME_NEW_PW }} PW_ART_DIR: logs/playwright_ime STUDIO_UI_STRICT: '1' diff --git a/tests/studio/playwright_chat_ime_i18n.py b/tests/studio/playwright_chat_ime_i18n.py index 9c01e95fd4..3189ff475c 100644 --- a/tests/studio/playwright_chat_ime_i18n.py +++ b/tests/studio/playwright_chat_ime_i18n.py @@ -28,6 +28,7 @@ from _playwright_robust import ( # noqa: E402 ) BASE = os.environ["BASE_URL"] +OLD = os.environ["STUDIO_OLD_PW"] NEW = os.environ["STUDIO_NEW_PW"] ART_DIR = os.environ.get("PW_ART_DIR", "logs/playwright_ime") ART = Path(ART_DIR) @@ -161,6 +162,10 @@ with sync_playwright() as p: pass pw_field = page.locator("#new-password") pw_field.wait_for(state = "visible", timeout = 60_000) + # Served page no longer autofills the seed; fill Current password when shown. + cur_pw = page.locator("#current-password") + if cur_pw.count(): + cur_pw.fill(OLD, timeout = 60_000) pw_field.fill(NEW, timeout = 60_000) page.fill("#confirm-password", NEW, timeout = 60_000) shoot("01-change-password-filled") diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index 065ba7a745..4d995aaebd 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -236,6 +236,10 @@ with sync_playwright() as p: pass # best-effort -- proceed even if network never idles pw_field = page.locator("#new-password") pw_field.wait_for(state = "visible", timeout = 60_000) + # Served page no longer autofills the seed; fill Current password when shown. + cur_pw = page.locator("#current-password") + if cur_pw.count(): + cur_pw.fill(OLD, timeout = 60_000) # Do NOT shoot() between wait_for and fill -- the screenshot's # font-load wait can let a background poll detach the form. pw_field.fill(NEW, timeout = 60_000) diff --git a/tests/studio/playwright_extra_ui.py b/tests/studio/playwright_extra_ui.py index 209a8a06f1..f6ada49f00 100644 --- a/tests/studio/playwright_extra_ui.py +++ b/tests/studio/playwright_extra_ui.py @@ -134,6 +134,10 @@ with sync_playwright() as p: pass pw_field = page.locator("#new-password") pw_field.wait_for(state = "visible", timeout = 60_000) + # Served page no longer autofills the seed; fill Current password when shown. + cur_pw = page.locator("#current-password") + if cur_pw.count(): + cur_pw.fill(OLD, timeout = 60_000) pw_field.fill(NEW, timeout = 60_000) page.fill("#confirm-password", NEW, timeout = 60_000) # Click submit AND wait for the POST response together so a server-side reject From 1a2cb620941bfb40ed6f78aaf8e02707956d7a19 Mon Sep 17 00:00:00 2001 From: danielhanchen <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:26:15 +0000 Subject: [PATCH 007/297] Update IME auth-flow guard tests for the non-autofill flow for PR #7140 --- .../test_composer_rtl_bidi_attribute.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/studio/test_composer_rtl_bidi_attribute.py b/tests/studio/test_composer_rtl_bidi_attribute.py index defbfab86c..c8a67c36d8 100644 --- a/tests/studio/test_composer_rtl_bidi_attribute.py +++ b/tests/studio/test_composer_rtl_bidi_attribute.py @@ -1,5 +1,5 @@ """RTL bidi contract on chat composers: all three need dir="auto", and the IME -smoke must drop the dead STUDIO_OLD_PW env var.""" +smoke must supply STUDIO_OLD_PW now that the served page no longer autofills the seed.""" from __future__ import annotations @@ -40,36 +40,36 @@ def test_compare_composer_has_dir_auto(): assert 'dir="auto"' in block, 'compare composer is missing dir="auto"' -def test_ime_workflow_step_does_not_set_studio_old_pw(): +def test_ime_workflow_step_sets_studio_old_pw(): yml = WORKFLOW_YML.read_text() drive_idx = yml.find("Drive IME + multilingual paste regression") assert drive_idx != -1, "IME drive step not found in workflow" next_step_idx = yml.find("- name:", drive_idx + 1) drive_block = yml[drive_idx : next_step_idx if next_step_idx != -1 else None] assert ( - "STUDIO_OLD_PW" not in drive_block - ), "IME drive step still passes dead STUDIO_OLD_PW env var" + "STUDIO_OLD_PW" in drive_block + ), "IME drive step must pass STUDIO_OLD_PW now the page no longer autofills the seed" assert "STUDIO_NEW_PW" in drive_block, "IME drive step missing STUDIO_NEW_PW" -def test_ime_pass_password_step_does_not_export_old_pw(): +def test_ime_pass_password_step_exports_old_pw(): yml = WORKFLOW_YML.read_text() pass_idx = yml.find("Pass bootstrap pw for IME / i18n test") assert pass_idx != -1, "IME password setup step not found" next_step_idx = yml.find("- name:", pass_idx + 1) pass_block = yml[pass_idx : next_step_idx if next_step_idx != -1 else None] assert ( - "STUDIO_IME_OLD_PW" not in pass_block - ), "IME password setup still exports dead STUDIO_IME_OLD_PW" + "STUDIO_IME_OLD_PW" in pass_block + ), "IME password setup must export STUDIO_IME_OLD_PW for the non-autofill flow" assert "STUDIO_IME_NEW_PW" in pass_block -def test_ime_playwright_script_does_not_read_studio_old_pw(): +def test_ime_playwright_script_reads_studio_old_pw(): src = IME_PY.read_text() code_only = re.sub(r'""".*?"""', "", src, flags = re.DOTALL) assert ( - "STUDIO_OLD_PW" not in code_only - ), "IME Playwright script still references dead STUDIO_OLD_PW env var" + 'os.environ["STUDIO_OLD_PW"]' in code_only + ), "IME Playwright script must read STUDIO_OLD_PW to fill the current-password field" assert 'os.environ["STUDIO_NEW_PW"]' in code_only From 70e4821f36e10e5f38f7e1ec51022f50ddee2cf6 Mon Sep 17 00:00:00 2001 From: danielhanchen <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:26:15 +0000 Subject: [PATCH 008/297] Point to bootstrap password on restart before first login for PR #7140 --- studio/backend/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/studio/backend/main.py b/studio/backend/main.py index 675124533e..4657b5c075 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -570,7 +570,13 @@ async def lifespan(app: FastAPI): print(" open that file to read the password, then sign in and change it.") print("=" * 60 + "\n") else: - app.state.bootstrap_password = storage.get_bootstrap_password() + bootstrap_pw = storage.get_bootstrap_password() + app.state.bootstrap_password = bootstrap_pw + # A restart before first login skips the creation banner above; still + # point the operator to the seed file while the bootstrap pw is unrotated. + if bootstrap_pw: + bootstrap_path = storage.DB_PATH.parent / ".bootstrap_password" + print(f"\nAdmin password change still required. Read it from: {bootstrap_path}\n") _lifespan_log.info( "lifespan startup completed in %.1fms", From 300b5f9b415ff2cad047dad4632e53df1f221a33 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Wed, 15 Jul 2026 07:43:40 -0700 Subject: [PATCH 009/297] Fix Studio toast close-button positioning (#7142) * Fix Studio toast close-button positioning * Use UTF-8 for locale regression test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Harden toast close-button positioning * Limit language menu height --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/frontend/src/components/ui/sonner.tsx | 2 +- .../settings/components/language-select.tsx | 6 +++- studio/frontend/src/i18n/locale-store.ts | 1 - studio/frontend/src/i18n/messages.ts | 25 ++++++++-------- .../test_locale_root_direction_contract.py | 30 +++++++++++++++++++ 5 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 tests/studio/test_locale_root_direction_contract.py diff --git a/studio/frontend/src/components/ui/sonner.tsx b/studio/frontend/src/components/ui/sonner.tsx index d6df0f4eaa..aec1235b81 100644 --- a/studio/frontend/src/components/ui/sonner.tsx +++ b/studio/frontend/src/components/ui/sonner.tsx @@ -90,7 +90,7 @@ const Toaster = ({ ...props }: ToasterProps) => { // Pin the close button inside the toast's top-right corner. // Sonner defaults to the left/outside edge, so keep the horizontal // override here and the top offset in index.css. - "--toast-close-button-start": "unset", + "--toast-close-button-start": "auto", "--toast-close-button-end": "8px", "--toast-close-button-transform": "none", } as React.CSSProperties diff --git a/studio/frontend/src/features/settings/components/language-select.tsx b/studio/frontend/src/features/settings/components/language-select.tsx index 01fe049a56..1f388d6f2c 100644 --- a/studio/frontend/src/features/settings/components/language-select.tsx +++ b/studio/frontend/src/features/settings/components/language-select.tsx @@ -35,7 +35,11 @@ export function LanguageSelect() { > - + {t("settings.appearance.language.autoDetect")} diff --git a/studio/frontend/src/i18n/locale-store.ts b/studio/frontend/src/i18n/locale-store.ts index cd5cac8db0..792e7e77fb 100644 --- a/studio/frontend/src/i18n/locale-store.ts +++ b/studio/frontend/src/i18n/locale-store.ts @@ -98,7 +98,6 @@ function writeStoredPreference(preference: LocalePreference): void { function syncDocumentLang(locale: Locale): void { if (typeof document === "undefined") return; document.documentElement.lang = locale; - document.documentElement.dir = LOCALES[locale].dir; } function notifySubscribers(): void { diff --git a/studio/frontend/src/i18n/messages.ts b/studio/frontend/src/i18n/messages.ts index 074f4ab44f..ead94a966e 100644 --- a/studio/frontend/src/i18n/messages.ts +++ b/studio/frontend/src/i18n/messages.ts @@ -15,19 +15,18 @@ import { de } from "./locales/de"; import { ko } from "./locales/ko"; import type { InterpolationValues, MessageKey } from "./types"; -// dir sets documentElement.dir; Arabic stays ltr (CSS still physical-direction) but renders rtl via bidi. export const LOCALES = { - en: { label: "English", nativeLabel: "English", dir: "ltr" }, - "zh-CN": { label: "Chinese (Simplified)", nativeLabel: "简体中文", dir: "ltr" }, - ja: { label: "Japanese", nativeLabel: "日本語", dir: "ltr" }, - ko: { label: "Korean", nativeLabel: "한국어", dir: "ltr" }, - es: { label: "Spanish", nativeLabel: "Español", dir: "ltr" }, - "pt-BR": { label: "Portuguese (Brazil)", nativeLabel: "Português (Brasil)", dir: "ltr" }, - fr: { label: "French", nativeLabel: "Français", dir: "ltr" }, - de: { label: "German", nativeLabel: "Deutsch", dir: "ltr" }, - ru: { label: "Russian", nativeLabel: "Русский", dir: "ltr" }, - hi: { label: "Hindi", nativeLabel: "हिन्दी", dir: "ltr" }, - ar: { label: "Arabic", nativeLabel: "العربية", dir: "ltr" }, + en: { label: "English", nativeLabel: "English" }, + "zh-CN": { label: "Chinese (Simplified)", nativeLabel: "简体中文" }, + ja: { label: "Japanese", nativeLabel: "日本語" }, + ko: { label: "Korean", nativeLabel: "한국어" }, + es: { label: "Spanish", nativeLabel: "Español" }, + "pt-BR": { label: "Portuguese (Brazil)", nativeLabel: "Português (Brasil)" }, + fr: { label: "French", nativeLabel: "Français" }, + de: { label: "German", nativeLabel: "Deutsch" }, + ru: { label: "Russian", nativeLabel: "Русский" }, + hi: { label: "Hindi", nativeLabel: "हिन्दी" }, + ar: { label: "Arabic", nativeLabel: "العربية" }, } as const; export type Locale = keyof typeof LOCALES; @@ -105,4 +104,4 @@ export function isSupportedLocale(value: unknown): value is Locale { typeof value === "string" && Object.prototype.hasOwnProperty.call(LOCALES, value) ); -} \ No newline at end of file +} diff --git a/tests/studio/test_locale_root_direction_contract.py b/tests/studio/test_locale_root_direction_contract.py new file mode 100644 index 0000000000..1baabdbbca --- /dev/null +++ b/tests/studio/test_locale_root_direction_contract.py @@ -0,0 +1,30 @@ +"""Regression guard for locale changes affecting the entire Studio layout.""" + +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[2] +LOCALE_STORE = REPO / "studio/frontend/src/i18n/locale-store.ts" +MESSAGES = REPO / "studio/frontend/src/i18n/messages.ts" +SONNER = REPO / "studio/frontend/src/components/ui/sonner.tsx" + + +def test_locale_changes_do_not_force_document_direction(): + src = LOCALE_STORE.read_text(encoding = "utf-8") + assert "document.documentElement.lang = locale" in src + assert "document.documentElement.dir" not in src, ( + "locale changes must not force the root direction; html[dir] changes " + "third-party component layout, including Sonner close-button positioning" + ) + + +def test_locale_metadata_does_not_advertise_unused_layout_direction(): + src = MESSAGES.read_text(encoding = "utf-8") + locales_block = src[src.index("export const LOCALES") : src.index("export type Locale")] + assert "dir:" not in locales_block + + +def test_toast_close_position_does_not_inherit_root_direction(): + src = SONNER.read_text(encoding = "utf-8") + assert '"--toast-close-button-start": "auto"' in src + assert '"--toast-close-button-start": "unset"' not in src From 9de84888cb9871b8c1a5ddec23501d7d52fe9e4a Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Wed, 15 Jul 2026 07:55:39 -0700 Subject: [PATCH 010/297] Studio: add Voice settings tab (dictation, dictionary, read aloud) (#7074) * Studio: add Voice settings tab (dictation, dictionary, read aloud) New Voice tab in Settings, placed just before About: - Dictation: microphone picker, browser STT engine, recognition language, and an inline mic test with a live transcript - Dictation dictionary: entries rewrite matching speech to their exact spelling and casing, applied in both dictation paths - Recent dictations: last 20 final transcripts with copy and clear, so text can be recovered if it lands in the wrong place - Read aloud: optional button on assistant responses with two engines, curated system voices (novelty and legacy voices filtered, quality ranked, capped at 20) or the TTS audio model loaded in Unsloth via /audio/generate (e.g. Orpheus), plus speed, pitch, volume and preview Settings persist in localStorage (unsloth_voice_settings) and are read at call time so changes apply without reloading the runtime. Adds en keys plus the tab label for ja, zh-CN and pt-BR. * Studio: drop the single option STT engine select, rename TTS option The STT engine dropdown only had one entry, so it added noise without giving a real choice. The engine row can come back once local STT models land. Also renames the TTS engine option Unsloth TTS model to Load TTS model to make the action clearer. * Studio: harden Voice settings against edge cases found in simulation Simulated the feature across Chromium, Firefox and WebKit plus node level unit runs and backend contract checks. Fixes from the findings: - Dictionary rewrite used a replacement string, so entries containing dollar patterns corrupted transcripts (A$$AP became A$AP, $& injected the match). Switched to the callback form of String.replace - Persisted voice settings now validate types on hydration: non string micDeviceId, dictationLanguage and ttsVoiceURI, and non boolean ttsEnabled fall back to defaults instead of flowing into the UI - Dictionary entries are trimmed, capped at 120 chars and re-sanitized on hydration - The Test dictation panel now falls back to the default microphone when the saved device is unplugged, matching the composer adapter Test coverage: 46 unit assertions (dictionary regex edge cases across unicode, word boundaries and injection, voice curation for simulated macOS, Windows and Linux voice inventories, corrupt storage merge), 13 backend contract checks against /audio/generate on an isolated instance, and 60 browser assertions across the three engines covering rendering, degradation without SpeechRecognition, curation in a real DOM, dictionary persistence with unicode and dollar entries, the no-model preview error path and corrupt localStorage recovery. * Studio: address Voice settings review feedback Verified each review comment before acting. Confirmed and fixed: - Editing a dictionary entry was broken in two ways: the store trimmed on every keystroke so spaces could not be typed, and clearing the field deleted the entry and unmounted the input mid edit. Updates now keep the raw value and a blur commit trims or removes the entry - The unplugged mic fallback checked instanceof DOMException, but a cross browser probe showed Firefox and WebKit throw OverconstrainedError objects that are not DOMExceptions, so the fallback never fired there. Matching on the error name now - When the browser ended a dictation test on its own (silence timeout), the mic stream stayed open. All recognition end paths now stop the tracks and save the transcript through a single finalize path - The studio TTS audio element now releases its WAV data URL as soon as playback ends, fails or is cancelled - Allow microphone now reports insecure contexts (no mediaDevices) accurately instead of claiming access was blocked - Voice tab copy moved into i18n keys per src/i18n/AGENTS.md, so locale overlays can translate it; en is the baseline and parity passes - unsloth_voice_settings added to the Reset all local preferences key list so voice preferences obey the reset - Non default microphones note that the system default is used when the browser speech engine cannot bind a specific device, since browsers without the start(track) overload ignore the argument silently Re-ran the full simulation set after the changes: 46 unit assertions, 13 backend contract checks and 60 browser assertions across Chromium, Firefox and WebKit all pass, plus a dedicated browser probe for the dictionary editing behavior. * Studio: use the chat mic icon in Voice settings for consistency The Voice tab and its buttons used the hugeicons Mic02 glyph while the chat composer uses a custom filled mic. Extract that composer icon into a shared lib/mic-icon component, drop the duplicate inline copies in thread.tsx and shared-composer.tsx, and use it for the Voice tab icon and the tab's mic buttons so the microphone looks the same everywhere. * Studio: address second round of Voice settings review feedback Verified each new comment against the current code first. One item was already fixed in the previous round (recording transcripts when the browser ends a dictation test on its own). Confirmed and fixed: - The microphone row showed a picker with generic names when browsers enumerate unlabeled devices before permission, leaving no way to grant access from the row. It now branches on whether labels are visible and shows Allow microphone otherwise - Compare chat dictation ignored the selected microphone. It now opens the chosen device with the same fallback rules as the main adapter, passes the track to recognition where supported and releases the stream when recognition ends - Closing the Voice tab cancelled the shared speechSynthesis even when read aloud was playing a chat message. Cleanup now only cancels when the tab owns an active preview - Double clicking Start test could race two recognizers and leak the first stream. A starting flag set before the getUserMedia await makes start reentrancy safe - Turning off the read aloud setting mid playback removed the only stop control. The stop button now renders whenever a message is speaking - When an engine lacks the start(track) overload, both dictation paths now release the selected device stream before retrying with the default microphone instead of holding it open - Read aloud support no longer requires Web Speech synthesis: the Unsloth TTS engine only needs audio playback, so it stays available in WebViews without speechSynthesis, with a clear error if the system engine is chosen there Not addressed here: cancelling in flight backend TTS generation on stop. The route runs generation in a worker thread without a cancellation path, which is shared pre existing behavior with audio chat generation and belongs in a backend change. All suites re-run green: 46 unit, 13 backend contract and 60 browser matrix assertions across Chromium, Firefox and WebKit, plus probes for the unlabeled device branch and the double click race. * Studio: drop empty and duplicate voiceURIs so the Voice tab never renders a crashing Select item * Studio: guard dictation mic lifecycle in Voice test and Compare composer Release a microphone opened after the component unmounts, and stop Compare dictation on a permission or security failure instead of silently recording from the default device, matching the main chat adapter. * Studio: fix dictation and read-aloud lifecycle edge cases in Voice settings - Join final dictation chunks with a space so recorded transcripts do not merge words - Ignore a stale recognizer onend so a quick stop then restart is not torn down - Use previewingRef so a double click on TTS preview does not orphan the first request - Keep the read-aloud stop control visible when a new run starts while a message is spoken - Stop the dictionary remove button from deleting an adjacent entry on a blur then click race * Studio: trim redundant Voice settings comments * Studio: fix Voice preview and Compare dictation edge cases - Only cancel the shared speechSynthesis for a system-voice preview, so stopping a Studio preview no longer stops an unrelated chat read-aloud - Release the Studio preview audio and its WAV data URL on normal completion - Iterate every finalized result in Compare dictation so batched phrases are kept - Cap persisted recent dictations to the last 20 on hydration * Studio: use clipboard fallback for recents and release failed preview audio - Copy recent dictations via the copyToClipboard helper so the execCommand fallback works in Safari and insecure http LAN contexts - Release the Studio preview audio when play() rejects, not just on ended/error * Studio: surface dictation and read-aloud failures instead of failing silently - Compare dictation reports microphone and speech-recognition errors via toast, reusing the main chat adapter's describeMediaError and describeSpeechError - Read-aloud toasts genuine model or synthesis failures while ignoring cancellations * Harden cross-browser microphone errors * Surface voice test recognition errors and fall back to Studio TTS - Voice test now toasts non-abort speech-recognition failures instead of ending silently, matching the main and Compare dictation paths. - Read-aloud routes to the backend model when the runtime lacks Web Speech synthesis (audio-only WebView), so it no longer errors immediately. * Fix read-aloud fallback controls * Guard read-aloud stop when deleting a non-speaking message aui.message().stopSpeaking() throws unless this message is the one being read aloud, so calling it unconditionally rejected the delete handler before the message was removed. Only stop speech when this message is speaking. * Cap recent dictation transcript length before persisting Recent dictations only limited entry count, so a long transcript stored the full text in the persisted voice settings and a few could exceed the localStorage quota, throwing synchronously from the uncaught dictation cleanup path. Truncate each entry on save and on hydration, matching the dictionary cap. * Harden read-aloud stop on delete and surface preview playback errors - Deleting a message now stops read-aloud when the spoken message is among those removed (including a user prompt's cascaded assistant replies), read at click time and guarded so a playback end between render and click cannot abort the delete. - Voice preview now reports playback failures instead of silently resetting the button, matching the read-aloud path. * Remove stray review notes; notify TTS subscribers; drop regex lookbehind - Remove plans/review_*.md scratch files accidentally committed earlier. - Studio read-aloud now notifies speech subscribers on the async starting -> running transition so status does not stay stuck at starting. - Dictionary correction captures the leading boundary instead of a lookbehind so it works on engines with dictation but no lookbehind (Safari < 16.4). * Fix keyboard deletion of an emptied dictionary entry Tabbing to a just-emptied row's Remove button blurred the input and commit-spliced the empty row, so with index-keyed rows the button's keyboard activation deleted the next entry. Skip the commit when focus moves to that row's Remove button; the existing mouse guard is kept. * Reapply Studio TTS playback rate on loadedmetadata Some browsers reset an Audio element's playbackRate to 1 once the source loads, so the selected speed could be dropped for read-aloud and voice preview. Reapply it on loadedmetadata in both paths. --------- Co-authored-by: danielhanchen --- .../src/components/assistant-ui/thread.tsx | 72 +- .../studio-speech-synthesis-adapter.ts | 338 +++++++ .../studio-web-speech-dictation-adapter.ts | 135 ++- .../src/features/chat/runtime-provider.tsx | 12 +- .../src/features/chat/shared-composer.tsx | 151 ++- .../src/features/settings/settings-dialog.tsx | 52 +- .../src/features/settings/settings-search.ts | 16 + .../settings/stores/settings-dialog-store.ts | 2 + .../settings/stores/voice-settings-store.ts | 245 +++++ .../features/settings/tabs/general-tab.tsx | 2 + .../src/features/settings/tabs/voice-tab.tsx | 882 ++++++++++++++++++ studio/frontend/src/i18n/locales/en.ts | 74 +- studio/frontend/src/i18n/locales/ja.ts | 1 + studio/frontend/src/i18n/locales/pt-br.ts | 1 + studio/frontend/src/i18n/locales/zh-CN.ts | 1 + studio/frontend/src/lib/mic-icon.tsx | 17 + 16 files changed, 1916 insertions(+), 85 deletions(-) create mode 100644 studio/frontend/src/features/chat/adapters/studio-speech-synthesis-adapter.ts create mode 100644 studio/frontend/src/features/settings/stores/voice-settings-store.ts create mode 100644 studio/frontend/src/features/settings/tabs/voice-tab.tsx create mode 100644 studio/frontend/src/lib/mic-icon.tsx diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index 9b502a5000..93cf162ca7 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -97,9 +97,11 @@ import { ThreadDocumentsBar } from "@/features/rag/components/thread-documents-b import { KnowledgeBaseComposerButton } from "@/features/rag/components/knowledge-base-composer-button"; import { DocumentPreviewMount } from "@/features/rag/components/document-preview-mount"; import { useUserProfileStore } from "@/features/profile/stores/user-profile-store"; +import { useVoiceSettingsStore } from "@/features/settings/stores/voice-settings-store"; import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; import { isTauri } from "@/lib/api-base"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { MicIcon } from "@/lib/mic-icon"; import { toast } from "@/lib/toast"; import { Tick02Icon } from "@/lib/tick-icon"; import { cn } from "@/lib/utils"; @@ -150,6 +152,8 @@ import { RefreshCwIcon, SquareIcon, TerminalIcon, + Volume2Icon, + VolumeXIcon, XIcon, } from "lucide-react"; import { @@ -2118,19 +2122,6 @@ function useImeComposerInputHandlers({ }; } -// Phosphor microphone. Inlined to avoid a new icon dependency. -const MicIcon: FC<{ className?: string }> = ({ className }) => ( - - - -); - // HugeIcons arrow-down-01 (stroke-standard): straight-line chevron. const ArrowDownStandardIcon: FC<{ className?: string }> = ({ className }) => ( { const isRunning = useAuiState(({ thread }) => thread.isRunning); const handleDelete = async () => { - const remoteId = aui.threadListItem().getState().remoteId; const thread = aui.thread(); + // Deleting a message, and for a user prompt its cascaded assistant replies, + // unmounts their only Stop reading control. Stop read-aloud first when the + // spoken message is among those removed. Read speech state at click time and + // guard the call, which throws if playback already ended. + const speakingId = thread.getState().speech?.messageId; + if (speakingId) { + const { messages } = thread.export(); + const target = messages.find(({ message }) => message.id === messageId); + const removed = new Set([messageId]); + if (target?.message.role === "user") { + for (const { parentId, message } of messages) { + if (parentId === messageId && message.role === "assistant") { + removed.add(message.id); + } + } + } + if (removed.has(speakingId)) { + try { + thread.stopSpeaking(); + } catch { + // Playback ended between reading the state and stopping it. + } + } + } + + const remoteId = aui.threadListItem().getState().remoteId; try { await deleteThreadMessage({ thread: { @@ -3883,11 +3899,15 @@ const EditAssistantMessageButton: FC = () => { const AssistantActionBar: FC = () => { const { forkMessage, forkDisabled } = useForkMessageAction(); const [detailsOpen, setDetailsOpen] = useState(false); + const ttsEnabled = useVoiceSettingsStore((s) => s.ttsEnabled); + // hideWhenRunning is thread-level, so a new run would hide this bar and its + // only Stop reading control while read-aloud keeps playing; keep it shown. + const speaking = useAuiState(({ message }) => message.speech != null); return ( <> @@ -3899,6 +3919,28 @@ const AssistantActionBar: FC = () => { + {ttsEnabled && ( + + + + + + + + )} + {/* Not gated on ttsEnabled: turning the setting off while a message + is being read aloud must not remove the only stop control. */} + + + + + + + voice.voiceURI === voiceURI); +} + +// macOS novelty and legacy Eloquence voices that sound robotic and flood the picker. +const LOW_QUALITY_VOICE_NAMES = new Set([ + "albert", + "bad news", + "bahh", + "bells", + "boing", + "bubbles", + "cellos", + "deranged", + "eddy", + "flo", + "fred", + "good news", + "grandma", + "grandpa", + "hysterical", + "jester", + "junior", + "kathy", + "organ", + "princess", + "ralph", + "reed", + "rocko", + "sandy", + "shelley", + "superstar", + "trinoids", + "whisper", + "wobble", + "zarvox", +]); + +function voiceBaseName(voice: SpeechSynthesisVoice): string { + // "Eddy (English (US))" -> "eddy"; "Bad News" -> "bad news" + const name = voice.name.split("(")[0]?.trim().toLowerCase() ?? ""; + return name; +} + +function voiceQualityScore(voice: SpeechSynthesisVoice): number { + const name = voice.name.toLowerCase(); + let score = 0; + if (name.includes("premium")) score += 8; + if (name.includes("enhanced")) score += 7; + if (name.includes("natural") || name.includes("neural")) score += 6; + if (name.includes("siri")) score += 6; + if (name.includes("google")) score += 5; + if (name.includes("microsoft")) score += 4; + if (voice.default) score += 3; + return score; +} + +function langBase(tag: string): string { + return tag.toLowerCase().split(/[-_]/)[0] ?? ""; +} + +const MAX_CURATED_VOICES = 20; + +/** + * Keep the best, most relevant voices: drop low-quality ones, keep English, + * the browser language, and the dictation language, rank by quality hints, + * and cap the list. The selected voice is always kept. + */ +export function curateSystemVoices( + voices: SpeechSynthesisVoice[], + selectedVoiceURI?: string, +): SpeechSynthesisVoice[] { + const { dictationLanguage } = useVoiceSettingsStore.getState(); + const wantedLangs = new Set(["en"]); + if (typeof navigator !== "undefined" && navigator.language) { + wantedLangs.add(langBase(navigator.language)); + } + if (dictationLanguage && dictationLanguage !== "auto") { + wantedLangs.add(langBase(dictationLanguage)); + } + + // WebKit and Linux engines report voices with empty or duplicate voiceURIs; + // drop them so the Radix Select never gets an empty or colliding value. + const seenVoiceURIs = new Set(); + const kept = voices.filter((voice) => { + if (!voice.voiceURI || seenVoiceURIs.has(voice.voiceURI)) return false; + seenVoiceURIs.add(voice.voiceURI); + if (LOW_QUALITY_VOICE_NAMES.has(voiceBaseName(voice))) return false; + return wantedLangs.has(langBase(voice.lang)); + }); + + kept.sort((a, b) => { + const scoreDiff = voiceQualityScore(b) - voiceQualityScore(a); + if (scoreDiff !== 0) return scoreDiff; + return a.name.localeCompare(b.name); + }); + + const curated = kept.slice(0, MAX_CURATED_VOICES); + if ( + selectedVoiceURI && + selectedVoiceURI !== "default" && + !curated.some((voice) => voice.voiceURI === selectedVoiceURI) + ) { + const selected = voices.find( + (voice) => voice.voiceURI === selectedVoiceURI, + ); + if (selected) curated.push(selected); + } + return curated; +} + +/** Build an utterance from the current Voice settings. */ +export function createConfiguredUtterance( + text: string, +): SpeechSynthesisUtterance { + const { ttsVoiceURI, ttsRate, ttsPitch, ttsVolume } = + useVoiceSettingsStore.getState(); + const utterance = new SpeechSynthesisUtterance(text); + const voice = findTtsVoice(ttsVoiceURI); + if (voice) { + utterance.voice = voice; + utterance.lang = voice.lang; + } + utterance.rate = ttsRate; + utterance.pitch = ttsPitch; + utterance.volume = ttsVolume; + return utterance; +} + +/** Generate speech via the loaded TTS audio model; returns a WAV data URL. */ +export async function generateStudioTtsAudio( + text: string, + signal?: AbortSignal, +): Promise { + const response = await authFetch("/api/inference/audio/generate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ role: "user", content: text }], + stream: false, + }), + signal, + }); + if (!response.ok) { + const body = (await response.json().catch(() => null)) as { + detail?: string; + } | null; + const detail = body?.detail ?? `HTTP ${response.status}`; + if (/no model loaded|not an audio model/i.test(detail)) { + throw new Error( + "No TTS model is loaded. Load an audio model (e.g. Orpheus TTS) from the model selector, then try again.", + ); + } + throw new Error(detail); + } + const data = (await response.json()) as { audio?: { data?: string } }; + if (!data.audio?.data) { + throw new Error("The TTS model returned no audio."); + } + return `data:audio/wav;base64,${data.audio.data}`; +} + +function speakWithStudioModel( + text: string, + handleEnd: ( + reason: "finished" | "error" | "cancelled", + error?: unknown, + ) => void, + markRunning: () => void, +): { cancel: () => void } { + const { ttsRate, ttsVolume } = useVoiceSettingsStore.getState(); + const controller = new AbortController(); + let audio: HTMLAudioElement | null = null; + let cancelled = false; + + // Release the element and its multi-MB WAV data URL as soon as playback ends. + const cleanup = () => { + if (audio) { + audio.pause(); + audio.removeAttribute("src"); + audio = null; + } + }; + + void (async () => { + try { + const url = await generateStudioTtsAudio(text, controller.signal); + if (cancelled) return; + audio = new Audio(url); + audio.playbackRate = ttsRate; + audio.volume = ttsVolume; + // Some browsers reset playbackRate to 1 once the source loads; reapply + // it on loadedmetadata so the speed setting reliably takes effect. + audio.addEventListener("loadedmetadata", () => { + if (audio) audio.playbackRate = ttsRate; + }); + audio.addEventListener("ended", () => { + cleanup(); + handleEnd("finished"); + }); + audio.addEventListener("error", () => { + if (cancelled) return; + cleanup(); + handleEnd("error", new Error("Audio playback failed.")); + }); + markRunning(); + await audio.play(); + } catch (error) { + if (cancelled || controller.signal.aborted) return; + cleanup(); + handleEnd("error", error); + } + })(); + + return { + cancel: () => { + cancelled = true; + controller.abort(); + cleanup(); + handleEnd("cancelled"); + }, + }; +} + +/** + * Text-to-speech for assistant messages. Reads Voice settings at speak time. + * Engines: "system" (speechSynthesis) or "studio" (loaded TTS audio model). + */ +export class StudioSpeechSynthesisAdapter implements SpeechSynthesisAdapter { + /** Web Speech synthesis, used by the "system" engine. */ + static systemVoicesSupported(): boolean { + return ( + typeof window !== "undefined" && + "speechSynthesis" in window && + typeof window.SpeechSynthesisUtterance !== "undefined" + ); + } + + // The "studio" engine only needs fetch + Audio playback, so a WebView + // without Web Speech synthesis can still read aloud through the backend. + static isSupported(): boolean { + return ( + StudioSpeechSynthesisAdapter.systemVoicesSupported() || + (typeof window !== "undefined" && typeof window.Audio !== "undefined") + ); + } + + speak(text: string): SpeechSynthesisAdapter.Utterance { + const subscribers = new Set<() => void>(); + + const handleEnd = ( + reason: "finished" | "error" | "cancelled", + error?: unknown, + ) => { + if (res.status.type === "ended") return; + // Surface genuine read-aloud failures; a cancelled/interrupted utterance + // is a normal stop, not an error, and must not toast. + if (reason === "error" && error !== "interrupted" && error !== "canceled") { + toast.error(error instanceof Error ? error.message : "Read aloud failed."); + } + res.status = { type: "ended", reason, error }; + for (const handler of subscribers) handler(); + }; + + let cancelImpl: () => void; + const { ttsEngine } = useVoiceSettingsStore.getState(); + + const res: SpeechSynthesisAdapter.Utterance = { + status: { type: "starting" }, + cancel: () => cancelImpl(), + subscribe: (callback) => { + if (res.status.type === "ended") { + let cancelled = false; + queueMicrotask(() => { + if (!cancelled) callback(); + }); + return () => { + cancelled = true; + }; + } + subscribers.add(callback); + return () => { + subscribers.delete(callback); + }; + }, + }; + + // Fall back to the backend model when the runtime lacks Web Speech + // synthesis (e.g. an audio-only WebView), so read-aloud still works. + if ( + ttsEngine === "studio" || + !StudioSpeechSynthesisAdapter.systemVoicesSupported() + ) { + const session = speakWithStudioModel(text, handleEnd, () => { + if (res.status.type === "ended") return; + // Notify subscribers of the async starting -> running transition; + // the adapter contract drives UI state off these subscribe callbacks. + res.status = { type: "running" }; + for (const handler of subscribers) handler(); + }); + cancelImpl = session.cancel; + return res; + } + + const utterance = createConfiguredUtterance(text); + utterance.addEventListener("end", () => handleEnd("finished")); + utterance.addEventListener("error", (e) => handleEnd("error", e.error)); + + // Chrome silently drops speak() while another utterance is queued from a + // cancelled run; clearing first keeps read-aloud deterministic. + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + res.status = { type: "running" }; + + cancelImpl = () => { + window.speechSynthesis.cancel(); + handleEnd("cancelled"); + }; + return res; + } +} diff --git a/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts b/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts index 146a12bd40..b7a8e904ed 100644 --- a/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts +++ b/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts @@ -1,10 +1,18 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +import { + applyDictationDictionary, + recordRecentDictation, + resolveDictationLanguage, + useVoiceSettingsStore, +} from "@/features/settings/stores/voice-settings-store"; import type { DictationAdapter } from "@assistant-ui/react"; import { toast } from "sonner"; -const getSpeechRecognitionAPI = (): SpeechRecognitionConstructor | undefined => { +const getSpeechRecognitionAPI = (): + | SpeechRecognitionConstructor + | undefined => { if (typeof window === "undefined") return undefined; return window.SpeechRecognition ?? window.webkitSpeechRecognition; }; @@ -13,23 +21,34 @@ const stopStream = (stream: MediaStream | null) => { stream?.getTracks().forEach((track) => track.stop()); }; -const describeMediaError = (error: unknown): string => { - if (!(error instanceof DOMException)) { - return "Dictation could not access the microphone."; - } - if (error.name === "NotAllowedError") { - return "Microphone access is blocked. Allow microphone access for this Unsloth page, then try again."; - } - if (error.name === "NotFoundError") { - return "No microphone was found for dictation."; - } - if (error.name === "NotReadableError") { - return "The microphone is already in use or unavailable."; - } - return error.message || "Dictation could not access the microphone."; +const mediaErrorName = (error: unknown): unknown => + error && typeof error === "object" && "name" in error + ? (error as { name?: unknown }).name + : undefined; + +/** True for getUserMedia errors meaning the requested device is gone. */ +export const isMissingDeviceError = (error: unknown): boolean => { + const name = mediaErrorName(error); + return name === "OverconstrainedError" || name === "NotFoundError"; }; -const describeSpeechError = (error: string, message?: string): string => { +export const describeMediaError = (error: unknown): string => { + const name = mediaErrorName(error); + if (name === "NotAllowedError" || name === "SecurityError") { + return "Microphone access is blocked. Allow microphone access for this Unsloth page, then try again."; + } + if (name === "NotFoundError" || name === "OverconstrainedError") { + return "No microphone was found for dictation."; + } + if (name === "NotReadableError" || name === "AbortError") { + return "The microphone is already in use or unavailable."; + } + return error instanceof Error && error.message + ? error.message + : "Dictation could not access the microphone."; +}; + +export const describeSpeechError = (error: string, message?: string): string => { if (error === "not-allowed") { return "Speech recognition was blocked by the browser. Check microphone permissions for this Unsloth page."; } @@ -46,7 +65,7 @@ const describeSpeechError = (error: string, message?: string): string => { }; export class StudioWebSpeechDictationAdapter implements DictationAdapter { - private readonly language: string; + private readonly language: string | undefined; private readonly continuous: boolean; private readonly interimResults: boolean; @@ -57,7 +76,8 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { interimResults?: boolean; } = {}, ) { - this.language = options.language ?? navigator.language ?? "en-US"; + // Resolved from Voice settings at listen() time unless overridden. + this.language = options.language; this.continuous = options.continuous ?? true; this.interimResults = options.interimResults ?? true; } @@ -78,13 +98,17 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { } const recognition = new SpeechRecognitionAPI(); - recognition.lang = this.language; + recognition.lang = this.language ?? resolveDictationLanguage(); recognition.continuous = this.continuous; recognition.interimResults = this.interimResults; const speechStartCallbacks = new Set<() => void>(); - const speechEndCallbacks = new Set<(result: DictationAdapter.Result) => void>(); - const speechCallbacks = new Set<(result: DictationAdapter.Result) => void>(); + const speechEndCallbacks = new Set< + (result: DictationAdapter.Result) => void + >(); + const speechCallbacks = new Set< + (result: DictationAdapter.Result) => void + >(); let stream: MediaStream | null = null; let finalTranscript = ""; @@ -147,6 +171,9 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { for (const callback of speechEndCallbacks) { callback({ transcript: finalTranscript }); } + if (reason !== "cancelled") { + recordRecentDictation(finalTranscript); + } finalTranscript = ""; } resolveEnded?.(); @@ -162,14 +189,26 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { recognition.addEventListener("result", (event) => { const speechEvent = event as SpeechRecognitionEvent; - for (let i = speechEvent.resultIndex; i < speechEvent.results.length; i++) { + for ( + let i = speechEvent.resultIndex; + i < speechEvent.results.length; + i++ + ) { const result = speechEvent.results[i]; if (!result) continue; const transcript = result[0]?.transcript ?? ""; if (result.isFinal) { - finalTranscript += transcript; + const corrected = applyDictationDictionary(transcript); + // Join final chunks with a single space so recorded transcripts do + // not merge words when a browser omits leading whitespace. + const trimmed = corrected.trim(); + if (trimmed) { + finalTranscript = finalTranscript + ? `${finalTranscript} ${trimmed}` + : trimmed; + } for (const callback of speechCallbacks) { - callback({ transcript, isFinal: true }); + callback({ transcript: corrected, isFinal: true }); } } else { for (const callback of speechCallbacks) { @@ -189,7 +228,10 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { finish("cancelled"); return; } - const description = describeSpeechError(errorEvent.error, errorEvent.message); + const description = describeSpeechError( + errorEvent.error, + errorEvent.message, + ); console.error("Dictation error:", errorEvent.error, errorEvent.message); toast.error(description); finish("error"); @@ -197,9 +239,30 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { void (async () => { try { - stream = await navigator.mediaDevices.getUserMedia({ - audio: { echoCancellation: true, noiseSuppression: true }, - }); + const { micDeviceId } = useVoiceSettingsStore.getState(); + const baseAudio: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: true, + }; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + micDeviceId && micDeviceId !== "default" + ? { ...baseAudio, deviceId: { exact: micDeviceId } } + : baseAudio, + }); + } catch (error) { + // Saved mic may be unplugged; fall back to the default device. + // Firefox and WebKit throw OverconstrainedError objects that are + // not DOMException instances, so match on the error name. + if (micDeviceId !== "default" && isMissingDeviceError(error)) { + stream = await navigator.mediaDevices.getUserMedia({ + audio: baseAudio, + }); + } else { + throw error; + } + } if (ended) { stopStream(stream); stream = null; @@ -207,13 +270,23 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { } const audioTrack = stream.getAudioTracks()[0]; if (!audioTrack || audioTrack.readyState !== "live") { - throw new DOMException("No live microphone track is available.", "NotFoundError"); + throw new DOMException( + "No live microphone track is available.", + "NotFoundError", + ); } try { recognition.start(audioTrack); } catch (error) { - // Older engines expose only start(); retry without the experimental track overload. - console.debug("Dictation start(audioTrack) failed; retrying start().", error); + // Older engines expose only start(); retry without the experimental + // track overload. Recognition then captures from the default device, + // so release the selected-device stream instead of holding it open. + console.debug( + "Dictation start(audioTrack) failed; retrying start().", + error, + ); + stopStream(stream); + stream = null; recognition.start(); } started = true; diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index 634e5ec8b0..20fc979777 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -33,6 +33,7 @@ import { useRef, } from "react"; import { toast } from "sonner"; +import { StudioSpeechSynthesisAdapter } from "./adapters/studio-speech-synthesis-adapter"; import { StudioWebSpeechDictationAdapter } from "./adapters/studio-web-speech-dictation-adapter"; import { ThreadAutosaveHandle, @@ -1033,6 +1034,13 @@ function useStudioRuntimeAdapters( : undefined, [], ); + const speech = useMemo( + () => + StudioSpeechSynthesisAdapter.isSupported() + ? new StudioSpeechSynthesisAdapter() + : undefined, + [], + ); const attachments = useMemo( () => new CompositeAttachmentAdapter([ @@ -1047,8 +1055,8 @@ function useStudioRuntimeAdapters( [], ); const adapters = useMemo( - () => ({ history, dictation, attachments }), - [history, dictation, attachments], + () => ({ history, dictation, speech, attachments }), + [history, dictation, speech, attachments], ); return adapters; diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index 2ed9589461..babff892bb 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -8,6 +8,7 @@ import { } from "@/components/assistant-ui/think-aria-label"; import { Button } from "@/components/ui/button"; import { BulbIcon } from "@/lib/bulb-icon"; +import { MicIcon } from "@/lib/mic-icon"; import { Tick02Icon } from "@/lib/tick-icon"; import { cn } from "@/lib/utils"; import { @@ -22,6 +23,17 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; +import { + describeMediaError, + describeSpeechError, + isMissingDeviceError, +} from "@/features/chat/adapters/studio-web-speech-dictation-adapter"; +import { + applyDictationDictionary, + recordRecentDictation, + resolveDictationLanguage, + useVoiceSettingsStore, +} from "@/features/settings/stores/voice-settings-store"; import { AUDIO_ACCEPT, MAX_AUDIO_SIZE, fileToBase64 } from "@/lib/audio-utils"; import { isTauri } from "@/lib/api-base"; import { isMultimodalResponse } from "./types/api"; @@ -148,18 +160,6 @@ const ArrowDownStandardIcon: FC<{ className?: string }> = ({ className }) => ( ); -const MicIcon: FC<{ className?: string }> = ({ className }) => ( - - - -); - function isNativeComposing(event: Event) { return "isComposing" in event && (event as InputEvent).isComposing === true; } @@ -214,7 +214,17 @@ function useDictation( const [isDictating, setIsDictating] = useState(false); const recognitionRef = useRef(null); - const start = useCallback(() => { + const streamRef = useRef(null); + const startingRef = useRef(false); + // Guards the getUserMedia await so a mic opened after unmount is released. + const disposedRef = useRef(false); + + const releaseStream = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + }, []); + + const start = useCallback(async () => { const SpeechRecognitionAPI = typeof window !== "undefined" && (window.SpeechRecognition ?? @@ -226,43 +236,136 @@ function useDictation( if (!SpeechRecognitionAPI) { return; } + if (startingRef.current || recognitionRef.current) return; + startingRef.current = true; + + // Open the microphone chosen in Voice settings, matching the main chat + // adapter, so Compare dictation honors the same device selection. + let audioTrack: MediaStreamTrack | undefined; + const { micDeviceId } = useVoiceSettingsStore.getState(); + if (navigator.mediaDevices?.getUserMedia) { + try { + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + micDeviceId && micDeviceId !== "default" + ? { deviceId: { exact: micDeviceId } } + : true, + }); + } catch (error) { + // Saved mic may be unplugged; fall back to the default device. + if (micDeviceId !== "default" && isMissingDeviceError(error)) { + stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + }); + } else { + throw error; + } + } + streamRef.current = stream; + audioTrack = stream.getAudioTracks()[0]; + } catch (error) { + // Permission/security failure: report it and stop instead of silently + // recording from a different default device, matching the main adapter. + startingRef.current = false; + releaseStream(); + setIsDictating(false); + toast.error(describeMediaError(error)); + return; + } + } + + if (disposedRef.current) { + releaseStream(); + startingRef.current = false; + return; + } + const recognition = new SpeechRecognitionAPI() as SpeechRecognition; recognition.continuous = true; recognition.interimResults = true; - recognition.lang = "en-US"; + recognition.lang = resolveDictationLanguage(); + let sessionTranscript = ""; recognition.onresult = (event: SpeechRecognitionEvent) => { - const last = event.resultIndex; - const result = event.results[last]; - if (!result?.isFinal) return; - const transcript = result[0]?.transcript?.trim(); - if (transcript) { + // Iterate every result from resultIndex; a single event can carry more + // than one finalized phrase and dropping the rest loses dictated words. + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i]; + if (!result?.isFinal) continue; + const transcript = applyDictationDictionary( + result[0]?.transcript?.trim() ?? "", + ); + if (!transcript) continue; + sessionTranscript = sessionTranscript + ? `${sessionTranscript} ${transcript}` + : transcript; setText((prev) => (prev ? `${prev} ${transcript}` : transcript)); } }; - recognition.onerror = () => { + recognition.onerror = (event) => { + // Report speech-service failures like the main adapter; aborted is a + // normal stop, not an error. + const errorEvent = event as SpeechRecognitionErrorEvent; + if (errorEvent.error !== "aborted") { + toast.error(describeSpeechError(errorEvent.error, errorEvent.message)); + } setIsDictating(false); }; recognition.onend = () => { - setIsDictating(false); + // A stop()+immediate restart can install a new recognizer before this + // old one ends; only tear down shared refs when we are still current. + if (recognitionRef.current === recognition) { + releaseStream(); + recognitionRef.current = null; + setIsDictating(false); + } + if (sessionTranscript) { + recordRecentDictation(sessionTranscript); + sessionTranscript = ""; + } }; - recognition.start(); + try { + if (audioTrack) { + try { + recognition.start(audioTrack); + } catch { + // No start(track) overload: recognition captures from the default + // device, so release the selected-device stream. + releaseStream(); + recognition.start(); + } + } else { + recognition.start(); + } + } catch { + startingRef.current = false; + releaseStream(); + return; + } recognitionRef.current = recognition; + startingRef.current = false; setIsDictating(true); - }, [setText]); + }, [setText, releaseStream]); const stop = useCallback(() => { if (recognitionRef.current) { recognitionRef.current.stop(); recognitionRef.current = null; } + releaseStream(); setIsDictating(false); - }, []); + }, [releaseStream]); useEffect(() => { + disposedRef.current = false; return () => { + disposedRef.current = true; if (recognitionRef.current) { recognitionRef.current.abort(); } + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; }; }, []); diff --git a/studio/frontend/src/features/settings/settings-dialog.tsx b/studio/frontend/src/features/settings/settings-dialog.tsx index d2f7c092f6..7625449648 100644 --- a/studio/frontend/src/features/settings/settings-dialog.tsx +++ b/studio/frontend/src/features/settings/settings-dialog.tsx @@ -10,6 +10,7 @@ import { } from "@/components/ui/dialog"; import { type TranslationKey, useT } from "@/i18n"; import { cn } from "@/lib/utils"; +import { MicIcon } from "@/lib/mic-icon"; import { Cancel01Icon, CloudIcon, @@ -24,7 +25,14 @@ import { } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { motion, useReducedMotion } from "motion/react"; -import { useDeferredValue, useEffect, useMemo, useRef, useState } from "react"; +import { + type FC, + useDeferredValue, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { SETTINGS_SEARCH_INDEX } from "./settings-search"; import { type SettingsTab, @@ -38,11 +46,14 @@ import { ConnectionsTab } from "./tabs/connections-tab"; import { GeneralTab } from "./tabs/general-tab"; import { ProfileTab } from "./tabs/profile-tab"; import { ResourcesTab } from "./tabs/resources-tab"; +import { VoiceTab } from "./tabs/voice-tab"; interface TabDef { id: SettingsTab; labelKey: TranslationKey; - icon: typeof Settings02Icon; + icon?: typeof Settings02Icon; + /** Plain component icon, for icons shared with chat (not hugeicons). */ + iconComponent?: FC<{ className?: string }>; badgeKey?: TranslationKey; } @@ -76,6 +87,12 @@ const TABS: TabDef[] = [ labelKey: "settings.tabs.connections", icon: CloudIcon, }, + { + id: "voice", + labelKey: "settings.tabs.voice", + iconComponent: MicIcon, + badgeKey: "common.new", + }, { id: "about", labelKey: "settings.tabs.about", icon: HelpCircleIcon }, ]; @@ -91,6 +108,8 @@ function renderTab(tab: SettingsTab) { return ; case "chat": return ; + case "voice": + return ; case "connections": return ; case "api-keys": @@ -189,6 +208,7 @@ export function SettingsDialog() { appearance: null, resources: null, chat: null, + voice: null, connections: null, "api-keys": null, about: null, @@ -279,11 +299,15 @@ export function SettingsDialog() { onClick={() => openResult(tab.id)} className="flex h-[30px] items-center gap-2.5 rounded-full pl-3 pr-2.5 text-[13.5px] font-medium text-muted-foreground transition-colors hover:bg-accent hover:text-accent-foreground" > - + {tab.iconComponent ? ( + + ) : tab.icon ? ( + + ) : null} {tabLabel} {entries.map((entry) => ( @@ -352,11 +376,15 @@ export function SettingsDialog() { } /> )} - + {tab.iconComponent ? ( + + ) : tab.icon ? ( + + ) : null} {t(tab.labelKey)} diff --git a/studio/frontend/src/features/settings/settings-search.ts b/studio/frontend/src/features/settings/settings-search.ts index 36191835e5..786ee11d0a 100644 --- a/studio/frontend/src/features/settings/settings-search.ts +++ b/studio/frontend/src/features/settings/settings-search.ts @@ -98,6 +98,22 @@ export const SETTINGS_SEARCH_INDEX: Record = { "settings.apiKeys.accessTokens", ], connections: [], + voice: [ + "settings.voice.dictation.sectionTitle", + "settings.voice.dictation.microphoneLabel", + "settings.voice.dictation.languageLabel", + "settings.voice.dictation.testLabel", + "settings.voice.dictionary.sectionTitle", + "settings.voice.recents.sectionTitle", + "settings.voice.readAloud.sectionTitle", + "settings.voice.readAloud.buttonLabel", + "settings.voice.readAloud.engineLabel", + "settings.voice.readAloud.voiceLabel", + "settings.voice.readAloud.speedLabel", + "settings.voice.readAloud.pitchLabel", + "settings.voice.readAloud.volumeLabel", + "settings.voice.readAloud.previewLabel", + ], about: [ "settings.about.updates", "settings.about.releaseNotes", diff --git a/studio/frontend/src/features/settings/stores/settings-dialog-store.ts b/studio/frontend/src/features/settings/stores/settings-dialog-store.ts index 234e92b3d0..b9e9c75b14 100644 --- a/studio/frontend/src/features/settings/stores/settings-dialog-store.ts +++ b/studio/frontend/src/features/settings/stores/settings-dialog-store.ts @@ -9,6 +9,7 @@ export type SettingsTab = | "appearance" | "resources" | "chat" + | "voice" | "connections" | "api-keys" | "about"; @@ -63,6 +64,7 @@ function loadInitialTab(): SettingsTab { "appearance", "resources", "chat", + "voice", "connections", "api-keys", "about", diff --git a/studio/frontend/src/features/settings/stores/voice-settings-store.ts b/studio/frontend/src/features/settings/stores/voice-settings-store.ts new file mode 100644 index 0000000000..9e38f4c6ce --- /dev/null +++ b/studio/frontend/src/features/settings/stores/voice-settings-store.ts @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +// Voice preferences in localStorage. Adapters read them at call time so +// changes apply without reloading the chat runtime. + +export interface RecentDictation { + text: string; + at: number; +} + +const MAX_RECENT_DICTATIONS = 20; +// Cap stored transcript length so a few long dictations cannot bloat the +// persisted blob and trip a synchronous localStorage quota error on save. +const MAX_RECENT_DICTATION_LENGTH = 2000; +const MAX_DICTIONARY_ENTRIES = 100; +const MAX_DICTIONARY_ENTRY_LENGTH = 120; + +export interface VoiceSettingsState { + /** Input device for dictation. "default" = system default microphone. */ + micDeviceId: string; + setMicDeviceId: (value: string) => void; + + /** BCP 47 tag for speech recognition, or "auto" for the browser locale. */ + dictationLanguage: string; + setDictationLanguage: (value: string) => void; + + /** Exact spellings applied to matching transcript words and phrases. */ + dictionary: string[]; + addDictionaryEntry: (value: string) => void; + updateDictionaryEntry: (index: number, value: string) => void; + /** Trim the entry; drop it when it was left empty. Call on input blur. */ + commitDictionaryEntry: (index: number) => void; + removeDictionaryEntry: (index: number) => void; + + /** Final transcripts, newest first, so text can be recovered. */ + recentDictations: RecentDictation[]; + addRecentDictation: (text: string) => void; + clearRecentDictations: () => void; + + /** Show the read-aloud button on assistant responses. */ + ttsEnabled: boolean; + setTtsEnabled: (value: boolean) => void; + + /** "system": speechSynthesis voices. "studio": the loaded TTS audio model. */ + ttsEngine: "system" | "studio"; + setTtsEngine: (value: "system" | "studio") => void; + + /** speechSynthesis voiceURI, or "default" for the system voice. */ + ttsVoiceURI: string; + setTtsVoiceURI: (value: string) => void; + + ttsRate: number; + setTtsRate: (value: number) => void; + ttsPitch: number; + setTtsPitch: (value: number) => void; + ttsVolume: number; + setTtsVolume: (value: number) => void; +} + +export const useVoiceSettingsStore = create()( + persist( + (set) => ({ + micDeviceId: "default", + setMicDeviceId: (micDeviceId) => set({ micDeviceId }), + + dictationLanguage: "auto", + setDictationLanguage: (dictationLanguage) => set({ dictationLanguage }), + + dictionary: [], + addDictionaryEntry: (value) => + set((state) => { + const trimmed = value.trim().slice(0, MAX_DICTIONARY_ENTRY_LENGTH); + if (!trimmed) return state; + if (state.dictionary.length >= MAX_DICTIONARY_ENTRIES) return state; + if ( + state.dictionary.some( + (entry) => entry.toLowerCase() === trimmed.toLowerCase(), + ) + ) { + return state; + } + return { dictionary: [...state.dictionary, trimmed] }; + }), + // Keep the raw value so the input edits freely; commitDictionaryEntry finalizes on blur. + updateDictionaryEntry: (index, value) => + set((state) => { + const dictionary = [...state.dictionary]; + if (index < 0 || index >= dictionary.length) return state; + dictionary[index] = value.slice(0, MAX_DICTIONARY_ENTRY_LENGTH); + return { dictionary }; + }), + commitDictionaryEntry: (index) => + set((state) => { + const dictionary = [...state.dictionary]; + if (index < 0 || index >= dictionary.length) return state; + const trimmed = dictionary[index]?.trim() ?? ""; + if (trimmed) { + dictionary[index] = trimmed; + } else { + dictionary.splice(index, 1); + } + return { dictionary }; + }), + removeDictionaryEntry: (index) => + set((state) => ({ + dictionary: state.dictionary.filter((_, i) => i !== index), + })), + + recentDictations: [], + addRecentDictation: (text) => + set((state) => { + const trimmed = text.trim().slice(0, MAX_RECENT_DICTATION_LENGTH); + if (!trimmed) return state; + return { + recentDictations: [ + { text: trimmed, at: Date.now() }, + ...state.recentDictations, + ].slice(0, MAX_RECENT_DICTATIONS), + }; + }), + clearRecentDictations: () => set({ recentDictations: [] }), + + ttsEnabled: true, + setTtsEnabled: (ttsEnabled) => set({ ttsEnabled }), + + ttsEngine: "system", + setTtsEngine: (ttsEngine) => set({ ttsEngine }), + + ttsVoiceURI: "default", + setTtsVoiceURI: (ttsVoiceURI) => set({ ttsVoiceURI }), + + ttsRate: 1, + setTtsRate: (ttsRate) => set({ ttsRate }), + ttsPitch: 1, + setTtsPitch: (ttsPitch) => set({ ttsPitch }), + ttsVolume: 1, + setTtsVolume: (ttsVolume) => set({ ttsVolume }), + }), + { + name: "unsloth_voice_settings", + merge: (persisted, current) => { + const saved = persisted as Partial | undefined; + return { + ...current, + micDeviceId: asString(saved?.micDeviceId, "default"), + dictationLanguage: asString(saved?.dictationLanguage, "auto"), + dictionary: Array.isArray(saved?.dictionary) + ? saved.dictionary + .filter((v): v is string => typeof v === "string" && !!v.trim()) + .map((v) => v.trim().slice(0, MAX_DICTIONARY_ENTRY_LENGTH)) + .slice(0, MAX_DICTIONARY_ENTRIES) + : [], + recentDictations: Array.isArray(saved?.recentDictations) + ? saved.recentDictations + .filter( + (v): v is RecentDictation => + typeof v?.text === "string" && typeof v?.at === "number", + ) + .slice(0, MAX_RECENT_DICTATIONS) + .map((v) => ({ + text: v.text.slice(0, MAX_RECENT_DICTATION_LENGTH), + at: v.at, + })) + : [], + ttsEnabled: + typeof saved?.ttsEnabled === "boolean" ? saved.ttsEnabled : true, + ttsEngine: saved?.ttsEngine === "studio" ? "studio" : "system", + ttsVoiceURI: asString(saved?.ttsVoiceURI, "default"), + ttsRate: clampNumber(saved?.ttsRate, 0.5, 2, 1), + ttsPitch: clampNumber(saved?.ttsPitch, 0, 2, 1), + ttsVolume: clampNumber(saved?.ttsVolume, 0, 1, 1), + }; + }, + }, + ), +); + +function asString(value: unknown, fallback: string): string { + return typeof value === "string" && value ? value : fallback; +} + +function clampNumber( + value: unknown, + min: number, + max: number, + fallback: number, +): number { + if (typeof value !== "number" || Number.isNaN(value)) return fallback; + return Math.min(max, Math.max(min, value)); +} + +/** Resolve the "auto" language setting to a concrete BCP 47 tag. */ +export function resolveDictationLanguage(setting?: string): string { + const value = setting ?? useVoiceSettingsStore.getState().dictationLanguage; + if (value && value !== "auto") return value; + return typeof navigator !== "undefined" && navigator.language + ? navigator.language + : "en-US"; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Rewrite dictionary phrases in a transcript to their exact stored form, + * matching case-insensitively on word boundaries ("jane doe" -> "Jane Doe"). + */ +export function applyDictationDictionary( + transcript: string, + dictionary?: string[], +): string { + const entries = dictionary ?? useVoiceSettingsStore.getState().dictionary; + if (!transcript || entries.length === 0) return transcript; + let result = transcript; + for (const entry of entries) { + const trimmed = entry.trim(); + if (!trimmed) continue; + // Whitespace-tolerant pattern so "jane doe" still matches. + const pattern = trimmed.split(/\s+/).map(escapeRegExp).join("\\s+"); + try { + // Capture the leading boundary instead of using a lookbehind, which + // engines that support dictation but not lookbehind (Safari < 16.4) + // cannot compile; the catch below would otherwise skip every entry. + const regex = new RegExp( + `(^|[^\\p{L}\\p{N}])(${pattern})(?![\\p{L}\\p{N}])`, + "giu", + ); + // Re-emit the boundary; callback form avoids $-pattern expansion. + result = result.replace(regex, (_match, prefix) => `${prefix}${trimmed}`); + } catch { + // Skip entries that produce an invalid pattern. + } + } + return result; +} + +/** Record a finished dictation so it can be recovered from settings. */ +export function recordRecentDictation(text: string): void { + useVoiceSettingsStore.getState().addRecentDictation(text); +} diff --git a/studio/frontend/src/features/settings/tabs/general-tab.tsx b/studio/frontend/src/features/settings/tabs/general-tab.tsx index 1b2adf066d..1460c7dea2 100644 --- a/studio/frontend/src/features/settings/tabs/general-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/general-tab.tsx @@ -114,6 +114,8 @@ const PREFS_KEYS: string[] = [ // Update notifications "unsloth_show_llama_update_banner", "unsloth_monitor_overlay", + // Voice settings + "unsloth_voice_settings", ]; // Set by resetAllPrefs so the unmount-commit effect skips writing back the diff --git a/studio/frontend/src/features/settings/tabs/voice-tab.tsx b/studio/frontend/src/features/settings/tabs/voice-tab.tsx new file mode 100644 index 0000000000..4b86105926 --- /dev/null +++ b/studio/frontend/src/features/settings/tabs/voice-tab.tsx @@ -0,0 +1,882 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Slider } from "@/components/ui/slider"; +import { Switch } from "@/components/ui/switch"; +import { + StudioSpeechSynthesisAdapter, + createConfiguredUtterance, + curateSystemVoices, + generateStudioTtsAudio, +} from "@/features/chat/adapters/studio-speech-synthesis-adapter"; +import { + StudioWebSpeechDictationAdapter, + describeSpeechError, + isMissingDeviceError, +} from "@/features/chat/adapters/studio-web-speech-dictation-adapter"; +import { useT } from "@/i18n"; +import { toast } from "@/lib/toast"; +import { MicIcon } from "@/lib/mic-icon"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { + Copy01Icon, + Delete02Icon, + PlusSignIcon, + VolumeHighIcon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { SquareIcon } from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { SettingsRow } from "../components/settings-row"; +import { SettingsSection } from "../components/settings-section"; +import { + applyDictationDictionary, + recordRecentDictation, + resolveDictationLanguage, + useVoiceSettingsStore, +} from "../stores/voice-settings-store"; + +// Languages offered for browser speech recognition. +const DICTATION_LANGUAGES: { value: string; label: string }[] = [ + { value: "auto", label: "" }, // label rendered via i18n + { value: "en-US", label: "English (US)" }, + { value: "en-GB", label: "English (UK)" }, + { value: "zh-CN", label: "中文 (简体)" }, + { value: "ja-JP", label: "日本語" }, + { value: "ko-KR", label: "한국어" }, + { value: "es-ES", label: "Español" }, + { value: "fr-FR", label: "Français" }, + { value: "de-DE", label: "Deutsch" }, + { value: "it-IT", label: "Italiano" }, + { value: "pt-BR", label: "Português (Brasil)" }, + { value: "ru-RU", label: "Русский" }, + { value: "hi-IN", label: "हिन्दी" }, + { value: "ar-SA", label: "العربية" }, +]; + +const TTS_PREVIEW_TEXT = + "Hello from Unsloth Studio! This is a preview of the selected voice."; + +function useAudioInputDevices() { + const t = useT(); + const [devices, setDevices] = useState([]); + const [hasLabels, setHasLabels] = useState(false); + + const refresh = useCallback(async () => { + if (!navigator.mediaDevices?.enumerateDevices) return; + try { + const all = await navigator.mediaDevices.enumerateDevices(); + const inputs = all.filter((d) => d.kind === "audioinput"); + setDevices(inputs); + setHasLabels(inputs.some((d) => d.label)); + } catch { + // Enumeration can fail in insecure contexts; leave the list empty. + } + }, []); + + useEffect(() => { + void refresh(); + const media = navigator.mediaDevices; + if (!media?.addEventListener) return; + media.addEventListener("devicechange", refresh); + return () => media.removeEventListener("devicechange", refresh); + }, [refresh]); + + // Labels are hidden until mic permission; open a short stream to get them. + const requestAccess = useCallback(async () => { + // Insecure contexts (plain http on a LAN address) have no mediaDevices. + if (!navigator.mediaDevices?.getUserMedia) { + toast.error(t("settings.voice.dictation.micAccessUnsupported")); + return; + } + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + }); + stream.getTracks().forEach((track) => track.stop()); + await refresh(); + } catch { + toast.error(t("settings.voice.dictation.micAccessBlocked")); + } + }, [refresh, t]); + + return { devices, hasLabels, requestAccess }; +} + +function useSystemVoices() { + const [voices, setVoices] = useState([]); + + useEffect(() => { + if (typeof window === "undefined" || !window.speechSynthesis) return; + const synth = window.speechSynthesis; + const load = () => setVoices(synth.getVoices()); + load(); + synth.addEventListener?.("voiceschanged", load); + return () => synth.removeEventListener?.("voiceschanged", load); + }, []); + + return voices; +} + +/** Inline mic test: runs speech recognition and shows the live transcript. */ +function DictationTest() { + const t = useT(); + const [testing, setTesting] = useState(false); + const [transcript, setTranscript] = useState(""); + const [interim, setInterim] = useState(""); + const recognitionRef = useRef(null); + const streamRef = useRef(null); + // Guards the getUserMedia await so a mic opened after unmount is released. + const disposedRef = useRef(false); + // Mirrors the transcript state so onend can record it without stale closures. + const transcriptRef = useRef(""); + + // Single cleanup path: the browser can end recognition on its own (silence + // timeout, service disconnect), so onend must release the mic and save the + // transcript, not just the Stop button. + const finalize = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + recognitionRef.current = null; + if (transcriptRef.current) { + recordRecentDictation(transcriptRef.current); + transcriptRef.current = ""; + } + setTesting(false); + setInterim(""); + }, []); + + const stop = useCallback(() => { + const recognition = recognitionRef.current; + if (recognition) { + // onend fires next and runs finalize() + recognition.stop(); + } else { + finalize(); + } + }, [finalize]); + + useEffect(() => { + disposedRef.current = false; + return () => { + disposedRef.current = true; + recognitionRef.current?.abort(); + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + }; + }, []); + + // Set before the getUserMedia await so a double click or a slow + // permission prompt cannot start a second recognizer over the first. + const startingRef = useRef(false); + + const start = useCallback(async () => { + const SpeechRecognitionAPI = + window.SpeechRecognition ?? window.webkitSpeechRecognition; + if (!SpeechRecognitionAPI) return; + if (startingRef.current || recognitionRef.current) return; + startingRef.current = true; + setTranscript(""); + setInterim(""); + transcriptRef.current = ""; + + const { micDeviceId } = useVoiceSettingsStore.getState(); + let audioTrack: MediaStreamTrack | undefined; + try { + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + micDeviceId && micDeviceId !== "default" + ? { deviceId: { exact: micDeviceId } } + : true, + }); + } catch (error) { + // Saved mic may be unplugged; fall back to the default device. + if (micDeviceId !== "default" && isMissingDeviceError(error)) { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } else { + throw error; + } + } + if (disposedRef.current) { + stream.getTracks().forEach((track) => track.stop()); + startingRef.current = false; + return; + } + streamRef.current = stream; + audioTrack = stream.getAudioTracks()[0]; + } catch { + startingRef.current = false; + toast.error(t("settings.voice.dictation.micOpenFailed")); + return; + } + + const recognition = new SpeechRecognitionAPI(); + recognition.lang = resolveDictationLanguage(); + recognition.continuous = true; + recognition.interimResults = true; + recognition.onresult = (event: SpeechRecognitionEvent) => { + let interimText = ""; + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i]; + const text = result?.[0]?.transcript ?? ""; + if (result?.isFinal) { + const corrected = applyDictationDictionary(text.trim()); + setTranscript((prev) => { + const next = prev ? `${prev} ${corrected}` : corrected; + transcriptRef.current = next; + return next; + }); + } else { + interimText += text; + } + } + setInterim(interimText); + }; + recognition.onerror = (event) => { + // onend follows and runs finalize(); surface non-abort failures here. + const errorEvent = event as SpeechRecognitionErrorEvent; + if (errorEvent.error !== "aborted") { + toast.error(describeSpeechError(errorEvent.error, errorEvent.message)); + } + }; + recognition.onend = () => finalize(); + try { + if (audioTrack) { + try { + recognition.start(audioTrack); + } catch { + // Engine has no start(track) overload: it will capture from the + // default device, so release the selected-device stream. + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + recognition.start(); + } + } else { + recognition.start(); + } + } catch { + startingRef.current = false; + finalize(); + return; + } + recognitionRef.current = recognition; + startingRef.current = false; + setTesting(true); + }, [finalize, t]); + + const finishedTest = !testing && transcript; + + return ( +
+ + + + {(testing || transcript) && ( +
+ {transcript || interim ? ( + <> + {transcript} + {interim ? ( + {interim} + ) : null} + + ) : ( + + {testing ? t("settings.voice.dictation.listening") : ""} + + )} + {finishedTest ? ( +
+ {t("settings.voice.dictation.testSaved")} +
+ ) : null} +
+ )} +
+ ); +} + +export function VoiceTab() { + const t = useT(); + const micDeviceId = useVoiceSettingsStore((s) => s.micDeviceId); + const setMicDeviceId = useVoiceSettingsStore((s) => s.setMicDeviceId); + const dictationLanguage = useVoiceSettingsStore((s) => s.dictationLanguage); + const setDictationLanguage = useVoiceSettingsStore( + (s) => s.setDictationLanguage, + ); + const dictionary = useVoiceSettingsStore((s) => s.dictionary); + const addDictionaryEntry = useVoiceSettingsStore((s) => s.addDictionaryEntry); + const updateDictionaryEntry = useVoiceSettingsStore( + (s) => s.updateDictionaryEntry, + ); + const commitDictionaryEntry = useVoiceSettingsStore( + (s) => s.commitDictionaryEntry, + ); + const removeDictionaryEntry = useVoiceSettingsStore( + (s) => s.removeDictionaryEntry, + ); + const recentDictations = useVoiceSettingsStore((s) => s.recentDictations); + const clearRecentDictations = useVoiceSettingsStore( + (s) => s.clearRecentDictations, + ); + const ttsEnabled = useVoiceSettingsStore((s) => s.ttsEnabled); + const setTtsEnabled = useVoiceSettingsStore((s) => s.setTtsEnabled); + const ttsEngine = useVoiceSettingsStore((s) => s.ttsEngine); + const setTtsEngine = useVoiceSettingsStore((s) => s.setTtsEngine); + const ttsVoiceURI = useVoiceSettingsStore((s) => s.ttsVoiceURI); + const setTtsVoiceURI = useVoiceSettingsStore((s) => s.setTtsVoiceURI); + const ttsRate = useVoiceSettingsStore((s) => s.ttsRate); + const setTtsRate = useVoiceSettingsStore((s) => s.setTtsRate); + const ttsPitch = useVoiceSettingsStore((s) => s.ttsPitch); + const setTtsPitch = useVoiceSettingsStore((s) => s.setTtsPitch); + const ttsVolume = useVoiceSettingsStore((s) => s.ttsVolume); + const setTtsVolume = useVoiceSettingsStore((s) => s.setTtsVolume); + + const { devices, hasLabels, requestAccess } = useAudioInputDevices(); + const rawVoices = useSystemVoices(); + const voices = useMemo( + () => curateSystemVoices(rawVoices, ttsVoiceURI), + // dictationLanguage feeds the curation language filter. + [rawVoices, ttsVoiceURI, dictationLanguage], + ); + const [newEntry, setNewEntry] = useState(""); + const [previewing, setPreviewing] = useState(false); + + const dictationSupported = StudioWebSpeechDictationAdapter.isSupported(); + const ttsSupported = StudioSpeechSynthesisAdapter.isSupported(); + const systemTtsSupported = + StudioSpeechSynthesisAdapter.systemVoicesSupported(); + const effectiveTtsEngine = systemTtsSupported ? ttsEngine : "studio"; + + // Keep an item for an unplugged saved mic so the value stays visible. + const knownMic = devices.some((d) => d.deviceId === micDeviceId); + + const handleAddEntry = () => { + const trimmed = newEntry.trim(); + if (!trimmed) return; + addDictionaryEntry(trimmed); + setNewEntry(""); + }; + + const previewAudioRef = useRef(null); + const previewAbortRef = useRef(null); + // Mirrors `previewing` so unmount cleanup can tell whether this tab owns + // the current speechSynthesis utterance; read-aloud shares the global + // synthesizer and must not be cancelled by merely closing settings. + const previewingRef = useRef(false); + // Only a system-voice preview owns the shared speechSynthesis channel; a + // studio (Audio) preview must not cancel an unrelated chat read-aloud. + const ownsSystemPreviewRef = useRef(false); + const markPreviewing = useCallback((value: boolean) => { + previewingRef.current = value; + setPreviewing(value); + }, []); + + const releasePreviewAudio = useCallback(() => { + if (previewAudioRef.current) { + previewAudioRef.current.pause(); + previewAudioRef.current.src = ""; + previewAudioRef.current = null; + } + }, []); + + const stopPreview = useCallback(() => { + if (!previewingRef.current) return; + if (ownsSystemPreviewRef.current) { + window.speechSynthesis?.cancel(); + ownsSystemPreviewRef.current = false; + } + previewAbortRef.current?.abort(); + previewAbortRef.current = null; + releasePreviewAudio(); + markPreviewing(false); + }, [markPreviewing, releasePreviewAudio]); + + const previewTts = async () => { + if (!ttsSupported) return; + // Ref, not state: a double-click before rerender still reads previewing + // as false and would start a second request that orphans the first. + if (previewingRef.current) { + stopPreview(); + return; + } + if (effectiveTtsEngine === "studio") { + const controller = new AbortController(); + previewAbortRef.current = controller; + ownsSystemPreviewRef.current = false; + markPreviewing(true); + try { + const url = await generateStudioTtsAudio( + TTS_PREVIEW_TEXT, + controller.signal, + ); + if (controller.signal.aborted) return; + const audio = new Audio(url); + audio.playbackRate = ttsRate; + audio.volume = ttsVolume; + // Some browsers reset playbackRate to 1 once the source loads; reapply + // it on loadedmetadata so the speed setting reliably takes effect. + audio.addEventListener("loadedmetadata", () => { + audio.playbackRate = ttsRate; + }); + audio.addEventListener("ended", () => { + releasePreviewAudio(); + markPreviewing(false); + }); + audio.addEventListener("error", () => { + releasePreviewAudio(); + markPreviewing(false); + // Surface playback failures like the catch below, instead of just + // resetting the button with no explanation. + toast.error("TTS preview failed"); + }); + previewAudioRef.current = audio; + await audio.play(); + } catch (error) { + if (!controller.signal.aborted) { + toast.error( + error instanceof Error ? error.message : "TTS preview failed", + ); + } + releasePreviewAudio(); + markPreviewing(false); + } + return; + } + if (!StudioSpeechSynthesisAdapter.systemVoicesSupported()) { + toast.error(t("settings.voice.readAloud.notSupported")); + return; + } + const utterance = createConfiguredUtterance(TTS_PREVIEW_TEXT); + utterance.addEventListener("end", () => { + ownsSystemPreviewRef.current = false; + markPreviewing(false); + }); + utterance.addEventListener("error", () => { + ownsSystemPreviewRef.current = false; + markPreviewing(false); + }); + ownsSystemPreviewRef.current = true; + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + markPreviewing(true); + }; + + // Stop any preview playback when the tab unmounts. + useEffect(() => stopPreview, [stopPreview]); + + return ( +
+
+

+ {t("settings.voice.title")} +

+

+ {t("settings.voice.description")} +

+
+ + + + {hasLabels ? ( + + ) : ( + + )} + + + + + + + {dictationSupported ? ( + + ) : ( + + )} + + + + {dictionary.map((entry, index) => ( +
+ updateDictionaryEntry(index, e.target.value)} + // Skip the empty-row commit-splice when focus moves to this row's + // Remove button (keyboard Tab), so its index stays valid and its + // activation deletes this row instead of the next one. + onBlur={(e) => { + if ( + (e.relatedTarget as HTMLElement | null)?.dataset.dictRemove === + String(index) + ) { + return; + } + commitDictionaryEntry(index); + }} + className="h-8 flex-1 text-sm" + aria-label={`Dictionary entry ${index + 1}`} + /> + +
+ ))} +
+ setNewEntry(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + handleAddEntry(); + } + }} + placeholder="Jane Doe" + className="h-8 flex-1 text-sm" + aria-label="New dictionary entry" + /> + +
+
+ + + {recentDictations.length === 0 ? ( +

+ {t("settings.voice.recents.empty")} +

+ ) : ( + <> + {recentDictations.map((item) => ( +
+
+

+ {item.text} +

+

+ {new Date(item.at).toLocaleString()} +

+
+ +
+ ))} +
+ +
+ + )} +
+ + + {ttsSupported ? ( + <> + + + + + + + + + {effectiveTtsEngine === "studio" ? ( + + ) : ( + + + + )} + + + v !== undefined && setTtsRate(v)} + className="w-48" + aria-label="Speaking rate" + /> + + + {effectiveTtsEngine === "system" && ( + + v !== undefined && setTtsPitch(v)} + className="w-48" + aria-label="Voice pitch" + /> + + )} + + + v !== undefined && setTtsVolume(v)} + className="w-48" + aria-label="Playback volume" + /> + + + + + + + ) : ( + + )} + +
+ ); +} diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index 3e351c22bc..fe3a6f8542 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -97,10 +97,81 @@ export const en = { appearance: "Appearance", resources: "System", chat: "Chat", + voice: "Voice", connections: "Connections", apiKeys: "API", about: "About", }, + voice: { + title: "Voice", + description: "Microphone, dictation, and read-aloud", + dictation: { + sectionTitle: "Dictation", + microphoneLabel: "Microphone", + microphoneDescription: "Used for dictation", + microphoneFallbackHint: + "Used for dictation. Falls back to the system default if the browser speech engine cannot use this device", + microphoneGrantDescription: "Allow mic access to show device names", + allowMicrophone: "Allow microphone", + micAccessBlocked: + "Microphone access was blocked. Allow microphone access for this Unsloth page, then try again.", + micAccessUnsupported: + "Microphone access is not supported in this browser or context.", + micOpenFailed: + "Could not open the selected microphone. Check permissions or pick another device.", + systemDefault: "System default", + savedMicDisconnected: "Saved microphone (not connected)", + languageLabel: "Dictation language", + languageDescription: "Language to recognize", + languageAuto: "Auto (browser language)", + testLabel: "Test dictation", + testDescription: "Speak to check your mic and settings", + startTest: "Start test", + stopTest: "Stop test", + listening: "Listening…", + testSaved: "Saved to recent dictations", + notSupported: "Not supported in this browser", + }, + dictionary: { + sectionTitle: "Dictation dictionary", + sectionDescription: + "Apply the spelling entered here when dictation recognizes the same words or phrase", + addEntry: "Add entry", + }, + recents: { + sectionTitle: "Recent dictations", + sectionDescription: + "Your recent dictations will appear here so you can recover text", + empty: "No dictations yet", + copied: "Copied to clipboard", + copyFailed: "Could not copy to clipboard", + clear: "Clear recent dictations", + }, + readAloud: { + sectionTitle: "Read aloud", + buttonLabel: "Read aloud button", + buttonDescription: "Show on assistant responses", + engineLabel: "TTS engine", + engineSystemDescription: "Built-in device voices", + engineStudioDescription: "Uses the loaded audio model (e.g. Orpheus)", + engineSystem: "System voices", + engineStudio: "Load TTS model", + modelLabel: "TTS model", + modelDescription: + "Load an audio model from the model selector (e.g. Orpheus TTS)", + voiceLabel: "Voice", + voiceDescription: "Best voices on this device", + speedLabel: "Speed", + pitchLabel: "Pitch", + volumeLabel: "Volume", + previewLabel: "Preview voice", + previewDescription: "Play a short sample", + previewAction: "Preview", + stopAction: "Stop", + ttsLabel: "Text to speech", + notSupported: "Not supported in this browser", + }, + }, general: { title: "General", description: "Global preferences for Unsloth.", @@ -549,7 +620,8 @@ export const en = { codingAgents: "Coding agents", codingAgentsHint: "Launch a coding agent against this server. It uses the loaded model; a local server mints an API key automatically, a remote one includes it in the command.", - codingAgentsSwap: "Swap claude for codex, openclaw, opencode, hermes, or pi.", + codingAgentsSwap: + "Swap claude for codex, openclaw, opencode, hermes, or pi.", codingAgentDetected: "Installed on this machine", codingAgentsDetectedHint: "Detected on this machine: {agents}.", relativeNever: "never", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index 44bb617512..23752b6c26 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -104,6 +104,7 @@ export const ja = { connections: "接続", apiKeys: "API", about: "情報", + voice: "音声", }, general: { title: "一般", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index d689b96428..07832ef1d0 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -103,6 +103,7 @@ export const ptBR = { connections: "Conexões", apiKeys: "API", about: "Sobre", + voice: "Voz", }, general: { title: "Geral", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index fc4e126d72..4c51755244 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -103,6 +103,7 @@ export const zhCN = { connections: "连接", apiKeys: "API", about: "关于", + voice: "语音", }, general: { title: "通用", diff --git a/studio/frontend/src/lib/mic-icon.tsx b/studio/frontend/src/lib/mic-icon.tsx new file mode 100644 index 0000000000..9ff6817845 --- /dev/null +++ b/studio/frontend/src/lib/mic-icon.tsx @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import type { FC } from "react"; + +/** Microphone icon used by the chat composer and the Voice settings tab. */ +export const MicIcon: FC<{ className?: string }> = ({ className }) => ( + + + +); From 73af334d1143102776c04738c2af2600bf07fea4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 15 Jul 2026 08:41:00 -0700 Subject: [PATCH 011/297] Studio: stream live tool output with SSE heartbeats, fix web page extraction, and surface interrupted turns (#7083) * Studio: stream live tool output with SSE heartbeats and fix web page extraction Server-side python/terminal tools now stream incremental stdout to the chat UI while running (new tool_output SSE event), and every blocking tool execution emits heartbeat keepalives so reverse proxies (Cloudflare tunnels cap idle streams at ~100s) cannot drop the connection mid-turn. The tool loop routes also emit a stall keepalive during silent prompt prefill between tool iterations. The final role=tool message the model sees is byte-identical to before, so tool-call parsing, nudging, and healing are untouched. web_search page fetches now extract main content: GitHub repo root pages are rewritten to the README API (with HTML fallback), hidden/aria-hidden client error placeholders are dropped, conversion scopes to article/main, and known boilerplate fragments are stripped. Non-HTML responses are returned raw instead of being run through the HTML converter. The frontend renders live-scrolling tool output inside running python and terminal cards, and a chat stream that ends without a terminal signal now surfaces an explicit interrupted state with a Retry action instead of silently ending the turn. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: fix content-type sniffing, unlimited-timeout drain, and env parity in tool streaming Content-Type sniffing: get_content_type() defaults to text/plain when the header is absent, so the sniffing fallback never fired and header-less HTML came back as raw markup. Report an empty type for a missing header and sniff the body whenever the declared type is not HTML, so mislabeled text/plain HTML pages are converted like before the extraction change. Unlimited timeout drain: with tool_call_timeout disabled the old path used communicate(timeout=None) and waited for EOF, but the streaming drain capped the post-exit drain at a 5 second join, truncating output from a grandchild that holds stdout open. When timeout is None, drain until EOF or the cancel event fires; finite timeouts keep the bounded remaining-budget join. Env parity: drop the PYTHONUNBUFFERED=1 injection on the streaming path so the child invocation is byte-identical with and without streaming (the env var was model-visible via os.getenv). Live streaming granularity now depends on the child flushing; unflushed output arrives in ~8 KB chunks or at exit and the final result is unchanged, with SSE heartbeats covering the gaps. * Studio: stream tool-call arguments while the model writes them A model writing a large tool call (a full python game is minutes of generation) produced nothing on the stream: the structured path accumulated delta.tool_calls fragments silently after the provisional card, and the text path's DRAINING state consumed everything until stream end. The user saw a dead Running spinner while the model was in fact writing code, and the byte-silent SSE segment was also the window where proxies drop the connection. New tool_args SSE events stream the arguments as they generate. The structured path forwards each fragment once a provisional card exists (backlog first, so the card starts from the top of the call). The text path sniffs the drained call for an enabled tool name and streams the raw call text under the id the stream-end parser assigns its first call (call_0), so the final tool_start reconciles the same card; the sniff is gated on enabled names plus the provisional size floor, and prose or ordinary JSON answers never spawn a card. The safetensors loop streams the drained render_html call to its existing provisional card the same way. The chat adapter accumulates the raw stream per card and feeds a partial JSON parse (call envelopes and stringified arguments unwrapped) into the part's args, so the python and terminal cards render the code live and the render_html canvas builds while streaming; both cards now say Writing code / Writing command during this phase via useToolArgsStatus. Display only: the parser input, the executed call, and the conversation the model sees are byte-identical, covered by new loop-level tests for the structured path, the text path, and the no-tool JSON answer. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: keep full tool output visible past the model cap; heal /mnt/data habits Live testing surfaced two issues in the tool streaming UX. First, a long python stdout ended in '... (truncated' in the finished card: the model-visible result is capped by tools._truncate (_MAX_OUTPUT_CHARS, previously 8000 chars) to protect the context window, and the card rendered that capped text even though the live stream had already shown everything. The cap stays (raised to 16000, overridable via UNSLOTH_TOOL_RESULT_MAX_CHARS) but display and model concerns are now split: the adapter preserves the accumulated live stream on tool_end whenever it captured more than the final result, and the finished python/terminal cards prefer it. The live-stream ceiling rises from 16 KB to 400 KB (chunks batch per poll, so SSE stays cheap), and both the live pane and the finished card render only the last 2000 lines with a Show all control so a huge output cannot jank the DOM. The truncation notice now tells the model the user saw the full output and that written files persist in the working directory. The final result string remains byte-identical with and without streaming. Second, models trained on ChatGPT code-interpreter transcripts write to /mnt/data, which does not exist here (the sandbox CWD is a per-thread persistent dir). Three layers, all identical across streaming and non-streaming paths: the python/terminal tool descriptions gain one sentence saying to use relative paths in the persistent CWD; a failed execution whose output shows a missing-file error on a known code-interpreter prefix (/mnt/data, /mnt/outputs, /home/sandbox, /workspace) gets a model-visible retry hint appended after truncation so it always survives; and a sitecustomize shim on the sandbox PYTHONPATH remaps those prefixes onto the CWD in open()/os.makedirs() with a one-line stderr notice, covering the python tool and any Python launched from the terminal tool without touching the exec wrapper (so tracebacks keep their line numbers). Bash-level file operations cannot be redirected without root or mount namespaces, so they rely on the description and the hint. * Studio: fix hidden-element parsing, heartbeat gaps, and tool output id collisions Review follow-ups on the tool streaming work: - _html_to_md: treat any present hidden attribute value as hidden (it is an enumerated attribute whose invalid value default is the Hidden state, so hidden="false" is still hidden), and implement HTML5 optional end tags so an unclosed