Merge branch 'main' into fix/web-search-ssl-sni
This commit is contained in:
Roland Tannous
GitHub
commit
2fc61ebab3
28 changed files with 2462 additions and 1141 deletions
19
install.ps1
19
install.ps1
|
|
@ -749,7 +749,6 @@ shell.Run cmd, 0, False
|
|||
} else {
|
||||
step "gpu" "none (chat-only / GGUF)" "Yellow"
|
||||
substep "Training and GPU inference require an NVIDIA GPU with drivers installed." "Yellow"
|
||||
substep "https://www.nvidia.com/Download/index.aspx" "Yellow"
|
||||
}
|
||||
|
||||
# ── Choose the correct PyTorch index URL based on driver CUDA version ──
|
||||
|
|
@ -777,10 +776,10 @@ shell.Run cmd, 0, False
|
|||
# ── Print CPU-only hint when no GPU detected ──
|
||||
if (-not $SkipTorch -and $TorchIndexUrl -like "*/cpu") {
|
||||
Write-Host ""
|
||||
Write-Host " NOTE: No NVIDIA GPU detected." -ForegroundColor Yellow
|
||||
Write-Host " Installing CPU-only PyTorch. If you only need GGUF chat/inference,"
|
||||
Write-Host " re-run with --no-torch for a faster, lighter install:"
|
||||
Write-Host " .\install.ps1 --no-torch"
|
||||
substep "No NVIDIA GPU detected." "Yellow"
|
||||
substep "Installing CPU-only PyTorch. If you only need GGUF chat/inference," "Yellow"
|
||||
substep "re-run with --no-torch for a faster, lighter install:" "Yellow"
|
||||
substep ".\install.ps1 --no-torch" "Yellow"
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
|
|
@ -820,7 +819,7 @@ shell.Run cmd, 0, False
|
|||
if ($SkipTorch) {
|
||||
# No-torch: install unsloth + unsloth-zoo with --no-deps, then
|
||||
# runtime deps (typer, safetensors, transformers, etc.) with --no-deps.
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.3.16" unsloth-zoo }
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.3.18" unsloth-zoo }
|
||||
if ($baseInstallExit -eq 0) {
|
||||
$NoTorchReq = Find-NoTorchRuntimeFile
|
||||
if ($NoTorchReq) {
|
||||
|
|
@ -828,7 +827,7 @@ shell.Run cmd, 0, False
|
|||
}
|
||||
}
|
||||
} else {
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.3.16" unsloth-zoo }
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.3.18" unsloth-zoo }
|
||||
}
|
||||
if ($baseInstallExit -ne 0) {
|
||||
Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red
|
||||
|
|
@ -858,7 +857,7 @@ shell.Run cmd, 0, False
|
|||
if ($SkipTorch) {
|
||||
# No-torch: install unsloth + unsloth-zoo with --no-deps, then
|
||||
# runtime deps (typer, safetensors, transformers, etc.) with --no-deps.
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.3.16" unsloth-zoo }
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.3.18" unsloth-zoo }
|
||||
if ($baseInstallExit -eq 0) {
|
||||
$NoTorchReq = Find-NoTorchRuntimeFile
|
||||
if ($NoTorchReq) {
|
||||
|
|
@ -866,7 +865,7 @@ shell.Run cmd, 0, False
|
|||
}
|
||||
}
|
||||
} elseif ($StudioLocalInstall) {
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.3.16" unsloth-zoo }
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.3.18" unsloth-zoo }
|
||||
} else {
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "$PackageName" }
|
||||
}
|
||||
|
|
@ -887,7 +886,7 @@ shell.Run cmd, 0, False
|
|||
# Fallback: GPU detection failed to produce a URL -- let uv resolve torch
|
||||
substep "installing unsloth (this may take a few minutes)..."
|
||||
if ($StudioLocalInstall) {
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.3.16" --torch-backend=auto }
|
||||
$baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.3.18" --torch-backend=auto }
|
||||
if ($baseInstallExit -ne 0) {
|
||||
Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red
|
||||
return
|
||||
|
|
|
|||
10
install.sh
10
install.sh
|
|
@ -1029,7 +1029,7 @@ if [ "$_MIGRATED" = true ]; then
|
|||
# to prevent transitive torch resolution.
|
||||
run_install_cmd "install unsloth (migrated no-torch)" uv pip install --python "$_VENV_PY" --no-deps \
|
||||
--reinstall-package unsloth --reinstall-package unsloth-zoo \
|
||||
"unsloth>=2026.3.16" unsloth-zoo
|
||||
"unsloth>=2026.3.18" unsloth-zoo
|
||||
_NO_TORCH_RT="$(_find_no_torch_runtime)"
|
||||
if [ -n "$_NO_TORCH_RT" ]; then
|
||||
run_install_cmd "install no-torch runtime deps" uv pip install --python "$_VENV_PY" --no-deps -r "$_NO_TORCH_RT"
|
||||
|
|
@ -1037,7 +1037,7 @@ if [ "$_MIGRATED" = true ]; then
|
|||
else
|
||||
run_install_cmd "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \
|
||||
--reinstall-package unsloth --reinstall-package unsloth-zoo \
|
||||
"unsloth>=2026.3.16" unsloth-zoo
|
||||
"unsloth>=2026.3.18" unsloth-zoo
|
||||
fi
|
||||
if [ "$STUDIO_LOCAL_INSTALL" = true ]; then
|
||||
substep "overlaying local repo (editable)..."
|
||||
|
|
@ -1059,7 +1059,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then
|
|||
# runtime deps (typer, safetensors, transformers, etc.) with --no-deps.
|
||||
run_install_cmd "install unsloth (no-torch)" uv pip install --python "$_VENV_PY" --no-deps \
|
||||
--upgrade-package unsloth --upgrade-package unsloth-zoo \
|
||||
"unsloth>=2026.3.16" unsloth-zoo
|
||||
"unsloth>=2026.3.18" unsloth-zoo
|
||||
_NO_TORCH_RT="$(_find_no_torch_runtime)"
|
||||
if [ -n "$_NO_TORCH_RT" ]; then
|
||||
run_install_cmd "install no-torch runtime deps" uv pip install --python "$_VENV_PY" --no-deps -r "$_NO_TORCH_RT"
|
||||
|
|
@ -1070,7 +1070,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then
|
|||
fi
|
||||
elif [ "$STUDIO_LOCAL_INSTALL" = true ]; then
|
||||
run_install_cmd "install unsloth (local)" uv pip install --python "$_VENV_PY" \
|
||||
--upgrade-package unsloth "unsloth>=2026.3.16" unsloth-zoo
|
||||
--upgrade-package unsloth "unsloth>=2026.3.18" unsloth-zoo
|
||||
substep "overlaying local repo (editable)..."
|
||||
run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps
|
||||
else
|
||||
|
|
@ -1081,7 +1081,7 @@ else
|
|||
# Fallback: GPU detection failed to produce a URL -- let uv resolve torch
|
||||
substep "installing unsloth (this may take a few minutes)..."
|
||||
if [ "$STUDIO_LOCAL_INSTALL" = true ]; then
|
||||
run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.3.16" --torch-backend=auto
|
||||
run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.3.18" --torch-backend=auto
|
||||
substep "overlaying local repo (editable)..."
|
||||
run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps
|
||||
else
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ huggingfacenotorch = [
|
|||
]
|
||||
huggingface = [
|
||||
"unsloth[huggingfacenotorch]",
|
||||
"unsloth_zoo>=2026.3.6",
|
||||
"unsloth_zoo>=2026.3.7",
|
||||
"torchvision",
|
||||
"unsloth[triton]",
|
||||
]
|
||||
|
|
@ -578,7 +578,7 @@ colab-ampere-torch220 = [
|
|||
"flash-attn>=2.6.3 ; ('linux' in sys_platform)",
|
||||
]
|
||||
colab-new = [
|
||||
"unsloth_zoo>=2026.3.6",
|
||||
"unsloth_zoo>=2026.3.7",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0",
|
||||
|
|
|
|||
439
studio/backend/core/inference/_html_to_md.py
Normal file
439
studio/backend/core/inference/_html_to_md.py
Normal file
|
|
@ -0,0 +1,439 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""
|
||||
Minimal HTML-to-Markdown converter using only the standard library.
|
||||
|
||||
Replaces the external ``html2text`` (GPL-3.0) dependency with a ~250-line
|
||||
``html.parser.HTMLParser`` subclass. Covers headings, links, bold/italic,
|
||||
lists, tables, blockquotes, code blocks, and entity decoding.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
|
||||
__all__ = ["html_to_markdown"]
|
||||
|
||||
_SKIP_TAGS = frozenset({"script", "style", "head", "noscript", "svg", "math"})
|
||||
_BLOCK_TAGS = frozenset(
|
||||
{
|
||||
"p",
|
||||
"div",
|
||||
"section",
|
||||
"article",
|
||||
"header",
|
||||
"footer",
|
||||
"main",
|
||||
"aside",
|
||||
"nav",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"details",
|
||||
"summary",
|
||||
"dl",
|
||||
"dt",
|
||||
"dd",
|
||||
}
|
||||
)
|
||||
_HEADING_TAGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
|
||||
_INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}
|
||||
|
||||
|
||||
class _MarkdownRenderer(HTMLParser):
|
||||
"""HTMLParser subclass that emits Markdown tokens into a list."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs = False)
|
||||
self._out: list[str] = []
|
||||
self._skip_depth: int = 0
|
||||
|
||||
# Link state
|
||||
self._link_href: str | None = None
|
||||
self._link_text_parts: list[str] = []
|
||||
self._in_link: bool = False
|
||||
|
||||
# List state
|
||||
self._list_stack: list[str] = [] # "ul" or "ol"
|
||||
self._ol_counter: list[int] = []
|
||||
|
||||
# Table state
|
||||
self._in_table: bool = False
|
||||
self._current_row: list[str] = []
|
||||
self._cell_parts: list[str] = []
|
||||
self._in_cell: bool = False
|
||||
self._header_row_done: bool = False
|
||||
self._row_has_th: bool = False
|
||||
self._is_first_row: bool = False
|
||||
|
||||
# Pre/code state
|
||||
self._in_pre: bool = False
|
||||
self._pre_parts: list[str] = []
|
||||
self._in_inline_code: bool = False
|
||||
|
||||
# Blockquote state -- stack of output buffers so nested
|
||||
# blockquotes each collect their own content and get prefixed
|
||||
# with the correct number of ">" markers on close.
|
||||
self._bq_stack: list[list[str]] = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _emit(self, text: str) -> None:
|
||||
if self._in_link:
|
||||
self._link_text_parts.append(text)
|
||||
elif self._in_cell:
|
||||
self._cell_parts.append(text)
|
||||
elif self._in_pre:
|
||||
self._pre_parts.append(text)
|
||||
elif self._bq_stack:
|
||||
self._bq_stack[-1].append(text)
|
||||
else:
|
||||
self._out.append(text)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _prefix_blockquote(self, content: str) -> str:
|
||||
"""Prefix every line of *content* with ``> ``."""
|
||||
# Strip trailing whitespace first, then collapse blank lines
|
||||
content = re.sub(r"[ \t]+$", "", content, flags = re.MULTILINE)
|
||||
content = re.sub(r"\n{3,}", "\n\n", content).strip()
|
||||
if not content:
|
||||
return ""
|
||||
lines = content.split("\n")
|
||||
prefixed: list[str] = []
|
||||
for line in lines:
|
||||
if line.strip():
|
||||
prefixed.append("> " + line)
|
||||
else:
|
||||
prefixed.append(">")
|
||||
return "\n".join(prefixed)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Table helpers -- flush open cells and rows so that HTML with
|
||||
# omitted optional end tags (</td>, </tr>) does not lose data.
|
||||
# ------------------------------------------------------------------
|
||||
def _finish_cell(self) -> None:
|
||||
if not self._in_cell:
|
||||
return
|
||||
self._in_cell = False
|
||||
cell_text = "".join(self._cell_parts).strip().replace("\n", " ")
|
||||
cell_text = cell_text.replace("|", "\\|")
|
||||
self._current_row.append(cell_text)
|
||||
self._cell_parts = []
|
||||
|
||||
def _finish_row(self) -> None:
|
||||
if not self._current_row:
|
||||
return
|
||||
line = "| " + " | ".join(self._current_row) + " |"
|
||||
self._emit(line + "\n")
|
||||
if not self._header_row_done and (self._row_has_th or self._is_first_row):
|
||||
sep = "| " + " | ".join("---" for _ in self._current_row) + " |"
|
||||
self._emit(sep + "\n")
|
||||
self._header_row_done = True
|
||||
self._is_first_row = False
|
||||
self._current_row = []
|
||||
self._row_has_th = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Link text helper -- normalize whitespace so block-level content
|
||||
# inside an <a> does not produce multiline Markdown link labels.
|
||||
# ------------------------------------------------------------------
|
||||
def _finish_link(self) -> None:
|
||||
text = re.sub(r"\s+", " ", "".join(self._link_text_parts)).strip()
|
||||
href = self._link_href or ""
|
||||
self._in_link = False
|
||||
if href and text:
|
||||
self._emit(f"[{text}]({href})")
|
||||
elif text:
|
||||
self._emit(text)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tag handlers
|
||||
# ------------------------------------------------------------------
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag = tag.lower()
|
||||
|
||||
if tag in _SKIP_TAGS:
|
||||
self._skip_depth += 1
|
||||
return
|
||||
if self._skip_depth:
|
||||
return
|
||||
|
||||
attr_dict = dict(attrs)
|
||||
|
||||
if tag in _HEADING_TAGS:
|
||||
level = int(tag[1])
|
||||
self._emit("\n\n" + "#" * level + " ")
|
||||
|
||||
elif tag == "a":
|
||||
self._link_href = attr_dict.get("href")
|
||||
self._link_text_parts = []
|
||||
self._in_link = True
|
||||
|
||||
elif tag in _INLINE_EMPHASIS:
|
||||
self._emit(_INLINE_EMPHASIS[tag])
|
||||
|
||||
elif tag == "br":
|
||||
self._emit("\n")
|
||||
|
||||
elif tag in _BLOCK_TAGS:
|
||||
self._emit("\n\n")
|
||||
|
||||
elif tag == "hr":
|
||||
self._emit("\n\n---\n\n")
|
||||
|
||||
elif tag == "blockquote":
|
||||
self._emit("\n\n")
|
||||
self._bq_stack.append([])
|
||||
|
||||
elif tag == "ul":
|
||||
self._list_stack.append("ul")
|
||||
self._emit("\n")
|
||||
|
||||
elif tag == "ol":
|
||||
self._list_stack.append("ol")
|
||||
start_attr = attr_dict.get("start")
|
||||
try:
|
||||
start = int(start_attr) if start_attr is not None else 1
|
||||
except (ValueError, TypeError):
|
||||
start = 1
|
||||
self._ol_counter.append(start - 1)
|
||||
self._emit("\n")
|
||||
|
||||
elif tag == "li":
|
||||
indent = " " * max(0, len(self._list_stack) - 1)
|
||||
if self._list_stack and self._list_stack[-1] == "ol":
|
||||
if self._ol_counter:
|
||||
self._ol_counter[-1] += 1
|
||||
self._emit(f"\n{indent}{self._ol_counter[-1]}. ")
|
||||
else:
|
||||
self._emit(f"\n{indent}1. ")
|
||||
else:
|
||||
self._emit(f"\n{indent}* ")
|
||||
|
||||
elif tag == "pre":
|
||||
self._pre_parts = []
|
||||
self._in_pre = True
|
||||
|
||||
elif tag == "code" and not self._in_pre:
|
||||
self._in_inline_code = True
|
||||
self._emit("`")
|
||||
|
||||
elif tag == "table":
|
||||
self._in_table = True
|
||||
self._header_row_done = False
|
||||
self._is_first_row = True
|
||||
self._emit("\n\n")
|
||||
|
||||
elif tag == "tr":
|
||||
# Flush any open cell/row from a previous row that may
|
||||
# have omitted its optional </td> or </tr> end tags.
|
||||
self._finish_cell()
|
||||
self._finish_row()
|
||||
|
||||
elif tag in ("th", "td"):
|
||||
# Flush any open cell (handles omitted </td>/<th>)
|
||||
self._finish_cell()
|
||||
self._cell_parts = []
|
||||
self._in_cell = True
|
||||
if tag == "th":
|
||||
self._row_has_th = True
|
||||
|
||||
elif tag == "img":
|
||||
# Skip images -- keeps fetched page text focused on readable
|
||||
# content and avoids data-URI amplification.
|
||||
return
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
tag = tag.lower()
|
||||
|
||||
if tag in _SKIP_TAGS:
|
||||
self._skip_depth = max(0, self._skip_depth - 1)
|
||||
return
|
||||
if self._skip_depth:
|
||||
return
|
||||
|
||||
if tag in _HEADING_TAGS:
|
||||
self._emit("\n\n")
|
||||
|
||||
elif tag == "a":
|
||||
self._finish_link()
|
||||
|
||||
elif tag in _INLINE_EMPHASIS:
|
||||
self._emit(_INLINE_EMPHASIS[tag])
|
||||
|
||||
elif tag in _BLOCK_TAGS:
|
||||
self._emit("\n\n")
|
||||
|
||||
elif tag == "blockquote":
|
||||
if self._bq_stack:
|
||||
content = "".join(self._bq_stack.pop())
|
||||
prefixed = self._prefix_blockquote(content)
|
||||
if prefixed:
|
||||
self._emit("\n\n" + prefixed + "\n\n")
|
||||
|
||||
elif tag == "ul":
|
||||
if self._list_stack and self._list_stack[-1] == "ul":
|
||||
self._list_stack.pop()
|
||||
self._emit("\n")
|
||||
|
||||
elif tag == "ol":
|
||||
if self._list_stack and self._list_stack[-1] == "ol":
|
||||
self._list_stack.pop()
|
||||
if self._ol_counter:
|
||||
self._ol_counter.pop()
|
||||
self._emit("\n")
|
||||
|
||||
elif tag == "pre":
|
||||
raw = "".join(self._pre_parts)
|
||||
self._in_pre = False
|
||||
block = "```\n" + raw + "\n```"
|
||||
self._emit("\n\n" + block + "\n\n")
|
||||
|
||||
elif tag == "code" and not self._in_pre:
|
||||
self._in_inline_code = False
|
||||
self._emit("`")
|
||||
|
||||
elif tag in ("th", "td"):
|
||||
self._finish_cell()
|
||||
|
||||
elif tag == "tr":
|
||||
self._finish_cell()
|
||||
self._finish_row()
|
||||
|
||||
elif tag == "table":
|
||||
# Flush any remaining row (handles omitted </tr>)
|
||||
self._finish_cell()
|
||||
self._finish_row()
|
||||
self._in_table = False
|
||||
self._emit("\n")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Text / entity handlers
|
||||
# ------------------------------------------------------------------
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._skip_depth:
|
||||
return
|
||||
if self._in_pre:
|
||||
self._pre_parts.append(data)
|
||||
return
|
||||
# Preserve literal whitespace inside inline <code> spans
|
||||
if self._in_inline_code:
|
||||
self._emit(data)
|
||||
return
|
||||
# Collapse all whitespace (including newlines) per HTML rules
|
||||
text = re.sub(r"\s+", " ", data)
|
||||
# Suppress whitespace-only text nodes between table structural
|
||||
# elements (indentation from source HTML) to prevent leading
|
||||
# spaces from breaking Markdown table row alignment.
|
||||
if self._in_table and not self._in_cell and not text.strip():
|
||||
return
|
||||
self._emit(text)
|
||||
|
||||
def handle_entityref(self, name: str) -> None:
|
||||
if self._skip_depth:
|
||||
return
|
||||
self._emit(html.unescape(f"&{name};"))
|
||||
|
||||
def handle_charref(self, name: str) -> None:
|
||||
if self._skip_depth:
|
||||
return
|
||||
self._emit(html.unescape(f"&#{name};"))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Flush pending buffers (handles truncated HTML from capped fetches)
|
||||
# ------------------------------------------------------------------
|
||||
def flush_pending(self) -> None:
|
||||
"""Flush any open side-buffers into ``_out``.
|
||||
|
||||
Called after ``close()`` to recover content from truncated HTML
|
||||
where closing tags were never seen (common when ``_fetch_page_text``
|
||||
caps the download by byte count).
|
||||
"""
|
||||
# Flush innermost buffers first so their content propagates outward.
|
||||
|
||||
if self._in_link:
|
||||
self._finish_link()
|
||||
|
||||
if self._in_inline_code:
|
||||
self._in_inline_code = False
|
||||
self._emit("`")
|
||||
|
||||
self._finish_cell()
|
||||
self._finish_row()
|
||||
|
||||
if self._in_pre:
|
||||
raw = "".join(self._pre_parts)
|
||||
self._in_pre = False
|
||||
block = "```\n" + raw + "\n```"
|
||||
self._emit("\n\n" + block + "\n\n")
|
||||
|
||||
# Flatten any open blockquote buffers (innermost first)
|
||||
while self._bq_stack:
|
||||
content = "".join(self._bq_stack.pop())
|
||||
prefixed = self._prefix_blockquote(content)
|
||||
if not prefixed:
|
||||
continue
|
||||
if self._bq_stack:
|
||||
self._bq_stack[-1].append("\n\n" + prefixed + "\n\n")
|
||||
else:
|
||||
self._out.append("\n\n" + prefixed + "\n\n")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Post-processing
|
||||
# ------------------------------------------------------------------
|
||||
def _cleanup(text: str) -> str:
|
||||
"""Normalize whitespace and blank lines in the final output.
|
||||
|
||||
Preserves content inside fenced code blocks verbatim so that
|
||||
intentional blank lines in ``<pre>`` content are not collapsed.
|
||||
"""
|
||||
lines = text.split("\n")
|
||||
out: list[str] = []
|
||||
in_fence = False
|
||||
blank_run = 0
|
||||
|
||||
for line in lines:
|
||||
stripped = line.rstrip(" \t")
|
||||
if stripped.startswith("```"):
|
||||
in_fence = not in_fence
|
||||
blank_run = 0
|
||||
out.append(stripped)
|
||||
continue
|
||||
|
||||
if in_fence:
|
||||
# Preserve code block content exactly as-is
|
||||
out.append(line)
|
||||
continue
|
||||
|
||||
if not stripped:
|
||||
blank_run += 1
|
||||
if blank_run <= 1:
|
||||
out.append("")
|
||||
continue
|
||||
|
||||
blank_run = 0
|
||||
out.append(stripped)
|
||||
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
def html_to_markdown(source_html: str) -> str:
|
||||
"""Convert an HTML string to Markdown.
|
||||
|
||||
Handles headings, links, bold/italic, lists (ordered and unordered),
|
||||
tables, blockquotes, code blocks, and HTML entities. ``<script>``,
|
||||
``<style>``, and ``<head>`` sections are stripped entirely.
|
||||
"""
|
||||
# Normalize line endings before parsing
|
||||
source_html = source_html.replace("\r\n", "\n").replace("\r", "\n")
|
||||
renderer = _MarkdownRenderer()
|
||||
renderer.feed(source_html)
|
||||
renderer.close()
|
||||
renderer.flush_pending()
|
||||
raw = "".join(renderer._out)
|
||||
return _cleanup(raw)
|
||||
|
|
@ -10,7 +10,6 @@ through its OpenAI-compatible /v1/chat/completions endpoint.
|
|||
|
||||
import atexit
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import struct
|
||||
|
|
@ -2181,22 +2180,6 @@ class LlamaCppBackend:
|
|||
# identical call succeeded).
|
||||
_tool_call_history: list[tuple[str, bool]] = [] # (key, failed)
|
||||
|
||||
def _tool_call_key(name: str, args: dict) -> str:
|
||||
raw = json.dumps({"t": name, "a": args}, sort_keys = True)
|
||||
return hashlib.md5(raw.encode()).hexdigest()
|
||||
|
||||
def _is_duplicate_call(name: str, args: dict) -> bool:
|
||||
"""Block if the immediately previous call was identical and succeeded."""
|
||||
if not _tool_call_history:
|
||||
return False
|
||||
key = _tool_call_key(name, args)
|
||||
last_key, last_failed = _tool_call_history[-1]
|
||||
return last_key == key and not last_failed
|
||||
|
||||
def _record_tool_call(name: str, args: dict, failed: bool) -> None:
|
||||
key = _tool_call_key(name, args)
|
||||
_tool_call_history.append((key, failed))
|
||||
|
||||
for iteration in range(max_tool_iterations):
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
return
|
||||
|
|
@ -2692,7 +2675,12 @@ class LlamaCppBackend:
|
|||
}
|
||||
|
||||
# ── Duplicate call detection ──────────────
|
||||
if _is_duplicate_call(tool_name, arguments):
|
||||
# str(dict) is stable here: arguments always comes from
|
||||
# json.loads on the same model output within one request,
|
||||
# so insertion order is deterministic (Python 3.7+).
|
||||
_tc_key = tool_name + str(arguments)
|
||||
_prev = _tool_call_history[-1] if _tool_call_history else None
|
||||
if _prev and _prev[0] == _tc_key and not _prev[1]:
|
||||
result = (
|
||||
"You already made this exact call. "
|
||||
"Do not repeat the same tool call. "
|
||||
|
|
@ -2734,7 +2722,7 @@ class LlamaCppBackend:
|
|||
_is_error = isinstance(result, str) and result.lstrip().startswith(
|
||||
_error_prefixes
|
||||
)
|
||||
_record_tool_call(tool_name, arguments, failed = _is_error)
|
||||
_tool_call_history.append((_tc_key, _is_error))
|
||||
_result_content = result
|
||||
if _is_error:
|
||||
_result_content = (
|
||||
|
|
@ -2752,6 +2740,8 @@ class LlamaCppBackend:
|
|||
tool_msg["tool_call_id"] = tool_call_id
|
||||
conversation.append(tool_msg)
|
||||
|
||||
# Clear tool status badge before next generation iteration
|
||||
yield {"type": "status", "text": ""}
|
||||
# Continue the loop to let model respond with context
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -201,7 +201,6 @@ def _fetch_page_text(
|
|||
Blocks private/loopback/link-local targets (SSRF protection) and caps
|
||||
the download size to avoid unbounded memory usage.
|
||||
"""
|
||||
import re as _re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
|
|
@ -351,28 +350,10 @@ def _fetch_page_text(
|
|||
except Exception as e:
|
||||
return f"Failed to fetch URL: {e}"
|
||||
|
||||
# Convert HTML to text -- prefer html2text for clean markdown output
|
||||
try:
|
||||
import html2text as _h2t
|
||||
# Convert HTML to Markdown using the builtin converter (no external deps)
|
||||
from ._html_to_md import html_to_markdown
|
||||
|
||||
converter = _h2t.HTML2Text()
|
||||
converter.ignore_links = False
|
||||
converter.ignore_images = True
|
||||
converter.body_width = 0 # no wrapping
|
||||
text = converter.handle(raw_html).strip()
|
||||
except ImportError:
|
||||
# Fallback: regex-based stripping
|
||||
text = _re.sub(
|
||||
r"<script[^>]*>.*?</script[^>]*>",
|
||||
"",
|
||||
raw_html,
|
||||
flags = _re.DOTALL | _re.IGNORECASE,
|
||||
)
|
||||
text = _re.sub(
|
||||
r"<style[^>]*>.*?</style[^>]*>", "", text, flags = _re.DOTALL | _re.IGNORECASE
|
||||
)
|
||||
text = _re.sub(r"<[^>]+>", " ", text)
|
||||
text = _re.sub(r"\s+", " ", text).strip()
|
||||
text = html_to_markdown(raw_html)
|
||||
|
||||
if not text:
|
||||
return "(page returned no readable text)"
|
||||
|
|
|
|||
|
|
@ -121,13 +121,13 @@ async def lifespan(app: FastAPI):
|
|||
if storage.ensure_default_admin():
|
||||
bootstrap_pw = storage.get_bootstrap_password()
|
||||
app.state.bootstrap_password = bootstrap_pw
|
||||
|
||||
bootstrap_path = storage.DB_PATH.parent / ".bootstrap_password"
|
||||
print("\n" + "=" * 60)
|
||||
print("DEFAULT ADMIN ACCOUNT CREATED")
|
||||
print(
|
||||
"Sign in with the seeded credentials and change the password immediately:\n"
|
||||
)
|
||||
print(f" username: {storage.DEFAULT_ADMIN_USERNAME}")
|
||||
print(f" password: {bootstrap_pw}\n")
|
||||
print(f" password saved to: {bootstrap_path}")
|
||||
print(" Open the Studio UI to sign in and change it.")
|
||||
print("=" * 60 + "\n")
|
||||
else:
|
||||
app.state.bootstrap_password = storage.get_bootstrap_password()
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ class LocalModelInfo(BaseModel):
|
|||
id: str = Field(..., description = "Identifier to use for loading/training")
|
||||
display_name: str = Field(..., description = "Display label")
|
||||
path: str = Field(..., description = "Local path where model data was discovered")
|
||||
source: Literal["models_dir", "hf_cache", "lmstudio"] = Field(
|
||||
source: Literal["models_dir", "hf_cache", "lmstudio", "custom"] = Field(
|
||||
...,
|
||||
description = "Discovery source",
|
||||
)
|
||||
|
|
@ -197,3 +197,19 @@ class LocalModelListResponse(BaseModel):
|
|||
default_factory = list,
|
||||
description = "Discovered local/cached models",
|
||||
)
|
||||
|
||||
|
||||
class AddScanFolderRequest(BaseModel):
|
||||
"""Request body for adding a custom scan folder."""
|
||||
|
||||
path: str = Field(
|
||||
..., description = "Absolute or relative directory path to scan for models"
|
||||
)
|
||||
|
||||
|
||||
class ScanFolderInfo(BaseModel):
|
||||
"""A registered custom model scan folder."""
|
||||
|
||||
id: int = Field(..., description = "Database row ID")
|
||||
path: str = Field(..., description = "Normalized absolute path")
|
||||
created_at: str = Field(..., description = "ISO 8601 creation timestamp")
|
||||
|
|
|
|||
|
|
@ -94,7 +94,13 @@ from models import (
|
|||
LoRAInfo,
|
||||
ModelListResponse,
|
||||
)
|
||||
from models.models import GgufVariantDetail, GgufVariantsResponse, ModelType
|
||||
from models.models import (
|
||||
GgufVariantDetail,
|
||||
GgufVariantsResponse,
|
||||
ModelType,
|
||||
ScanFolderInfo,
|
||||
AddScanFolderRequest,
|
||||
)
|
||||
from models.responses import (
|
||||
LoRABaseModelResponse,
|
||||
VisionCheckResponse,
|
||||
|
|
@ -128,21 +134,32 @@ def _resolve_hf_cache_dir() -> Path:
|
|||
return Path.home() / ".cache" / "huggingface" / "hub"
|
||||
|
||||
|
||||
def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]:
|
||||
def _scan_models_dir(
|
||||
models_dir: Path,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
) -> List[LocalModelInfo]:
|
||||
if not models_dir.exists() or not models_dir.is_dir():
|
||||
return []
|
||||
|
||||
found: List[LocalModelInfo] = []
|
||||
for child in models_dir.iterdir():
|
||||
if not child.is_dir():
|
||||
if limit is not None and len(found) >= limit:
|
||||
break
|
||||
try:
|
||||
if not child.is_dir():
|
||||
continue
|
||||
has_model_files = (
|
||||
(child / "config.json").exists()
|
||||
or (child / "adapter_config.json").exists()
|
||||
or any(child.glob("*.safetensors"))
|
||||
or any(child.glob("*.bin"))
|
||||
or any(child.glob("*.gguf"))
|
||||
)
|
||||
except OSError:
|
||||
# Skip individual children that are unreadable (permissions, broken
|
||||
# symlinks, etc.) rather than failing the entire scan.
|
||||
continue
|
||||
has_model_files = (
|
||||
(child / "config.json").exists()
|
||||
or (child / "adapter_config.json").exists()
|
||||
or any(child.glob("*.safetensors"))
|
||||
or any(child.glob("*.bin"))
|
||||
or any(child.glob("*.gguf"))
|
||||
)
|
||||
if not has_model_files:
|
||||
continue
|
||||
try:
|
||||
|
|
@ -159,21 +176,24 @@ def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]:
|
|||
),
|
||||
)
|
||||
# Also scan for standalone .gguf files directly in the models directory
|
||||
for gguf_file in models_dir.glob("*.gguf"):
|
||||
if gguf_file.is_file():
|
||||
try:
|
||||
updated_at = gguf_file.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(gguf_file),
|
||||
display_name = gguf_file.stem,
|
||||
path = str(gguf_file),
|
||||
source = "models_dir",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
if limit is None or len(found) < limit:
|
||||
for gguf_file in models_dir.glob("*.gguf"):
|
||||
if limit is not None and len(found) >= limit:
|
||||
break
|
||||
if gguf_file.is_file():
|
||||
try:
|
||||
updated_at = gguf_file.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(gguf_file),
|
||||
display_name = gguf_file.stem,
|
||||
path = str(gguf_file),
|
||||
source = "models_dir",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
|
||||
return found
|
||||
|
||||
|
|
@ -221,63 +241,69 @@ def _scan_lmstudio_dir(lm_dir: Path) -> List[LocalModelInfo]:
|
|||
|
||||
found: List[LocalModelInfo] = []
|
||||
for child in lm_dir.iterdir():
|
||||
if not child.is_dir():
|
||||
if child.suffix == ".gguf" and child.is_file():
|
||||
try:
|
||||
updated_at = child.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(child),
|
||||
display_name = child.stem,
|
||||
path = str(child),
|
||||
source = "lmstudio",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
continue
|
||||
try:
|
||||
if not child.is_dir():
|
||||
if child.suffix == ".gguf" and child.is_file():
|
||||
try:
|
||||
updated_at = child.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(child),
|
||||
display_name = child.stem,
|
||||
path = str(child),
|
||||
source = "lmstudio",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# child is a publisher directory — scan its sub-directories
|
||||
for model_dir in child.iterdir():
|
||||
if model_dir.is_dir():
|
||||
has_model = (
|
||||
any(model_dir.glob("*.gguf"))
|
||||
or (model_dir / "config.json").exists()
|
||||
or any(model_dir.glob("*.safetensors"))
|
||||
)
|
||||
if not has_model:
|
||||
# child is a publisher directory -- scan its sub-directories
|
||||
for model_dir in child.iterdir():
|
||||
try:
|
||||
if model_dir.is_dir():
|
||||
has_model = (
|
||||
any(model_dir.glob("*.gguf"))
|
||||
or (model_dir / "config.json").exists()
|
||||
or any(model_dir.glob("*.safetensors"))
|
||||
)
|
||||
if not has_model:
|
||||
continue
|
||||
model_id = f"{child.name}/{model_dir.name}"
|
||||
try:
|
||||
updated_at = model_dir.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(model_dir),
|
||||
model_id = model_id,
|
||||
display_name = model_dir.name,
|
||||
path = str(model_dir),
|
||||
source = "lmstudio",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
elif model_dir.suffix == ".gguf" and model_dir.is_file():
|
||||
try:
|
||||
updated_at = model_dir.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(model_dir),
|
||||
model_id = f"{child.name}/{model_dir.stem}",
|
||||
display_name = model_dir.stem,
|
||||
path = str(model_dir),
|
||||
source = "lmstudio",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
model_id = f"{child.name}/{model_dir.name}"
|
||||
try:
|
||||
updated_at = model_dir.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(model_dir),
|
||||
model_id = model_id,
|
||||
display_name = model_dir.name,
|
||||
path = str(model_dir),
|
||||
source = "lmstudio",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
elif model_dir.suffix == ".gguf" and model_dir.is_file():
|
||||
try:
|
||||
updated_at = model_dir.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id = str(model_dir),
|
||||
model_id = f"{child.name}/{model_dir.stem}",
|
||||
display_name = model_dir.stem,
|
||||
path = str(model_dir),
|
||||
source = "lmstudio",
|
||||
updated_at = updated_at,
|
||||
),
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
return found
|
||||
|
||||
|
||||
|
|
@ -351,10 +377,39 @@ async def list_local_models(
|
|||
for lm_dir in lm_dirs:
|
||||
local_models += _scan_lmstudio_dir(lm_dir)
|
||||
|
||||
# Scan user-added custom folders (cap per-folder to avoid unbounded scans)
|
||||
from storage.studio_db import list_scan_folders
|
||||
|
||||
_MAX_MODELS_PER_FOLDER = 200
|
||||
try:
|
||||
custom_folders = list_scan_folders()
|
||||
except Exception as e:
|
||||
logger.warning("Could not load custom scan folders: %s", e)
|
||||
custom_folders = []
|
||||
for folder in custom_folders:
|
||||
folder_path = Path(folder["path"])
|
||||
try:
|
||||
custom_models = (
|
||||
_scan_models_dir(folder_path, limit = _MAX_MODELS_PER_FOLDER)
|
||||
+ _scan_hf_cache(folder_path)
|
||||
+ _scan_lmstudio_dir(folder_path)
|
||||
)[:_MAX_MODELS_PER_FOLDER]
|
||||
except OSError as e:
|
||||
logger.warning("Skipping unreadable scan folder %s: %s", folder_path, e)
|
||||
continue
|
||||
local_models += [
|
||||
m.model_copy(update = {"source": "custom"}) for m in custom_models
|
||||
]
|
||||
|
||||
# Deduplicate models, but always keep custom folder entries so they
|
||||
# appear in the "Custom Folders" UI section even when the same model
|
||||
# also exists in the HF cache or default models directory. Use a
|
||||
# (id, source) key for custom entries to avoid collisions.
|
||||
deduped: dict[str, LocalModelInfo] = {}
|
||||
for model in local_models:
|
||||
if model.id not in deduped:
|
||||
deduped[model.id] = model
|
||||
key = f"{model.id}\x00custom" if model.source == "custom" else model.id
|
||||
if key not in deduped:
|
||||
deduped[key] = model
|
||||
|
||||
models = sorted(
|
||||
deduped.values(),
|
||||
|
|
@ -376,6 +431,46 @@ async def list_local_models(
|
|||
)
|
||||
|
||||
|
||||
@router.get("/scan-folders")
|
||||
async def get_scan_folders(
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""List all registered custom model scan folders."""
|
||||
from storage.studio_db import list_scan_folders
|
||||
|
||||
return {"folders": list_scan_folders()}
|
||||
|
||||
|
||||
@router.post("/scan-folders", response_model = ScanFolderInfo, status_code = 201)
|
||||
async def add_scan_folder_endpoint(
|
||||
body: AddScanFolderRequest,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Register a new directory to scan for local models."""
|
||||
from storage.studio_db import add_scan_folder
|
||||
|
||||
try:
|
||||
folder = add_scan_folder(body.path)
|
||||
except ValueError as e:
|
||||
logger.warning("Scan folder rejected: %s (path=%s)", e, body.path)
|
||||
raise HTTPException(status_code = 400, detail = str(e))
|
||||
logger.info("Scan folder added: %s", folder.get("path"))
|
||||
return folder
|
||||
|
||||
|
||||
@router.delete("/scan-folders/{folder_id}")
|
||||
async def remove_scan_folder_endpoint(
|
||||
folder_id: int,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Remove a registered custom scan folder."""
|
||||
from storage.studio_db import remove_scan_folder
|
||||
|
||||
remove_scan_folder(folder_id)
|
||||
logger.info("Scan folder removed: id=%s", folder_id)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/list")
|
||||
async def list_models(
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
|
|
|
|||
|
|
@ -12,14 +12,46 @@ raw sqlite3, per-function connections. Enhancements over auth:
|
|||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from typing import Optional
|
||||
|
||||
|
||||
from utils.paths import studio_db_path, ensure_dir
|
||||
|
||||
|
||||
def _denied_path_prefixes() -> list[str]:
|
||||
"""Platform-aware denylist of system directories."""
|
||||
system = platform.system()
|
||||
if system == "Linux":
|
||||
return ["/proc", "/sys", "/dev", "/etc", "/boot", "/run"]
|
||||
if system == "Darwin":
|
||||
# realpath() resolves /etc -> /private/etc, /tmp -> /private/tmp on macOS,
|
||||
# so include the /private variants to avoid bypasses.
|
||||
return [
|
||||
"/System",
|
||||
"/Library",
|
||||
"/dev",
|
||||
"/etc",
|
||||
"/private/etc",
|
||||
"/tmp",
|
||||
"/private/tmp",
|
||||
"/var",
|
||||
"/private/var",
|
||||
]
|
||||
if system == "Windows":
|
||||
win = os.environ.get("SystemRoot", r"C:\Windows")
|
||||
pf = os.environ.get("ProgramFiles", r"C:\Program Files")
|
||||
pf86 = os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")
|
||||
return [os.path.normcase(p) for p in [win, pf, pf86]]
|
||||
return []
|
||||
|
||||
|
||||
_schema_lock = threading.Lock()
|
||||
_schema_ready = False
|
||||
|
||||
|
|
@ -67,6 +99,19 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
|
|||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_metrics_run_id ON training_metrics(run_id)"
|
||||
)
|
||||
# Use COLLATE NOCASE on Windows so C:\Models and c:\models dedup via the
|
||||
# UNIQUE constraint. On Linux/macOS (case-sensitive FS) keep the default
|
||||
# BINARY collation so /Models and /models remain distinct.
|
||||
collation = "COLLATE NOCASE" if platform.system() == "Windows" else ""
|
||||
conn.execute(
|
||||
f"""
|
||||
CREATE TABLE IF NOT EXISTS scan_folders (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
path TEXT NOT NULL UNIQUE {collation},
|
||||
created_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def get_connection() -> sqlite3.Connection:
|
||||
|
|
@ -343,8 +388,6 @@ def delete_run(id: str) -> None:
|
|||
|
||||
def cleanup_orphaned_runs() -> None:
|
||||
"""Mark any 'running' rows as errored on startup (server restarted mid-training)."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
|
|
@ -360,3 +403,86 @@ def cleanup_orphaned_runs() -> None:
|
|||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_scan_folders() -> list[dict]:
|
||||
conn = get_connection()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT id, path, created_at FROM scan_folders ORDER BY created_at"
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def add_scan_folder(path: str) -> dict:
|
||||
"""Add a directory to the custom scan folder list. Returns the row."""
|
||||
if not path or not path.strip():
|
||||
raise ValueError("Path cannot be empty")
|
||||
normalized = os.path.realpath(os.path.expanduser(path.strip()))
|
||||
|
||||
# Validate the path is an existing, readable directory before persisting.
|
||||
if not os.path.exists(normalized):
|
||||
raise ValueError("Path does not exist")
|
||||
if not os.path.isdir(normalized):
|
||||
raise ValueError("Path must be a directory, not a file")
|
||||
if not os.access(normalized, os.R_OK | os.X_OK):
|
||||
raise ValueError("Path is not readable")
|
||||
|
||||
# On Windows, use normcase for denylist comparison but store the
|
||||
# original-cased path so downstream consumers see the native
|
||||
# drive-letter casing the user expects (e.g. C:\Models, not c:\models).
|
||||
is_win = platform.system() == "Windows"
|
||||
check = os.path.normcase(normalized) if is_win else normalized
|
||||
for prefix in _denied_path_prefixes():
|
||||
if check == prefix or check.startswith(prefix + os.sep):
|
||||
raise ValueError(f"Path under {prefix} is not allowed")
|
||||
|
||||
conn = get_connection()
|
||||
try:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
# On Windows, use case-insensitive lookup so C:\Models and c:\models
|
||||
# dedup correctly while preserving the originally-stored casing.
|
||||
if is_win:
|
||||
existing = conn.execute(
|
||||
"SELECT id, path, created_at FROM scan_folders WHERE path = ? COLLATE NOCASE",
|
||||
(normalized,),
|
||||
).fetchone()
|
||||
else:
|
||||
existing = conn.execute(
|
||||
"SELECT id, path, created_at FROM scan_folders WHERE path = ?",
|
||||
(normalized,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
return dict(existing)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO scan_folders (path, created_at) VALUES (?, ?)",
|
||||
(normalized, now),
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.IntegrityError:
|
||||
pass # duplicate -- fall through to SELECT
|
||||
# Use the same collation as the pre-check so we find the row even
|
||||
# when a concurrent writer stored it with different casing (Windows).
|
||||
fallback_sql = (
|
||||
"SELECT id, path, created_at FROM scan_folders WHERE path = ? COLLATE NOCASE"
|
||||
if is_win
|
||||
else "SELECT id, path, created_at FROM scan_folders WHERE path = ?"
|
||||
)
|
||||
row = conn.execute(fallback_sql, (normalized,)).fetchone()
|
||||
if row is None:
|
||||
raise ValueError("Folder was concurrently removed")
|
||||
return dict(row)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def remove_scan_folder(id: int) -> None:
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute("DELETE FROM scan_folders WHERE id = ?", (id,))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
# GGUF Tool Calling Benchmark Results
|
||||
|
||||
Prompt: "List and categorize all the songs that charted #3 on the Billboard Hot 100 in 2015."
|
||||
10 runs per configuration, web search + code execution + thinking enabled.
|
||||
GPU: NVIDIA B200, CUDA_VISIBLE_DEVICES=2.
|
||||
|
||||
Ground truth: 4 songs peaked at #3 in 2015 -- "Love Me like You Do" (Ellie Goulding), "Earned It" (The Weeknd), "Watch Me" (Silento), "Drag Me Down" (One Direction).
|
||||
|
||||
## Cartesian Grid: Model x Quant x KV Cache
|
||||
|
||||
| Model | Quant | KV Cache | OK/10 | Avg Time | Avg Tools | XML Leaks | URL Fetch | Peak3 Avg | All 4/4 | Best Songs |
|
||||
|-------|-------|----------|-------|----------|-----------|-----------|-----------|-----------|---------|------------|
|
||||
| 4B | UD-Q4_K_XL | f16 | 10/10 | 9.8s | 3.5 | 0/10 | 4/10 | 0.8/4 | 2/10 | 9 |
|
||||
| 4B | UD-Q4_K_XL | bf16 | 10/10 | 10.6s | 4.5 | 0/10 | 4/10 | 0.4/4 | 1/10 | 5 |
|
||||
| 4B | Q8_0 | f16 | 10/10 | 4.9s | 2.4 | 0/10 | 8/10 | 0.4/4 | 1/10 | 5 |
|
||||
| 4B | Q8_0 | bf16 | 10/10 | 8.0s | 3.0 | 0/10 | 5/10 | 0.0/4 | 0/10 | 0 |
|
||||
| 9B | UD-Q4_K_XL | f16 | 10/10 | 6.7s | 2.0 | 0/10 | 5/10 | 0.0/4 | 0/10 | 3 |
|
||||
| 9B | UD-Q4_K_XL | bf16 | 9/10 | 49.5s | 2.4 | 0/10 | 5/10 | 0.0/4 | 0/10 | 1 |
|
||||
| 9B | Q8_0 | f16 | 10/10 | 7.4s | 2.5 | 0/10 | 5/10 | 0.0/4 | 0/10 | 2 |
|
||||
| 9B | Q8_0 | bf16 | 10/10 | 10.4s | 2.7 | 0/10 | 6/10 | 1.0/4 | 2/10 | 15 |
|
||||
| **27B** | **UD-Q4_K_XL** | **bf16** | **9/10** | **131.1s** | **13.8** | **0/10** | **7/10** | **2.7/4** | **6/10** | **27** |
|
||||
| 27B | UD-Q4_K_XL | f16 | 7/10 | 201.6s | 14.1 | 0/10 | 8/10 | 2.0/4 | 5/10 | 26 |
|
||||
| 27B | Q8_0 | f16 | 4/10 | 312.5s | 16.0 | 1/10 | 10/10 | 2.4/4 | 6/10 | 28 |
|
||||
| 27B | Q8_0 | bf16 | 5/10 | 258.4s | 16.5 | 2/10 | 10/10 | 0.9/4 | 1/10 | 27 |
|
||||
| 35B-A3B | UD-Q4_K_XL | f16 | 3/10 | 353.6s | 14.7 | 1/10 | 6/10 | 1.2/4 | 3/10 | 27 |
|
||||
| 35B-A3B | UD-Q4_K_XL | bf16 | 3/10 | 356.2s | 17.2 | 1/10 | 8/10 | 1.6/4 | 4/10 | 27 |
|
||||
| 35B-A3B | Q8_0 | f16 | 2/10 | 372.1s | 17.6 | 1/10 | 7/10 | 1.2/4 | 3/10 | 26 |
|
||||
| 35B-A3B | Q8_0 | bf16 | 6/10 | 267.7s | 17.5 | 1/10 | 8/10 | 2.4/4 | 6/10 | 27 |
|
||||
|
||||
**Column definitions:**
|
||||
- **Peak3 Avg**: Average number of correct peak-#3 songs found per run (out of 4)
|
||||
- **All 4/4**: Runs where all 4 correct songs were identified
|
||||
- **Best Songs**: Maximum number of Billboard 2015 songs mentioned in any single run (out of 31 tracked)
|
||||
- **URL Fetch**: Runs where the model used web_search with `url` parameter to fetch full page content
|
||||
|
||||
## Key Findings
|
||||
|
||||
1. **27B UD-Q4_K_XL + bf16 KV is the sweet spot.** 6/10 runs found all 4 correct songs, 0 XML leaks, 131s average. Best balance of accuracy, speed, and reliability.
|
||||
|
||||
2. **Larger models use tools more effectively.** 27B and 35B-A3B models used 13-17 tool calls per query (vs 2-4 for 4B/9B), performing multiple searches and URL fetches to find the answer.
|
||||
|
||||
3. **27B Q8_0 had the highest raw accuracy (6/10 all-4/4) but lower reliability** -- only 4/10 OK runs due to timeouts on long agentic chains. The UD-Q4_K_XL quant is more practical.
|
||||
|
||||
4. **4B models were fastest (5-10s) but least accurate.** They occasionally found all 4 songs (2/10 best case) when they happened to fetch the right Wikipedia page.
|
||||
|
||||
5. **9B was surprisingly weaker than 4B on this task.** It used fewer tool calls and rarely extracted song data from fetched pages. The 9B model may need higher temperature or different prompting for this specific task type.
|
||||
|
||||
6. **35B-A3B had reliability issues.** Most runs timed out or errored due to slow per-token generation with many tool iterations. When it completed (2-6/10 OK), accuracy was comparable to 27B.
|
||||
|
||||
7. **bf16 KV cache had mixed effects.** For 27B it improved both speed (131s vs 202s) and accuracy (6/10 vs 5/10 all-4/4). For smaller models it had no consistent benefit.
|
||||
|
||||
8. **XML leaks are nearly eliminated.** 0/10 for all 4B and 9B configs, and only 1-2/10 for the largest models (which generate much more text in complex agentic loops).
|
||||
|
||||
## Before vs After (4B UD-Q4_K_XL, f16 KV)
|
||||
|
||||
| Metric | Before Changes | After Changes |
|
||||
|--------|---------------|---------------|
|
||||
| XML leaks | 10/10 | 0/10 |
|
||||
| URL fetches | 0/10 | 4/10 |
|
||||
| Peak3 accuracy | 0.0/4 | 0.8/4 |
|
||||
| Runs with all 4 songs | 0/10 | 2/10 |
|
||||
| Avg time | 12.3s | 9.8s |
|
||||
|
|
@ -17,6 +17,7 @@ import structlog
|
|||
from loggers import get_logger
|
||||
|
||||
from utils.models.model_config import load_model_defaults
|
||||
from utils.paths import is_local_path, normalize_path
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
|
@ -93,8 +94,28 @@ def _has_specific_yaml(model_identifier: str) -> bool:
|
|||
if model_identifier.lower() in _REVERSE_MODEL_MAPPING:
|
||||
return True
|
||||
|
||||
# Check for exact filename match
|
||||
model_filename = model_identifier.replace("/", "_") + ".yaml"
|
||||
# For local filesystem paths (e.g. C:\Users\...\model on Windows),
|
||||
# normalize backslashes so Path().parts splits correctly on POSIX/WSL,
|
||||
# then try matching the last 1-2 path components against the registry
|
||||
# (mirrors the logic in load_model_defaults).
|
||||
_is_local = is_local_path(model_identifier)
|
||||
_normalized = normalize_path(model_identifier) if _is_local else model_identifier
|
||||
|
||||
if _is_local:
|
||||
parts = Path(_normalized).parts
|
||||
for depth in (2, 1):
|
||||
if len(parts) >= depth:
|
||||
suffix = "/".join(parts[-depth:])
|
||||
if suffix.lower() in _REVERSE_MODEL_MAPPING:
|
||||
return True
|
||||
_lookup = Path(_normalized).name
|
||||
else:
|
||||
_lookup = model_identifier
|
||||
|
||||
# Check for exact filename match (basename for local paths to avoid
|
||||
# passing absolute paths into rglob which raises
|
||||
# "Non-relative patterns are unsupported" on Windows).
|
||||
model_filename = _lookup.replace("/", "_") + ".yaml"
|
||||
for config_path in defaults_dir.rglob(model_filename):
|
||||
if config_path.is_file():
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -1420,17 +1420,20 @@ def load_model_defaults(model_name: str) -> Dict[str, Any]:
|
|||
return config
|
||||
|
||||
# If model_name is a local path (e.g. /home/.../Spark-TTS-0.5B/LLM from
|
||||
# adapter_config.json), try matching the last 1-2 path components against
|
||||
# the registry (e.g. "Spark-TTS-0.5B/LLM").
|
||||
if model_name not in _REVERSE_MODEL_MAPPING and (
|
||||
model_name.startswith("/") or model_name.startswith(".")
|
||||
):
|
||||
parts = Path(model_name).parts
|
||||
# adapter_config.json, or C:\Users\...\model on Windows), try matching
|
||||
# the last 1-2 path components against the registry
|
||||
# (e.g. "Spark-TTS-0.5B/LLM").
|
||||
_is_local_path = is_local_path(model_name)
|
||||
# Normalize Windows backslash paths so Path().parts splits correctly
|
||||
# on POSIX/WSL hosts (pathlib treats backslashes as literals on Linux).
|
||||
_normalized = normalize_path(model_name) if _is_local_path else model_name
|
||||
if model_name.lower() not in _REVERSE_MODEL_MAPPING and _is_local_path:
|
||||
parts = Path(_normalized).parts
|
||||
for depth in [2, 1]:
|
||||
if len(parts) >= depth:
|
||||
suffix = "/".join(parts[-depth:])
|
||||
if suffix in _REVERSE_MODEL_MAPPING:
|
||||
canonical_file = _REVERSE_MODEL_MAPPING[suffix]
|
||||
if suffix.lower() in _REVERSE_MODEL_MAPPING:
|
||||
canonical_file = _REVERSE_MODEL_MAPPING[suffix.lower()]
|
||||
for config_path in defaults_dir.rglob(canonical_file):
|
||||
if config_path.is_file():
|
||||
with open(config_path, "r", encoding = "utf-8") as f:
|
||||
|
|
@ -1440,8 +1443,12 @@ def load_model_defaults(model_name: str) -> Dict[str, Any]:
|
|||
)
|
||||
return config
|
||||
|
||||
# Try exact model name match (for backward compatibility)
|
||||
model_filename = model_name.replace("/", "_") + ".yaml"
|
||||
# Try exact model name match (for backward compatibility).
|
||||
# For local filesystem paths, use only the directory basename to
|
||||
# avoid passing absolute paths (e.g. C:\...) into rglob which
|
||||
# raises "Non-relative patterns are unsupported" on Windows.
|
||||
_lookup_name = Path(_normalized).name if _is_local_path else model_name
|
||||
model_filename = _lookup_name.replace("/", "_") + ".yaml"
|
||||
# Search in subfolders and root
|
||||
for config_path in defaults_dir.rglob(model_filename):
|
||||
if config_path.is_file():
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ interface ModelSelectorProps {
|
|||
activeGgufVariant?: string | null;
|
||||
onValueChange?: (value: string, meta: ModelSelectorChangeMeta) => void;
|
||||
onEject?: () => void;
|
||||
onFoldersChange?: () => void;
|
||||
variant?: "outline" | "ghost" | "muted";
|
||||
size?: "sm" | "default" | "lg";
|
||||
className?: string;
|
||||
|
|
@ -100,6 +101,7 @@ function ModelSelectorContent({
|
|||
value,
|
||||
onSelect,
|
||||
onEject,
|
||||
onFoldersChange,
|
||||
className,
|
||||
dataTour,
|
||||
}: {
|
||||
|
|
@ -108,6 +110,7 @@ function ModelSelectorContent({
|
|||
value?: string;
|
||||
onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
|
||||
onEject?: () => void;
|
||||
onFoldersChange?: () => void;
|
||||
className?: string;
|
||||
dataTour?: string;
|
||||
}) {
|
||||
|
|
@ -124,7 +127,7 @@ function ModelSelectorContent({
|
|||
)}
|
||||
>
|
||||
{chatOnly ? (
|
||||
<HubModelPicker models={models} value={value} onSelect={onSelect} />
|
||||
<HubModelPicker models={models} value={value} onSelect={onSelect} onFoldersChange={onFoldersChange} />
|
||||
) : (
|
||||
<Tabs defaultValue="hub" className="w-full">
|
||||
<TabsList className="mb-2 w-full">
|
||||
|
|
@ -133,7 +136,7 @@ function ModelSelectorContent({
|
|||
</TabsList>
|
||||
|
||||
<TabsContent value="hub" className="m-0">
|
||||
<HubModelPicker models={models} value={value} onSelect={onSelect} />
|
||||
<HubModelPicker models={models} value={value} onSelect={onSelect} onFoldersChange={onFoldersChange} />
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="lora" className="m-0">
|
||||
|
|
@ -171,6 +174,7 @@ export function ModelSelector({
|
|||
activeGgufVariant,
|
||||
onValueChange,
|
||||
onEject,
|
||||
onFoldersChange,
|
||||
variant = "outline",
|
||||
size = "default",
|
||||
className,
|
||||
|
|
@ -253,6 +257,7 @@ export function ModelSelector({
|
|||
value={selected}
|
||||
onSelect={handleSelect}
|
||||
onEject={onEject ? handleEject : undefined}
|
||||
onFoldersChange={onFoldersChange}
|
||||
className={contentClassName}
|
||||
dataTour={contentDataTour}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -18,10 +18,24 @@ import {
|
|||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { deleteCachedModel, listCachedGguf, listCachedModels, listGgufVariants, listLocalModels } from "@/features/chat/api/chat-api";
|
||||
import type { CachedGgufRepo, CachedModelRepo, LocalModelInfo } from "@/features/chat/api/chat-api";
|
||||
import type { GgufVariantDetail } from "@/features/chat/types/api";
|
||||
import { usePlatformStore } from "@/config/env";
|
||||
import {
|
||||
type ScanFolderInfo,
|
||||
addScanFolder,
|
||||
deleteCachedModel,
|
||||
listCachedGguf,
|
||||
listCachedModels,
|
||||
listGgufVariants,
|
||||
listLocalModels,
|
||||
listScanFolders,
|
||||
removeScanFolder,
|
||||
} from "@/features/chat/api/chat-api";
|
||||
import type {
|
||||
CachedGgufRepo,
|
||||
CachedModelRepo,
|
||||
LocalModelInfo,
|
||||
} from "@/features/chat/api/chat-api";
|
||||
import type { GgufVariantDetail } from "@/features/chat/types/api";
|
||||
import {
|
||||
useDebouncedValue,
|
||||
useGpuInfo,
|
||||
|
|
@ -32,10 +46,16 @@ import {
|
|||
import { cn, formatCompact } from "@/lib/utils";
|
||||
import type { VramFitStatus } from "@/lib/vram";
|
||||
import { checkVramFit, estimateLoadingVram } from "@/lib/vram";
|
||||
import { Search01Icon } from "@hugeicons/core-free-icons";
|
||||
import { Add01Icon, Cancel01Icon, Folder02Icon, Search01Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { Trash2Icon } from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react";
|
||||
import {
|
||||
type ReactNode,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
import { toast } from "sonner";
|
||||
import type {
|
||||
LoraModelOption,
|
||||
|
|
@ -107,23 +127,22 @@ function ModelRow({
|
|||
className={cn(
|
||||
"flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left text-sm transition-colors hover:bg-accent",
|
||||
selected && "bg-accent/60",
|
||||
exceeds && "opacity-50",
|
||||
)}
|
||||
>
|
||||
<span
|
||||
className={cn(
|
||||
"block min-w-0 flex-1 truncate",
|
||||
exceeds && "line-through decoration-muted-foreground/50",
|
||||
exceeds && "!text-gray-500 dark:!text-gray-400",
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
<span className="ml-auto flex items-center gap-1.5 shrink-0">
|
||||
{vramStatus === "exceeds" && (
|
||||
<span className="text-[9px] font-medium text-red-400">OOM</span>
|
||||
<span className="text-[9px] font-medium !text-red-700 !bg-red-50 dark:!text-red-400 dark:!bg-red-950 px-1.5 py-0.5 rounded">OOM</span>
|
||||
)}
|
||||
{vramStatus === "tight" && (
|
||||
<span className="text-[9px] font-medium text-amber-400">TIGHT</span>
|
||||
<span className="text-[9px] font-medium !text-amber-400">TIGHT</span>
|
||||
)}
|
||||
{meta ? (
|
||||
<span className="text-[10px] text-muted-foreground">{meta}</span>
|
||||
|
|
@ -135,7 +154,7 @@ function ModelRow({
|
|||
if (vramTooltipText) {
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>{content}</TooltipTrigger>
|
||||
<TooltipTrigger asChild={true}>{content}</TooltipTrigger>
|
||||
<TooltipContent side="left" className="max-w-xs break-all">
|
||||
{label}
|
||||
<span className="block text-[10px] mt-1">{vramTooltipText}</span>
|
||||
|
|
@ -147,7 +166,7 @@ function ModelRow({
|
|||
if (tooltipText) {
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>{content}</TooltipTrigger>
|
||||
<TooltipTrigger asChild={true}>{content}</TooltipTrigger>
|
||||
<TooltipContent side="left" className="max-w-xs break-all">
|
||||
{tooltipText}
|
||||
</TooltipContent>
|
||||
|
|
@ -192,7 +211,9 @@ function GgufVariantExpander({
|
|||
})
|
||||
.catch((err) => {
|
||||
if (canceled) return;
|
||||
setError(err instanceof Error ? err.message : "Failed to load variants");
|
||||
setError(
|
||||
err instanceof Error ? err.message : "Failed to load variants",
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
if (!canceled) setLoading(false);
|
||||
|
|
@ -204,7 +225,9 @@ function GgufVariantExpander({
|
|||
}, [repoId]);
|
||||
|
||||
// Covers Unix absolute (/), Windows drive (C:\, D:/), UNC (\\server), relative (./, ../), tilde (~/)
|
||||
const isLocalPath = /^(\/|\.{1,2}[\\\/]|~[\\\/]|[A-Za-z]:[\\\/]|\\\\)/.test(repoId);
|
||||
const isLocalPath = /^(\/|\.{1,2}[\\\/]|~[\\\/]|[A-Za-z]:[\\\/]|\\\\)/.test(
|
||||
repoId,
|
||||
);
|
||||
|
||||
const handleVariantClick = useCallback(
|
||||
(quant: string, downloaded?: boolean, sizeBytes?: number) => {
|
||||
|
|
@ -223,13 +246,13 @@ function GgufVariantExpander({
|
|||
// fits = model <= 0.7 * total GPU memory
|
||||
// tight = model > 0.7 * GPU but <= 0.7 * GPU + 0.7 * system RAM (--fit uses CPU offload)
|
||||
// oom = model > 0.7 * GPU + 0.7 * system RAM
|
||||
const gpuBudgetGb = (gpuGb ?? 0) * 0.70;
|
||||
const totalBudgetGb = gpuBudgetGb + (systemRamGb ?? 0) * 0.70;
|
||||
const gpuBudgetGb = (gpuGb ?? 0) * 0.7;
|
||||
const totalBudgetGb = gpuBudgetGb + (systemRamGb ?? 0) * 0.7;
|
||||
|
||||
const getGgufFit = useCallback(
|
||||
(sizeBytes: number): "fits" | "tight" | "oom" => {
|
||||
if (!gpuGb || gpuGb <= 0) return "fits";
|
||||
const gb = sizeBytes / (1024 ** 3);
|
||||
const gb = sizeBytes / 1024 ** 3;
|
||||
if (gb <= 0 || gb <= gpuBudgetGb) return "fits";
|
||||
if (gb <= totalBudgetGb) return "tight";
|
||||
return "oom";
|
||||
|
|
@ -242,7 +265,8 @@ function GgufVariantExpander({
|
|||
const effectiveRecommended = useMemo(() => {
|
||||
if (!variants || !gpuGb || gpuGb <= 0) return defaultVariant;
|
||||
const defaultV = variants.find((v) => v.quant === defaultVariant);
|
||||
if (defaultV && getGgufFit(defaultV.size_bytes) !== "oom") return defaultVariant;
|
||||
if (defaultV && getGgufFit(defaultV.size_bytes) !== "oom")
|
||||
return defaultVariant;
|
||||
// Default is OOM -- pick largest non-OOM variant (best quality that fits)
|
||||
const fitting = variants.filter((v) => getGgufFit(v.size_bytes) !== "oom");
|
||||
if (fitting.length > 0) {
|
||||
|
|
@ -276,7 +300,9 @@ function GgufVariantExpander({
|
|||
// fits: largest first (best quality that fits in GPU)
|
||||
// tight/OOM: smallest first (closest to fitting, fastest to run)
|
||||
const fitsInGpu = aTier === 0 || aTier === 2;
|
||||
return fitsInGpu ? b.size_bytes - a.size_bytes : a.size_bytes - b.size_bytes;
|
||||
return fitsInGpu
|
||||
? b.size_bytes - a.size_bytes
|
||||
: a.size_bytes - b.size_bytes;
|
||||
});
|
||||
}, [variants, effectiveRecommended, getGgufFit]);
|
||||
|
||||
|
|
@ -290,9 +316,7 @@ function GgufVariantExpander({
|
|||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className="px-5 py-2 text-xs text-destructive">{error}</div>
|
||||
);
|
||||
return <div className="px-5 py-2 text-xs text-destructive">{error}</div>;
|
||||
}
|
||||
|
||||
if (!sortedVariants || sortedVariants.length === 0) {
|
||||
|
|
@ -321,13 +345,15 @@ function GgufVariantExpander({
|
|||
<div key={v.filename} className="flex items-center gap-0.5">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleVariantClick(v.quant, v.downloaded, v.size_bytes)}
|
||||
onClick={() =>
|
||||
handleVariantClick(v.quant, v.downloaded, v.size_bytes)
|
||||
}
|
||||
className={cn(
|
||||
"flex min-w-0 flex-1 items-center justify-between gap-2 rounded-md px-2.5 py-1 text-left text-sm transition-colors hover:bg-accent",
|
||||
)}
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-xs">
|
||||
{v.quant}
|
||||
<span className={cn(oom && "!text-gray-500 dark:!text-gray-400")}>{v.quant}</span>
|
||||
{v.downloaded ? (
|
||||
<span className="ml-1.5 text-[9px] font-sans font-medium text-green-400">
|
||||
downloaded
|
||||
|
|
@ -340,10 +366,14 @@ function GgufVariantExpander({
|
|||
</span>
|
||||
<span className="flex items-center gap-1.5 shrink-0">
|
||||
{oom && (
|
||||
<span className="text-[9px] font-medium text-red-400">OOM</span>
|
||||
<span className="text-[9px] font-medium !text-red-700 !bg-red-50 dark:!text-red-400 dark:!bg-red-950 px-1.5 py-0.5 rounded">
|
||||
OOM
|
||||
</span>
|
||||
)}
|
||||
{tight && (
|
||||
<span className="text-[9px] font-medium text-amber-400">TIGHT</span>
|
||||
<span className="text-[9px] font-medium !text-amber-400">
|
||||
TIGHT
|
||||
</span>
|
||||
)}
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{formatBytes(v.size_bytes)}
|
||||
|
|
@ -353,7 +383,10 @@ function GgufVariantExpander({
|
|||
{v.downloaded && onDeleteVariant && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => { e.stopPropagation(); onDeleteVariant(v.quant); }}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onDeleteVariant(v.quant);
|
||||
}}
|
||||
className="shrink-0 rounded-md p-1 text-muted-foreground/60 transition-colors hover:bg-destructive/10 hover:text-destructive"
|
||||
>
|
||||
<Trash2Icon className="size-3" />
|
||||
|
|
@ -384,6 +417,8 @@ function extractParamLabel(id: string): string | undefined {
|
|||
let _cachedGgufCache: CachedGgufRepo[] = [];
|
||||
let _cachedModelsCache: CachedModelRepo[] = [];
|
||||
let _lmStudioCache: LocalModelInfo[] = [];
|
||||
let _customFolderCache: LocalModelInfo[] = [];
|
||||
let _scanFoldersCache: ScanFolderInfo[] = [];
|
||||
|
||||
/** Sort LM Studio models with unsloth publisher first. */
|
||||
function sortLmStudio(models: LocalModelInfo[]): LocalModelInfo[] {
|
||||
|
|
@ -391,7 +426,9 @@ function sortLmStudio(models: LocalModelInfo[]): LocalModelInfo[] {
|
|||
const aUnsloth = (a.model_id ?? "").startsWith("unsloth/") ? 0 : 1;
|
||||
const bUnsloth = (b.model_id ?? "").startsWith("unsloth/") ? 0 : 1;
|
||||
if (aUnsloth !== bUnsloth) return aUnsloth - bUnsloth;
|
||||
return (a.model_id ?? a.display_name).localeCompare(b.model_id ?? b.display_name);
|
||||
return (a.model_id ?? a.display_name).localeCompare(
|
||||
b.model_id ?? b.display_name,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -401,17 +438,18 @@ export function HubModelPicker({
|
|||
models,
|
||||
value,
|
||||
onSelect,
|
||||
onFoldersChange,
|
||||
}: {
|
||||
models: ModelOption[];
|
||||
value?: string;
|
||||
onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
|
||||
onFoldersChange?: () => void;
|
||||
}) {
|
||||
const gpu = useGpuInfo();
|
||||
const [query, setQuery] = useState("");
|
||||
const debouncedQuery = useDebouncedValue(query);
|
||||
const { results, isLoading, isLoadingMore, fetchMore } = useHfModelSearch(
|
||||
debouncedQuery,
|
||||
);
|
||||
const { results, isLoading, isLoadingMore, fetchMore } =
|
||||
useHfModelSearch(debouncedQuery);
|
||||
|
||||
// Track which GGUF repo is expanded for variant selection
|
||||
const [expandedGguf, setExpandedGguf] = useState<string | null>(null);
|
||||
|
|
@ -422,39 +460,135 @@ export function HubModelPicker({
|
|||
|
||||
// Cached (already downloaded) repos -- use module-level cache so
|
||||
// re-mounting the popover does not flash an empty "Downloaded" section.
|
||||
const [cachedGguf, setCachedGguf] = useState<CachedGgufRepo[]>(_cachedGgufCache);
|
||||
const [cachedModels, setCachedModels] = useState<CachedModelRepo[]>(_cachedModelsCache);
|
||||
const alreadyCached = _cachedGgufCache.length > 0 || _cachedModelsCache.length > 0;
|
||||
const [cachedGguf, setCachedGguf] =
|
||||
useState<CachedGgufRepo[]>(_cachedGgufCache);
|
||||
const [cachedModels, setCachedModels] =
|
||||
useState<CachedModelRepo[]>(_cachedModelsCache);
|
||||
const alreadyCached =
|
||||
_cachedGgufCache.length > 0 || _cachedModelsCache.length > 0;
|
||||
const [cachedReady, setCachedReady] = useState(alreadyCached);
|
||||
|
||||
// LM Studio local models -- module-level cache so re-mounting the
|
||||
// popover does not flash an empty section (same pattern as GGUF/models).
|
||||
const [lmStudioModels, setLmStudioModels] = useState<LocalModelInfo[]>(_lmStudioCache);
|
||||
const [lmStudioModels, setLmStudioModels] =
|
||||
useState<LocalModelInfo[]>(_lmStudioCache);
|
||||
const [customFolderModels, setCustomFolderModels] =
|
||||
useState<LocalModelInfo[]>(_customFolderCache);
|
||||
|
||||
const refreshCachedLists = useCallback(() => {
|
||||
listCachedGguf().then((v) => { _cachedGgufCache = v; setCachedGguf(v); }).catch(() => {});
|
||||
listCachedModels().then((v) => { _cachedModelsCache = v; setCachedModels(v); }).catch(() => {});
|
||||
listLocalModels().then((res) => {
|
||||
const next = sortLmStudio(res.models.filter((m) => m.source === "lmstudio"));
|
||||
_lmStudioCache = next;
|
||||
setLmStudioModels(next);
|
||||
}).catch(() => {});
|
||||
// Custom scan folders management
|
||||
const [scanFolders, setScanFolders] = useState<ScanFolderInfo[]>(_scanFoldersCache);
|
||||
const [folderInput, setFolderInput] = useState("");
|
||||
const [folderError, setFolderError] = useState<string | null>(null);
|
||||
const [showFolderInput, setShowFolderInput] = useState(false);
|
||||
const [folderLoading, setFolderLoading] = useState(false);
|
||||
|
||||
const refreshLocalModelsList = useCallback(() => {
|
||||
listLocalModels()
|
||||
.then((res) => {
|
||||
const lm = sortLmStudio(
|
||||
res.models.filter((m) => m.source === "lmstudio"),
|
||||
);
|
||||
_lmStudioCache = lm;
|
||||
setLmStudioModels(lm);
|
||||
const cf = res.models.filter((m) => m.source === "custom");
|
||||
_customFolderCache = cf;
|
||||
setCustomFolderModels(cf);
|
||||
})
|
||||
.catch(() => {});
|
||||
}, []);
|
||||
|
||||
const refreshScanFolders = useCallback(() => {
|
||||
listScanFolders()
|
||||
.then((v) => {
|
||||
_scanFoldersCache = v;
|
||||
setScanFolders(v);
|
||||
})
|
||||
.catch(() => {});
|
||||
}, []);
|
||||
|
||||
const handleAddFolder = useCallback(async () => {
|
||||
const trimmed = folderInput.trim();
|
||||
if (!trimmed || folderLoading) return;
|
||||
setFolderError(null);
|
||||
setFolderLoading(true);
|
||||
try {
|
||||
const created = await addScanFolder(trimmed);
|
||||
// Backend returns existing row for duplicates, so deduplicate
|
||||
const next = _scanFoldersCache.some((f) => f.id === created.id || f.path === created.path)
|
||||
? _scanFoldersCache
|
||||
: [..._scanFoldersCache, created];
|
||||
_scanFoldersCache = next;
|
||||
setScanFolders(next);
|
||||
setFolderInput("");
|
||||
setShowFolderInput(false);
|
||||
refreshLocalModelsList();
|
||||
onFoldersChange?.();
|
||||
// Background reconciliation with the server
|
||||
void refreshScanFolders();
|
||||
} catch (e) {
|
||||
setFolderError(e instanceof Error ? e.message : "Failed to add folder");
|
||||
} finally {
|
||||
setFolderLoading(false);
|
||||
}
|
||||
}, [folderInput, folderLoading, refreshScanFolders, refreshLocalModelsList, onFoldersChange]);
|
||||
|
||||
const handleRemoveFolder = useCallback(async (id: number) => {
|
||||
try {
|
||||
await removeScanFolder(id);
|
||||
// Optimistic update so the folder disappears immediately
|
||||
const next = _scanFoldersCache.filter((f) => f.id !== id);
|
||||
_scanFoldersCache = next;
|
||||
setScanFolders(next);
|
||||
refreshScanFolders();
|
||||
refreshLocalModelsList();
|
||||
onFoldersChange?.();
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Failed to remove folder");
|
||||
refreshScanFolders();
|
||||
}
|
||||
}, [refreshScanFolders, refreshLocalModelsList, onFoldersChange]);
|
||||
|
||||
const refreshCachedLists = useCallback(() => {
|
||||
listCachedGguf()
|
||||
.then((v) => {
|
||||
_cachedGgufCache = v;
|
||||
setCachedGguf(v);
|
||||
})
|
||||
.catch(() => {});
|
||||
listCachedModels()
|
||||
.then((v) => {
|
||||
_cachedModelsCache = v;
|
||||
setCachedModels(v);
|
||||
})
|
||||
.catch(() => {});
|
||||
refreshLocalModelsList();
|
||||
}, [refreshLocalModelsList]);
|
||||
|
||||
useEffect(() => {
|
||||
// Always refresh LM Studio models (not gated by alreadyCached)
|
||||
listLocalModels().then((res) => {
|
||||
const next = sortLmStudio(res.models.filter((m) => m.source === "lmstudio"));
|
||||
_lmStudioCache = next;
|
||||
setLmStudioModels(next);
|
||||
}).catch(() => {});
|
||||
// Always refresh LM Studio + custom folder models (not gated by alreadyCached)
|
||||
refreshLocalModelsList();
|
||||
refreshScanFolders();
|
||||
|
||||
if (alreadyCached) return;
|
||||
let done = 0;
|
||||
const check = () => { if (++done >= 2) setCachedReady(true); };
|
||||
listCachedGguf().then((v) => { _cachedGgufCache = v; setCachedGguf(v); }).catch(() => {}).finally(check);
|
||||
listCachedModels().then((v) => { _cachedModelsCache = v; setCachedModels(v); }).catch(() => {}).finally(check);
|
||||
}, [alreadyCached]);
|
||||
const check = () => {
|
||||
if (++done >= 2) setCachedReady(true);
|
||||
};
|
||||
listCachedGguf()
|
||||
.then((v) => {
|
||||
_cachedGgufCache = v;
|
||||
setCachedGguf(v);
|
||||
})
|
||||
.catch(() => {})
|
||||
.finally(check);
|
||||
listCachedModels()
|
||||
.then((v) => {
|
||||
_cachedModelsCache = v;
|
||||
setCachedModels(v);
|
||||
})
|
||||
.catch(() => {})
|
||||
.finally(check);
|
||||
}, [alreadyCached, refreshLocalModelsList, refreshScanFolders]);
|
||||
|
||||
const handleDeleteConfirm = useCallback(async () => {
|
||||
if (!deleteTarget) return;
|
||||
|
|
@ -468,7 +602,9 @@ export function HubModelPicker({
|
|||
toast.success(`Deleted ${variant ? `${repoId} ${variant}` : repoId}`);
|
||||
refreshCachedLists();
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : "Failed to delete model");
|
||||
toast.error(
|
||||
err instanceof Error ? err.message : "Failed to delete model",
|
||||
);
|
||||
} finally {
|
||||
setDeleting(false);
|
||||
setDeleteTarget(null);
|
||||
|
|
@ -504,12 +640,18 @@ export function HubModelPicker({
|
|||
// Infinite scroll paging for the recommended section
|
||||
const [recommendedPage, setRecommendedPage] = useState(1);
|
||||
// Reset page when the underlying list changes
|
||||
useEffect(() => { setRecommendedPage(1); }, [models, chatOnly]);
|
||||
useEffect(() => {
|
||||
setRecommendedPage(1);
|
||||
}, [models, chatOnly]);
|
||||
|
||||
const visibleRecommendedIds = useMemo(() => {
|
||||
const hubStartIndex = recommendedIds.findIndex((id) => !isGgufRepo(id));
|
||||
const allGguf = hubStartIndex === -1 ? recommendedIds : recommendedIds.slice(0, hubStartIndex);
|
||||
const allHub = hubStartIndex === -1 ? [] : recommendedIds.slice(hubStartIndex);
|
||||
const allGguf =
|
||||
hubStartIndex === -1
|
||||
? recommendedIds
|
||||
: recommendedIds.slice(0, hubStartIndex);
|
||||
const allHub =
|
||||
hubStartIndex === -1 ? [] : recommendedIds.slice(hubStartIndex);
|
||||
// Interleave in chunks of 4: [4 gguf, 4 hub, 4 gguf, 4 hub, ...]
|
||||
const result: string[] = [];
|
||||
for (let p = 0; p < recommendedPage; p++) {
|
||||
|
|
@ -519,7 +661,8 @@ export function HubModelPicker({
|
|||
return result;
|
||||
}, [recommendedIds, recommendedPage]);
|
||||
|
||||
const hasMoreRecommended = visibleRecommendedIds.length < recommendedIds.length;
|
||||
const hasMoreRecommended =
|
||||
visibleRecommendedIds.length < recommendedIds.length;
|
||||
|
||||
const showHfSection = debouncedQuery.trim().length > 0;
|
||||
|
||||
|
|
@ -544,7 +687,8 @@ export function HubModelPicker({
|
|||
useRecommendedModelVram(idsForVram);
|
||||
|
||||
const recommendedSet = useMemo(
|
||||
() => new Set(showHfSection ? filteredRecommendedIds : visibleRecommendedIds),
|
||||
() =>
|
||||
new Set(showHfSection ? filteredRecommendedIds : visibleRecommendedIds),
|
||||
[showHfSection, filteredRecommendedIds, visibleRecommendedIds],
|
||||
);
|
||||
|
||||
|
|
@ -610,15 +754,25 @@ export function HubModelPicker({
|
|||
}
|
||||
}
|
||||
return map;
|
||||
}, [showHfSection, filteredRecommendedIds, visibleRecommendedIds, recommendedParamCountById, gpu]);
|
||||
}, [
|
||||
showHfSection,
|
||||
filteredRecommendedIds,
|
||||
visibleRecommendedIds,
|
||||
recommendedParamCountById,
|
||||
gpu,
|
||||
]);
|
||||
|
||||
const { scrollRef, sentinelRef } = useInfiniteScroll(fetchMore, results.length);
|
||||
const { scrollRef, sentinelRef } = useInfiniteScroll(
|
||||
fetchMore,
|
||||
results.length,
|
||||
);
|
||||
|
||||
// Sentinel + IntersectionObserver for recommended infinite scroll.
|
||||
// We disconnect after each fire so the observer doesn't loop while
|
||||
// React re-renders; the effect re-creates it on the next page.
|
||||
// Uses a callback ref for the sentinel so we detect mount/unmount reliably.
|
||||
const [recommendedSentinel, setRecommendedSentinel] = useState<HTMLDivElement | null>(null);
|
||||
const [recommendedSentinel, setRecommendedSentinel] =
|
||||
useState<HTMLDivElement | null>(null);
|
||||
const recommendedSentinelRef = useCallback((node: HTMLDivElement | null) => {
|
||||
setRecommendedSentinel(node);
|
||||
}, []);
|
||||
|
|
@ -637,7 +791,10 @@ export function HubModelPicker({
|
|||
);
|
||||
// Small delay so the browser finishes layout after the previous page render
|
||||
const timer = setTimeout(() => obs.observe(recommendedSentinel), 100);
|
||||
return () => { clearTimeout(timer); obs.disconnect(); };
|
||||
return () => {
|
||||
clearTimeout(timer);
|
||||
obs.disconnect();
|
||||
};
|
||||
}, [recommendedSentinel, hasMoreRecommended, recommendedPage, scrollRef]);
|
||||
|
||||
/** Handle clicking a model row — GGUF repos expand, others load directly. */
|
||||
|
|
@ -676,9 +833,13 @@ export function HubModelPicker({
|
|||
{!cachedReady && !showHfSection ? (
|
||||
<div className="flex items-center gap-2 px-5 py-3">
|
||||
<Spinner className="size-3 text-muted-foreground" />
|
||||
<span className="text-xs text-muted-foreground">Loading models…</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Loading models…
|
||||
</span>
|
||||
</div>
|
||||
) : !showHfSection && (cachedGguf.length > 0 || (!chatOnly && cachedModels.length > 0)) ? (
|
||||
) : !showHfSection &&
|
||||
(cachedGguf.length > 0 ||
|
||||
(!chatOnly && cachedModels.length > 0)) ? (
|
||||
<>
|
||||
<ListLabel>{"\uD83E\uDDA5"} Downloaded</ListLabel>
|
||||
{cachedGguf.map((c) => (
|
||||
|
|
@ -695,32 +856,46 @@ export function HubModelPicker({
|
|||
repoId={c.repo_id}
|
||||
onSelect={onSelect}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={gpu.available ? gpu.systemRamAvailableGb : undefined}
|
||||
onDeleteVariant={(quant) => setDeleteTarget(`${c.repo_id}::${quant}`)}
|
||||
systemRamGb={
|
||||
gpu.available ? gpu.systemRamAvailableGb : undefined
|
||||
}
|
||||
onDeleteVariant={(quant) =>
|
||||
setDeleteTarget(`${c.repo_id}::${quant}`)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{!chatOnly && cachedModels.map((c) => (
|
||||
<div key={c.repo_id} className="flex items-center gap-0.5">
|
||||
<div className="min-w-0 flex-1">
|
||||
<ModelRow
|
||||
label={c.repo_id}
|
||||
meta={formatBytes(c.size_bytes)}
|
||||
selected={value === c.repo_id}
|
||||
onClick={() => onSelect(c.repo_id, { source: "hub", isLora: false, isDownloaded: true })}
|
||||
vramStatus={null}
|
||||
/>
|
||||
{!chatOnly &&
|
||||
cachedModels.map((c) => (
|
||||
<div key={c.repo_id} className="flex items-center gap-0.5">
|
||||
<div className="min-w-0 flex-1">
|
||||
<ModelRow
|
||||
label={c.repo_id}
|
||||
meta={formatBytes(c.size_bytes)}
|
||||
selected={value === c.repo_id}
|
||||
onClick={() =>
|
||||
onSelect(c.repo_id, {
|
||||
source: "hub",
|
||||
isLora: false,
|
||||
isDownloaded: true,
|
||||
})
|
||||
}
|
||||
vramStatus={null}
|
||||
/>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setDeleteTarget(c.repo_id);
|
||||
}}
|
||||
className="shrink-0 rounded-md p-1.5 text-muted-foreground/60 transition-colors hover:bg-destructive/10 hover:text-destructive"
|
||||
>
|
||||
<Trash2Icon className="size-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => { e.stopPropagation(); setDeleteTarget(c.repo_id); }}
|
||||
className="shrink-0 rounded-md p-1.5 text-muted-foreground/60 transition-colors hover:bg-destructive/10 hover:text-destructive"
|
||||
>
|
||||
<Trash2Icon className="size-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
))}
|
||||
</>
|
||||
) : null}
|
||||
|
||||
|
|
@ -733,13 +908,21 @@ export function HubModelPicker({
|
|||
<div key={m.id}>
|
||||
<ModelRow
|
||||
label={m.model_id ?? m.display_name}
|
||||
meta={isGguf || m.path.endsWith(".gguf") ? "GGUF" : "Local"}
|
||||
meta={
|
||||
isGguf || m.path.endsWith(".gguf") ? "GGUF" : "Local"
|
||||
}
|
||||
selected={value === m.id}
|
||||
onClick={() => {
|
||||
if (isGguf) {
|
||||
setExpandedGguf((prev) => (prev === m.id ? null : m.id));
|
||||
setExpandedGguf((prev) =>
|
||||
prev === m.id ? null : m.id,
|
||||
);
|
||||
} else {
|
||||
onSelect(m.id, { source: "local", isLora: false, isDownloaded: true });
|
||||
onSelect(m.id, {
|
||||
source: "local",
|
||||
isLora: false,
|
||||
isDownloaded: true,
|
||||
});
|
||||
}
|
||||
}}
|
||||
vramStatus={null}
|
||||
|
|
@ -749,7 +932,140 @@ export function HubModelPicker({
|
|||
repoId={m.id}
|
||||
onSelect={onSelect}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={gpu.available ? gpu.systemRamAvailableGb : undefined}
|
||||
systemRamGb={
|
||||
gpu.available ? gpu.systemRamAvailableGb : undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</>
|
||||
) : null}
|
||||
|
||||
{!showHfSection ? (
|
||||
<>
|
||||
<div className="flex items-center justify-between px-2.5 py-1.5">
|
||||
<span className="text-[10px] font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
Custom Folders
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
aria-label={showFolderInput ? "Cancel adding folder" : "Add scan folder"}
|
||||
onClick={() => {
|
||||
setShowFolderInput((open) => {
|
||||
if (open) { setFolderInput(""); setFolderError(null); }
|
||||
return !open;
|
||||
});
|
||||
}}
|
||||
className="rounded p-0.5 text-muted-foreground/60 transition-colors hover:text-foreground"
|
||||
>
|
||||
<HugeiconsIcon icon={showFolderInput ? Cancel01Icon : Add01Icon} className="size-3" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Folder paths */}
|
||||
{scanFolders.map((f) => (
|
||||
<div
|
||||
key={f.id}
|
||||
className="group flex items-center gap-1.5 px-3 py-0.5"
|
||||
>
|
||||
<HugeiconsIcon icon={Folder02Icon} className="size-3 shrink-0 text-muted-foreground/40" />
|
||||
<span
|
||||
className="min-w-0 flex-1 truncate font-mono text-[10px] text-muted-foreground/70"
|
||||
title={f.path}
|
||||
>
|
||||
{f.path}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleRemoveFolder(f.id)}
|
||||
aria-label={`Remove folder ${f.path}`}
|
||||
className="shrink-0 rounded p-0.5 text-muted-foreground/40 opacity-100 md:opacity-0 md:group-hover:opacity-100 focus-visible:opacity-100 transition-opacity hover:text-destructive"
|
||||
>
|
||||
<HugeiconsIcon icon={Cancel01Icon} className="size-2.5" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* Add folder input */}
|
||||
{showFolderInput && (
|
||||
<div className="px-2.5 pb-1 pt-0.5">
|
||||
<div className="flex items-center gap-1">
|
||||
<HugeiconsIcon icon={Folder02Icon} className="size-3 shrink-0 text-muted-foreground/40" />
|
||||
<input
|
||||
value={folderInput}
|
||||
onChange={(e) => { setFolderInput(e.target.value); setFolderError(null); }}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") { e.preventDefault(); handleAddFolder(); }
|
||||
if (e.key === "Escape") { e.preventDefault(); e.stopPropagation(); setShowFolderInput(false); setFolderInput(""); setFolderError(null); }
|
||||
}}
|
||||
placeholder="/path/to/models"
|
||||
className="h-6 min-w-0 flex-1 rounded border border-border/50 bg-transparent px-1.5 font-mono text-[10px] text-foreground outline-none placeholder:text-muted-foreground/40 focus:border-foreground/20"
|
||||
disabled={folderLoading}
|
||||
autoFocus={true}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleAddFolder}
|
||||
disabled={folderLoading || !folderInput.trim()}
|
||||
className="h-6 shrink-0 rounded border border-border/50 px-1.5 text-[10px] text-muted-foreground transition-colors hover:bg-accent disabled:opacity-40"
|
||||
>
|
||||
Add
|
||||
</button>
|
||||
</div>
|
||||
{folderError && (
|
||||
<p className="px-0.5 pt-0.5 text-[10px] text-destructive">{folderError}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Empty state */}
|
||||
{scanFolders.length === 0 && customFolderModels.length === 0 && !showFolderInput && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowFolderInput(true)}
|
||||
className="px-2.5 pb-1.5 text-left text-[10px] text-muted-foreground/60 transition-colors hover:text-muted-foreground"
|
||||
>
|
||||
+ Add a folder to scan for local models
|
||||
</button>
|
||||
)}
|
||||
|
||||
{/* Models from custom folders */}
|
||||
{customFolderModels.map((m) => {
|
||||
const isGguf =
|
||||
isGgufRepo(m.id) ||
|
||||
isGgufRepo(m.display_name) ||
|
||||
m.path.endsWith(".gguf");
|
||||
return (
|
||||
<div key={m.id}>
|
||||
<ModelRow
|
||||
label={m.model_id ?? m.display_name}
|
||||
meta={isGguf ? "GGUF" : "Local"}
|
||||
selected={value === m.id}
|
||||
onClick={() => {
|
||||
if (isGguf) {
|
||||
setExpandedGguf((prev) =>
|
||||
prev === m.id ? null : m.id,
|
||||
);
|
||||
} else {
|
||||
onSelect(m.id, {
|
||||
source: "local",
|
||||
isLora: false,
|
||||
isDownloaded: true,
|
||||
});
|
||||
}
|
||||
}}
|
||||
vramStatus={null}
|
||||
/>
|
||||
{expandedGguf === m.id && (
|
||||
<GgufVariantExpander
|
||||
repoId={m.id}
|
||||
onSelect={onSelect}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={
|
||||
gpu.available ? gpu.systemRamAvailableGb : undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -775,16 +1091,25 @@ export function HubModelPicker({
|
|||
meta={
|
||||
isGgufRepo(id)
|
||||
? "GGUF"
|
||||
: vram?.detail ?? extractParamLabel(id)
|
||||
: (vram?.detail ?? extractParamLabel(id))
|
||||
}
|
||||
selected={value === id}
|
||||
onClick={() => handleModelClick(id)}
|
||||
vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
|
||||
vramStatus={
|
||||
isGgufRepo(id) ? null : (vram?.status ?? null)
|
||||
}
|
||||
vramEst={isGgufRepo(id) ? undefined : vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} systemRamGb={gpu.available ? gpu.systemRamAvailableGb : undefined} />
|
||||
<GgufVariantExpander
|
||||
repoId={id}
|
||||
onSelect={onSelect}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={
|
||||
gpu.available ? gpu.systemRamAvailableGb : undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -813,16 +1138,25 @@ export function HubModelPicker({
|
|||
meta={
|
||||
isGgufRepo(id)
|
||||
? "GGUF"
|
||||
: vram?.detail ?? extractParamLabel(id)
|
||||
: (vram?.detail ?? extractParamLabel(id))
|
||||
}
|
||||
selected={value === id}
|
||||
onClick={() => handleModelClick(id)}
|
||||
vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
|
||||
vramStatus={
|
||||
isGgufRepo(id) ? null : (vram?.status ?? null)
|
||||
}
|
||||
vramEst={isGgufRepo(id) ? undefined : vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} systemRamGb={gpu.available ? gpu.systemRamAvailableGb : undefined} />
|
||||
<GgufVariantExpander
|
||||
repoId={id}
|
||||
onSelect={onSelect}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={
|
||||
gpu.available ? gpu.systemRamAvailableGb : undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -832,7 +1166,9 @@ export function HubModelPicker({
|
|||
|
||||
{showHfSection ? (
|
||||
<>
|
||||
{(hfIds.length > 0 || isLoading) && <ListLabel>Hugging Face</ListLabel>}
|
||||
{(hfIds.length > 0 || isLoading) && (
|
||||
<ListLabel>Hugging Face</ListLabel>
|
||||
)}
|
||||
{hfIds.length === 0 && !isLoading ? (
|
||||
filteredRecommendedIds.length === 0 ? (
|
||||
<div className="px-2.5 py-2 text-xs text-muted-foreground">
|
||||
|
|
@ -849,16 +1185,25 @@ export function HubModelPicker({
|
|||
meta={
|
||||
isGgufRepo(id)
|
||||
? "GGUF"
|
||||
: metricsById.get(id) ?? extractParamLabel(id)
|
||||
: (metricsById.get(id) ?? extractParamLabel(id))
|
||||
}
|
||||
selected={value === id}
|
||||
onClick={() => handleModelClick(id)}
|
||||
vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
|
||||
vramStatus={
|
||||
isGgufRepo(id) ? null : (vram?.status ?? null)
|
||||
}
|
||||
vramEst={isGgufRepo(id) ? undefined : vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} systemRamGb={gpu.available ? gpu.systemRamAvailableGb : undefined} />
|
||||
<GgufVariantExpander
|
||||
repoId={id}
|
||||
onSelect={onSelect}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={
|
||||
gpu.available ? gpu.systemRamAvailableGb : undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -875,12 +1220,23 @@ export function HubModelPicker({
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<AlertDialog open={deleteTarget !== null} onOpenChange={(open) => { if (!open && !deleting) setDeleteTarget(null); }}>
|
||||
<AlertDialog
|
||||
open={deleteTarget !== null}
|
||||
onOpenChange={(open) => {
|
||||
if (!open && !deleting) setDeleteTarget(null);
|
||||
}}
|
||||
>
|
||||
<AlertDialogContent size="sm">
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Delete cached model?</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
This will remove <span className="font-medium text-foreground">{deleteTarget?.includes("::") ? `${deleteTarget.split("::")[0]} (${deleteTarget.split("::")[1]})` : deleteTarget}</span> from disk. You can re-download it later.
|
||||
This will remove{" "}
|
||||
<span className="font-medium text-foreground">
|
||||
{deleteTarget?.includes("::")
|
||||
? `${deleteTarget.split("::")[0]} (${deleteTarget.split("::")[1]})`
|
||||
: deleteTarget}
|
||||
</span>{" "}
|
||||
from disk. You can re-download it later.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
|
|
@ -888,7 +1244,10 @@ export function HubModelPicker({
|
|||
<AlertDialogAction
|
||||
variant="destructive"
|
||||
disabled={deleting}
|
||||
onClick={(e) => { e.preventDefault(); handleDeleteConfirm(); }}
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
handleDeleteConfirm();
|
||||
}}
|
||||
>
|
||||
{deleting ? "Deleting..." : "Yes"}
|
||||
</AlertDialogAction>
|
||||
|
|
@ -917,7 +1276,8 @@ export function LoraModelPicker({
|
|||
loraModels
|
||||
.map((model) => ({
|
||||
...model,
|
||||
baseModel: model.baseModel || model.description || "Unknown base model",
|
||||
baseModel:
|
||||
model.baseModel || model.description || "Unknown base model",
|
||||
}))
|
||||
.sort((a, b) => {
|
||||
const baseCmp = a.baseModel.localeCompare(b.baseModel);
|
||||
|
|
@ -941,7 +1301,9 @@ export function LoraModelPicker({
|
|||
const out = new Map<string, LoraModelOption[]>();
|
||||
|
||||
for (const model of normalized) {
|
||||
const searchText = normalizeForSearch(`${model.name} ${model.baseModel} ${model.id}`);
|
||||
const searchText = normalizeForSearch(
|
||||
`${model.name} ${model.baseModel} ${model.id}`,
|
||||
);
|
||||
if (needle && !searchText.includes(needle)) continue;
|
||||
|
||||
const key = model.baseModel || "Unknown base model";
|
||||
|
|
@ -989,15 +1351,27 @@ export function LoraModelPicker({
|
|||
const isExported = adapter.source === "exported";
|
||||
const isMerged = adapter.exportType === "merged";
|
||||
const isGguf = adapter.exportType === "gguf";
|
||||
const isLocalGgufDir = isLocal && (isGgufRepo(adapter.id) || isGgufRepo(adapter.name));
|
||||
const isLocalGgufDir =
|
||||
isLocal &&
|
||||
(isGgufRepo(adapter.id) || isGgufRepo(adapter.name));
|
||||
const tag = isLocal
|
||||
? isLocalGgufDir ? "GGUF" : "Local"
|
||||
? isLocalGgufDir
|
||||
? "GGUF"
|
||||
: "Local"
|
||||
: isGguf
|
||||
? "GGUF"
|
||||
: isExported
|
||||
? isMerged ? "Merged" : "LoRA"
|
||||
? isMerged
|
||||
? "Merged"
|
||||
: "LoRA"
|
||||
: "LoRA";
|
||||
const meta = isLocal ? (isLocalGgufDir ? "GGUF" : "Local") : isExported ? `${tag} · Exported` : tag;
|
||||
const meta = isLocal
|
||||
? isLocalGgufDir
|
||||
? "GGUF"
|
||||
: "Local"
|
||||
: isExported
|
||||
? `${tag} · Exported`
|
||||
: tag;
|
||||
return (
|
||||
<div key={adapter.id}>
|
||||
<ModelRow
|
||||
|
|
@ -1006,10 +1380,16 @@ export function LoraModelPicker({
|
|||
selected={value === adapter.id}
|
||||
onClick={() => {
|
||||
if (isLocalGgufDir) {
|
||||
setExpandedGguf((prev) => (prev === adapter.id ? null : adapter.id));
|
||||
setExpandedGguf((prev) =>
|
||||
prev === adapter.id ? null : adapter.id,
|
||||
);
|
||||
} else {
|
||||
onSelect(adapter.id, {
|
||||
source: isLocal ? "local" : isExported ? "exported" : "lora",
|
||||
source: isLocal
|
||||
? "local"
|
||||
: isExported
|
||||
? "exported"
|
||||
: "lora",
|
||||
isLora: !isLocal && !isMerged && !isGguf,
|
||||
isDownloaded: true,
|
||||
});
|
||||
|
|
@ -1017,7 +1397,9 @@ export function LoraModelPicker({
|
|||
}}
|
||||
tooltipText={
|
||||
<>
|
||||
<span className="block break-words">{adapter.name}</span>
|
||||
<span className="block break-words">
|
||||
{adapter.name}
|
||||
</span>
|
||||
<span className="block mt-1 text-[10px] text-muted-foreground break-all">
|
||||
{adapter.id}
|
||||
</span>
|
||||
|
|
@ -1029,7 +1411,9 @@ export function LoraModelPicker({
|
|||
repoId={adapter.id}
|
||||
onSelect={onSelect}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={gpu.available ? gpu.systemRamAvailableGb : undefined}
|
||||
systemRamGb={
|
||||
gpu.available ? gpu.systemRamAvailableGb : undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -437,21 +437,40 @@ const CodeToolsToggle: FC = () => {
|
|||
|
||||
const ToolStatusDisplay: FC = () => {
|
||||
const toolStatus = useChatRuntimeStore((s) => s.toolStatus);
|
||||
const isThreadRunning = useAuiState(({ thread }) => thread.isRunning);
|
||||
const [elapsed, setElapsed] = useState(0);
|
||||
const [visible, setVisible] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!toolStatus) {
|
||||
setElapsed(0);
|
||||
if (!isThreadRunning) {
|
||||
setVisible(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
setElapsed(0);
|
||||
|
||||
// Debounce badge visibility by 300ms when the badge is not
|
||||
// already on screen. Once visible from a prior tool, consecutive
|
||||
// tools show immediately so the badge does not flicker. Fast
|
||||
// tool calls that all complete under 300ms never show the badge.
|
||||
let showTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
if (!visible) {
|
||||
showTimer = setTimeout(() => setVisible(true), 300);
|
||||
}
|
||||
|
||||
const interval = setInterval(() => {
|
||||
setElapsed((prev) => prev + 1);
|
||||
}, 1000);
|
||||
return () => clearInterval(interval);
|
||||
}, [toolStatus]);
|
||||
return () => {
|
||||
clearInterval(interval);
|
||||
if (showTimer) clearTimeout(showTimer);
|
||||
};
|
||||
}, [toolStatus, isThreadRunning]);
|
||||
|
||||
if (!toolStatus) return null;
|
||||
if (!toolStatus || !visible) return null;
|
||||
const isRunning = toolStatus.startsWith("Running");
|
||||
const StatusIcon = isRunning ? TerminalIcon : GlobeIcon;
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ export interface LocalModelInfo {
|
|||
id: string;
|
||||
display_name: string;
|
||||
path: string;
|
||||
source: "models_dir" | "hf_cache" | "lmstudio";
|
||||
source: "models_dir" | "hf_cache" | "lmstudio" | "custom";
|
||||
model_id?: string | null;
|
||||
updated_at?: number | null;
|
||||
}
|
||||
|
|
@ -174,6 +174,34 @@ export async function deleteCachedModel(repoId: string, variant?: string): Promi
|
|||
await parseJsonOrThrow<unknown>(response);
|
||||
}
|
||||
|
||||
export interface ScanFolderInfo {
|
||||
id: number;
|
||||
path: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export async function listScanFolders(): Promise<ScanFolderInfo[]> {
|
||||
const response = await authFetch("/api/models/scan-folders");
|
||||
const data = await parseJsonOrThrow<{ folders: ScanFolderInfo[] }>(response);
|
||||
return data.folders;
|
||||
}
|
||||
|
||||
export async function addScanFolder(path: string): Promise<ScanFolderInfo> {
|
||||
const response = await authFetch("/api/models/scan-folders", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ path }),
|
||||
});
|
||||
return parseJsonOrThrow<ScanFolderInfo>(response);
|
||||
}
|
||||
|
||||
export async function removeScanFolder(id: number): Promise<void> {
|
||||
const response = await authFetch(`/api/models/scan-folders/${id}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
await parseJsonOrThrow<unknown>(response);
|
||||
}
|
||||
|
||||
export async function listGgufVariants(
|
||||
repoId: string,
|
||||
hfToken?: string,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import {
|
|||
} from "@/components/assistant-ui/model-selector";
|
||||
import { Thread } from "@/components/assistant-ui/thread";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { SidebarProvider, SidebarTrigger, useSidebar } from "@/components/ui/sidebar";
|
||||
import {
|
||||
Sheet,
|
||||
SheetContent,
|
||||
|
|
@ -16,7 +15,17 @@ import {
|
|||
SheetHeader,
|
||||
SheetTitle,
|
||||
} from "@/components/ui/sheet";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import {
|
||||
SidebarProvider,
|
||||
SidebarTrigger,
|
||||
useSidebar,
|
||||
} from "@/components/ui/sidebar";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { GuidedTour, useGuidedTourController } from "@/features/tour";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
ColumnInsertIcon,
|
||||
|
|
@ -36,7 +45,6 @@ import {
|
|||
useState,
|
||||
} from "react";
|
||||
import { toast } from "sonner";
|
||||
import { GuidedTour, useGuidedTourController } from "@/features/tour";
|
||||
import { listLocalModels } from "./api/chat-api";
|
||||
import { ChatSettingsPanel } from "./chat-settings-sheet";
|
||||
import { ContextUsageBar } from "./components/context-usage-bar";
|
||||
|
|
@ -48,16 +56,16 @@ import {
|
|||
getTrainingCompareHandoff,
|
||||
} from "./lib/training-compare-handoff";
|
||||
import { ChatRuntimeProvider } from "./runtime-provider";
|
||||
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
|
||||
import {
|
||||
type CompareHandle,
|
||||
CompareHandlesProvider,
|
||||
RegisterCompareHandle,
|
||||
SharedComposer,
|
||||
} from "./shared-composer";
|
||||
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
|
||||
import { ThreadSidebar } from "./thread-sidebar";
|
||||
import type { ChatView, MessageRecord } from "./types";
|
||||
import { buildChatTourSteps } from "./tour";
|
||||
import type { ChatView, MessageRecord } from "./types";
|
||||
|
||||
type LoraCandidate = {
|
||||
id: string;
|
||||
|
|
@ -101,7 +109,9 @@ function messageHasImage(message: MessageRecord): boolean {
|
|||
if (contentParts.some((part) => part.type === "image")) {
|
||||
return true;
|
||||
}
|
||||
const attachments = Array.isArray(message.attachments) ? message.attachments : [];
|
||||
const attachments = Array.isArray(message.attachments)
|
||||
? message.attachments
|
||||
: [];
|
||||
for (const attachment of attachments) {
|
||||
const parts = Array.isArray(attachment.content) ? attachment.content : [];
|
||||
for (const part of parts as Array<{ type?: string }>) {
|
||||
|
|
@ -152,12 +162,25 @@ const CompareContent = memo(function CompareContent({
|
|||
pairId,
|
||||
models,
|
||||
loraModels,
|
||||
}: { pairId: string; models: ModelOption[]; loraModels: LoraModelOption[] }): ReactElement {
|
||||
onFoldersChange,
|
||||
}: {
|
||||
pairId: string;
|
||||
models: ModelOption[];
|
||||
loraModels: LoraModelOption[];
|
||||
onFoldersChange?: () => void;
|
||||
}): ReactElement {
|
||||
const isLoraCompare = useIsLoraCompare();
|
||||
|
||||
return isLoraCompare
|
||||
? <LoraCompareContent pairId={pairId} />
|
||||
: <GeneralCompareContent pairId={pairId} models={models} loraModels={loraModels} />;
|
||||
return isLoraCompare ? (
|
||||
<LoraCompareContent pairId={pairId} />
|
||||
) : (
|
||||
<GeneralCompareContent
|
||||
pairId={pairId}
|
||||
models={models}
|
||||
loraModels={loraModels}
|
||||
onFoldersChange={onFoldersChange}
|
||||
/>
|
||||
);
|
||||
});
|
||||
|
||||
/** Fast path: same model, adapter on/off, simultaneous generation. */
|
||||
|
|
@ -179,7 +202,9 @@ const LoraCompareContent = memo(function LoraCompareContent({
|
|||
setBaseThreadId(threads.find((t) => t.modelType === "base")?.id);
|
||||
setLoraThreadId(threads.find((t) => t.modelType === "lora")?.id);
|
||||
});
|
||||
return () => { isActive = false; };
|
||||
return () => {
|
||||
isActive = false;
|
||||
};
|
||||
}, [pairId]);
|
||||
|
||||
return (
|
||||
|
|
@ -196,7 +221,11 @@ const LoraCompareContent = memo(function LoraCompareContent({
|
|||
</span>
|
||||
</div>
|
||||
<div className="min-h-0 flex-1">
|
||||
<ChatRuntimeProvider modelType="base" pairId={pairId} initialThreadId={baseThreadId}>
|
||||
<ChatRuntimeProvider
|
||||
modelType="base"
|
||||
pairId={pairId}
|
||||
initialThreadId={baseThreadId}
|
||||
>
|
||||
<RegisterCompareHandle name="base" />
|
||||
<Thread hideComposer={true} hideWelcome={true} />
|
||||
</ChatRuntimeProvider>
|
||||
|
|
@ -209,7 +238,11 @@ const LoraCompareContent = memo(function LoraCompareContent({
|
|||
</span>
|
||||
</div>
|
||||
<div className="min-h-0 flex-1">
|
||||
<ChatRuntimeProvider modelType="lora" pairId={pairId} initialThreadId={loraThreadId}>
|
||||
<ChatRuntimeProvider
|
||||
modelType="lora"
|
||||
pairId={pairId}
|
||||
initialThreadId={loraThreadId}
|
||||
>
|
||||
<RegisterCompareHandle name="lora" />
|
||||
<Thread hideComposer={true} hideWelcome={true} />
|
||||
</ChatRuntimeProvider>
|
||||
|
|
@ -229,7 +262,13 @@ const GeneralCompareContent = memo(function GeneralCompareContent({
|
|||
pairId,
|
||||
models,
|
||||
loraModels,
|
||||
}: { pairId: string; models: ModelOption[]; loraModels: LoraModelOption[] }): ReactElement {
|
||||
onFoldersChange,
|
||||
}: {
|
||||
pairId: string;
|
||||
models: ModelOption[];
|
||||
loraModels: LoraModelOption[];
|
||||
onFoldersChange?: () => void;
|
||||
}): ReactElement {
|
||||
const handlesRef = useRef<Record<string, CompareHandle>>({});
|
||||
const [model1ThreadId, setModel1ThreadId] = useState<string>();
|
||||
const [model2ThreadId, setModel2ThreadId] = useState<string>();
|
||||
|
|
@ -241,7 +280,10 @@ const GeneralCompareContent = memo(function GeneralCompareContent({
|
|||
isLora: false,
|
||||
ggufVariant: globalGgufVariant ?? undefined,
|
||||
});
|
||||
const [model2, setModel2] = useState<CompareModelSelection>({ id: "", isLora: false });
|
||||
const [model2, setModel2] = useState<CompareModelSelection>({
|
||||
id: "",
|
||||
isLora: false,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
let isActive = true;
|
||||
|
|
@ -252,13 +294,19 @@ const GeneralCompareContent = memo(function GeneralCompareContent({
|
|||
.then((threads) => {
|
||||
if (!isActive) return;
|
||||
setModel1ThreadId(
|
||||
threads.find((t) => t.modelType === "model1" || t.modelType === "base")?.id,
|
||||
threads.find(
|
||||
(t) => t.modelType === "model1" || t.modelType === "base",
|
||||
)?.id,
|
||||
);
|
||||
setModel2ThreadId(
|
||||
threads.find((t) => t.modelType === "model2" || t.modelType === "lora")?.id,
|
||||
threads.find(
|
||||
(t) => t.modelType === "model2" || t.modelType === "lora",
|
||||
)?.id,
|
||||
);
|
||||
});
|
||||
return () => { isActive = false; };
|
||||
return () => {
|
||||
isActive = false;
|
||||
};
|
||||
}, [pairId]);
|
||||
|
||||
return (
|
||||
|
|
@ -277,7 +325,14 @@ const GeneralCompareContent = memo(function GeneralCompareContent({
|
|||
models={models}
|
||||
loraModels={loraModels}
|
||||
value={model1.id}
|
||||
onValueChange={(id, meta) => setModel1({ id, isLora: meta.isLora, ggufVariant: meta.ggufVariant })}
|
||||
onValueChange={(id, meta) =>
|
||||
setModel1({
|
||||
id,
|
||||
isLora: meta.isLora,
|
||||
ggufVariant: meta.ggufVariant,
|
||||
})
|
||||
}
|
||||
onFoldersChange={onFoldersChange}
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="max-w-[50%]"
|
||||
|
|
@ -303,7 +358,14 @@ const GeneralCompareContent = memo(function GeneralCompareContent({
|
|||
models={models}
|
||||
loraModels={loraModels}
|
||||
value={model2.id}
|
||||
onValueChange={(id, meta) => setModel2({ id, isLora: meta.isLora, ggufVariant: meta.ggufVariant })}
|
||||
onValueChange={(id, meta) =>
|
||||
setModel2({
|
||||
id,
|
||||
isLora: meta.isLora,
|
||||
ggufVariant: meta.ggufVariant,
|
||||
})
|
||||
}
|
||||
onFoldersChange={onFoldersChange}
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="max-w-[50%]"
|
||||
|
|
@ -322,7 +384,11 @@ const GeneralCompareContent = memo(function GeneralCompareContent({
|
|||
</div>
|
||||
</div>
|
||||
<div className="mx-auto w-full max-w-4xl px-4 py-4">
|
||||
<SharedComposer handlesRef={handlesRef} model1={model1} model2={model2} />
|
||||
<SharedComposer
|
||||
handlesRef={handlesRef}
|
||||
model1={model1}
|
||||
model2={model2}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</CompareHandlesProvider>
|
||||
|
|
@ -364,8 +430,7 @@ function InlineSidebar({
|
|||
data-sidebar="sidebar"
|
||||
className={cn(
|
||||
"bg-muted/70 text-sidebar-foreground h-full overflow-hidden rounded-2xl corner-squircle transition-[width] duration-200 ease-linear",
|
||||
!collapsed &&
|
||||
side === "right" && "border-l border-sidebar-border/70",
|
||||
!collapsed && side === "right" && "border-l border-sidebar-border/70",
|
||||
collapsed ? "w-0" : "w-(--sidebar-width)",
|
||||
)}
|
||||
>
|
||||
|
|
@ -381,7 +446,11 @@ function TopBarActions({
|
|||
onNewThread,
|
||||
onNewCompare,
|
||||
showCompare,
|
||||
}: { onNewThread: () => void; onNewCompare: () => void; showCompare: boolean }) {
|
||||
}: {
|
||||
onNewThread: () => void;
|
||||
onNewCompare: () => void;
|
||||
showCompare: boolean;
|
||||
}) {
|
||||
const { state } = useSidebar();
|
||||
if (state !== "collapsed") {
|
||||
return null;
|
||||
|
|
@ -424,8 +493,12 @@ export function ChatPage(): ReactElement {
|
|||
);
|
||||
const inferenceParams = useChatRuntimeStore((state) => state.params);
|
||||
const setInferenceParams = useChatRuntimeStore((state) => state.setParams);
|
||||
const activeGgufVariant = useChatRuntimeStore((state) => state.activeGgufVariant);
|
||||
const ggufContextLength = useChatRuntimeStore((state) => state.ggufContextLength);
|
||||
const activeGgufVariant = useChatRuntimeStore(
|
||||
(state) => state.activeGgufVariant,
|
||||
);
|
||||
const ggufContextLength = useChatRuntimeStore(
|
||||
(state) => state.ggufContextLength,
|
||||
);
|
||||
const contextUsage = useChatRuntimeStore((state) => state.contextUsage);
|
||||
const autoTitle = useChatRuntimeStore((state) => state.autoTitle);
|
||||
const setAutoTitle = useChatRuntimeStore((state) => state.setAutoTitle);
|
||||
|
|
@ -441,8 +514,7 @@ export function ChatPage(): ReactElement {
|
|||
loadingModel,
|
||||
loadProgress,
|
||||
loadToastDismissed,
|
||||
} =
|
||||
useChatModelRuntime();
|
||||
} = useChatModelRuntime();
|
||||
const refreshRef = useRef(refresh);
|
||||
const selectModelRef = useRef(selectModel);
|
||||
|
||||
|
|
@ -455,11 +527,24 @@ export function ChatPage(): ReactElement {
|
|||
}, [inferenceParams.checkpoint]);
|
||||
|
||||
const handleCheckpointChange = useCallback(
|
||||
(value: string, meta?: { isLora: boolean; ggufVariant?: string; isDownloaded?: boolean; expectedBytes?: number }) => {
|
||||
(
|
||||
value: string,
|
||||
meta?: {
|
||||
isLora: boolean;
|
||||
ggufVariant?: string;
|
||||
isDownloaded?: boolean;
|
||||
expectedBytes?: number;
|
||||
},
|
||||
) => {
|
||||
const store = useChatRuntimeStore.getState();
|
||||
const currentCheckpoint = store.params.checkpoint;
|
||||
const currentVariant = store.activeGgufVariant;
|
||||
if (!value || (value === currentCheckpoint && (meta?.ggufVariant ?? null) === (currentVariant ?? null))) return;
|
||||
if (
|
||||
!value ||
|
||||
(value === currentCheckpoint &&
|
||||
(meta?.ggufVariant ?? null) === (currentVariant ?? null))
|
||||
)
|
||||
return;
|
||||
void (async () => {
|
||||
let showImageCompatibilityWarning = false;
|
||||
if (view.mode === "single" && activeThreadId) {
|
||||
|
|
@ -471,7 +556,9 @@ export function ChatPage(): ReactElement {
|
|||
.toArray();
|
||||
if (messages.length > 0) {
|
||||
const hasImage = messages.some(messageHasImage);
|
||||
const targetModel = modelsFromStore.find((model) => model.id === value);
|
||||
const targetModel = modelsFromStore.find(
|
||||
(model) => model.id === value,
|
||||
);
|
||||
showImageCompatibilityWarning =
|
||||
hasImage && targetModel?.isVision === false;
|
||||
}
|
||||
|
|
@ -499,20 +586,14 @@ export function ChatPage(): ReactElement {
|
|||
const handleEject = useCallback(() => {
|
||||
void ejectModel();
|
||||
}, [ejectModel]);
|
||||
const handleNewThread = useCallback(
|
||||
() => {
|
||||
useChatRuntimeStore.getState().setActiveThreadId(null);
|
||||
setView({ mode: "single", newThreadNonce: crypto.randomUUID() });
|
||||
},
|
||||
[],
|
||||
);
|
||||
const handleNewCompare = useCallback(
|
||||
() => {
|
||||
setView({ mode: "compare", pairId: crypto.randomUUID() });
|
||||
useChatRuntimeStore.getState().setContextUsage(null);
|
||||
},
|
||||
[],
|
||||
);
|
||||
const handleNewThread = useCallback(() => {
|
||||
useChatRuntimeStore.getState().setActiveThreadId(null);
|
||||
setView({ mode: "single", newThreadNonce: crypto.randomUUID() });
|
||||
}, []);
|
||||
const handleNewCompare = useCallback(() => {
|
||||
setView({ mode: "compare", pairId: crypto.randomUUID() });
|
||||
useChatRuntimeStore.getState().setContextUsage(null);
|
||||
}, []);
|
||||
|
||||
const openModelSelector = useCallback(() => {
|
||||
setModelSelectorLocked(true);
|
||||
|
|
@ -556,18 +637,17 @@ export function ChatPage(): ReactElement {
|
|||
.first()
|
||||
.then((msg) => {
|
||||
const saved = msg?.metadata as Record<string, unknown> | undefined;
|
||||
const usage = saved?.contextUsage as typeof store.contextUsage | undefined;
|
||||
const usage = saved?.contextUsage as
|
||||
| typeof store.contextUsage
|
||||
| undefined;
|
||||
if (usage) store.setContextUsage(usage);
|
||||
});
|
||||
}
|
||||
}, [viewBeforeCompare]);
|
||||
|
||||
const handleThreadSelect = useCallback(
|
||||
(nextView: ChatView) => {
|
||||
setView(nextView);
|
||||
},
|
||||
[],
|
||||
);
|
||||
const handleThreadSelect = useCallback((nextView: ChatView) => {
|
||||
setView(nextView);
|
||||
}, []);
|
||||
|
||||
const models = useMemo<ModelOption[]>(
|
||||
() =>
|
||||
|
|
@ -581,6 +661,37 @@ export function ChatPage(): ReactElement {
|
|||
|
||||
const [localModels, setLocalModels] = useState<LoraModelOption[]>([]);
|
||||
|
||||
const refreshLocalModels = useCallback(() => {
|
||||
void listLocalModels()
|
||||
.then((res) => {
|
||||
setLocalModels(
|
||||
res.models
|
||||
.filter(
|
||||
(m) =>
|
||||
m.source === "lmstudio" ||
|
||||
m.source === "models_dir" ||
|
||||
m.source === "custom",
|
||||
)
|
||||
.map((m) => ({
|
||||
id: m.id,
|
||||
name:
|
||||
m.source === "lmstudio" && m.model_id
|
||||
? m.model_id
|
||||
: m.display_name,
|
||||
baseModel:
|
||||
m.source === "lmstudio"
|
||||
? "LM Studio"
|
||||
: m.source === "custom"
|
||||
? "Custom Folders"
|
||||
: "Local models",
|
||||
updatedAt: m.updated_at ?? undefined,
|
||||
source: "local" as const,
|
||||
})),
|
||||
);
|
||||
})
|
||||
.catch(() => {});
|
||||
}, []);
|
||||
|
||||
const loraModels = useMemo<LoraModelOption[]>(() => {
|
||||
const fromLoras = lorasFromStore.map((lora) => ({
|
||||
id: lora.id,
|
||||
|
|
@ -596,20 +707,8 @@ export function ChatPage(): ReactElement {
|
|||
useEffect(() => {
|
||||
if (getTrainingCompareHandoff()) return;
|
||||
void refresh();
|
||||
void listLocalModels().then((res) => {
|
||||
setLocalModels(
|
||||
res.models
|
||||
.filter((m) => m.source === "lmstudio" || m.source === "models_dir")
|
||||
.map((m) => ({
|
||||
id: m.id,
|
||||
name: m.source === "lmstudio" && m.model_id ? m.model_id : m.display_name,
|
||||
baseModel: m.source === "lmstudio" ? "LM Studio" : "Local models",
|
||||
updatedAt: m.updated_at ?? undefined,
|
||||
source: "local" as const,
|
||||
})),
|
||||
);
|
||||
}).catch(() => {});
|
||||
}, [refresh]);
|
||||
refreshLocalModels();
|
||||
}, [refresh, refreshLocalModels]);
|
||||
|
||||
useEffect(() => {
|
||||
const handoff = getTrainingCompareHandoff();
|
||||
|
|
@ -649,7 +748,10 @@ export function ChatPage(): ReactElement {
|
|||
console.info("[chat-handoff] no lora match, loading base", {
|
||||
id: handoff.baseModel,
|
||||
});
|
||||
await selectModelRef.current({ id: handoff.baseModel, isLora: false });
|
||||
await selectModelRef.current({
|
||||
id: handoff.baseModel,
|
||||
isLora: false,
|
||||
});
|
||||
if (canceled) return;
|
||||
} else {
|
||||
console.warn("[chat-handoff] no lora/base match found", {
|
||||
|
|
@ -751,6 +853,7 @@ export function ChatPage(): ReactElement {
|
|||
activeGgufVariant={activeGgufVariant}
|
||||
onValueChange={handleCheckpointChange}
|
||||
onEject={handleEject}
|
||||
onFoldersChange={refreshLocalModels}
|
||||
variant="ghost"
|
||||
open={modelSelectorOpen}
|
||||
onOpenChange={handleModelSelectorOpenChange}
|
||||
|
|
@ -767,9 +870,11 @@ export function ChatPage(): ReactElement {
|
|||
? "Loading model…"
|
||||
: "Downloading model…"
|
||||
}
|
||||
title={loadingModel.isDownloaded
|
||||
? `Loading ${loadingModel.displayName} from cache.`
|
||||
: `Loading ${loadingModel.displayName}. This may include downloading.`}
|
||||
title={
|
||||
loadingModel.isDownloaded
|
||||
? `Loading ${loadingModel.displayName} from cache.`
|
||||
: `Loading ${loadingModel.displayName}. This may include downloading.`
|
||||
}
|
||||
progressPercent={loadProgress?.percent}
|
||||
progressLabel={loadProgress?.label}
|
||||
onStop={cancelLoading}
|
||||
|
|
@ -809,7 +914,13 @@ export function ChatPage(): ReactElement {
|
|||
newThreadNonce={view.newThreadNonce}
|
||||
/>
|
||||
) : (
|
||||
<CompareContent key={view.pairId} pairId={view.pairId} models={models} loraModels={loraModels} />
|
||||
<CompareContent
|
||||
key={view.pairId}
|
||||
pairId={view.pairId}
|
||||
models={models}
|
||||
loraModels={loraModels}
|
||||
onFoldersChange={refreshLocalModels}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,6 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Slider } from "@/components/ui/slider";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Dialog,
|
||||
|
|
@ -20,6 +10,25 @@ import {
|
|||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import {
|
||||
Sheet,
|
||||
SheetContent,
|
||||
SheetDescription,
|
||||
SheetHeader,
|
||||
SheetTitle,
|
||||
} from "@/components/ui/sheet";
|
||||
import { Slider } from "@/components/ui/slider";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { useIsMobile } from "@/hooks/use-mobile";
|
||||
import {
|
||||
ArrowDown01Icon,
|
||||
CodeIcon,
|
||||
|
|
@ -33,22 +42,13 @@ import {
|
|||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { AnimatePresence, motion } from "motion/react";
|
||||
import {
|
||||
Sheet,
|
||||
SheetContent,
|
||||
SheetDescription,
|
||||
SheetHeader,
|
||||
SheetTitle,
|
||||
} from "@/components/ui/sheet";
|
||||
import { useIsMobile } from "@/hooks/use-mobile";
|
||||
import type { ReactNode } from "react";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
|
||||
import {
|
||||
DEFAULT_INFERENCE_PARAMS,
|
||||
type InferenceParams,
|
||||
} from "./types/runtime";
|
||||
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
|
||||
export const defaultInferenceParams = DEFAULT_INFERENCE_PARAMS;
|
||||
export type { InferenceParams } from "./types/runtime";
|
||||
|
|
@ -174,7 +174,11 @@ function loadCollapsibleState(): Record<string, boolean> {
|
|||
const raw = localStorage.getItem(COLLAPSIBLE_STATE_KEY);
|
||||
if (!raw) return {};
|
||||
const parsed = JSON.parse(raw);
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
||||
if (
|
||||
typeof parsed !== "object" ||
|
||||
parsed === null ||
|
||||
Array.isArray(parsed)
|
||||
) {
|
||||
return {};
|
||||
}
|
||||
return Object.fromEntries(
|
||||
|
|
@ -277,12 +281,16 @@ export function ChatSettingsPanel({
|
|||
const isMobile = useIsMobile();
|
||||
const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null;
|
||||
const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength);
|
||||
const ggufMaxContextLength = useChatRuntimeStore((s) => s.ggufMaxContextLength);
|
||||
const ggufMaxContextLength = useChatRuntimeStore(
|
||||
(s) => s.ggufMaxContextLength,
|
||||
);
|
||||
const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);
|
||||
const setKvCacheDtype = useChatRuntimeStore((s) => s.setKvCacheDtype);
|
||||
const loadedKvCacheDtype = useChatRuntimeStore((s) => s.loadedKvCacheDtype);
|
||||
const customContextLength = useChatRuntimeStore((s) => s.customContextLength);
|
||||
const setCustomContextLength = useChatRuntimeStore((s) => s.setCustomContextLength);
|
||||
const setCustomContextLength = useChatRuntimeStore(
|
||||
(s) => s.setCustomContextLength,
|
||||
);
|
||||
|
||||
const ctxDisplayValue = customContextLength ?? ggufContextLength ?? "";
|
||||
const ctxMaxValue = ggufMaxContextLength ?? ggufContextLength ?? null;
|
||||
|
|
@ -292,7 +300,9 @@ export function ChatSettingsPanel({
|
|||
const [customPresets, setCustomPresets] = useState<Preset[]>(() =>
|
||||
loadSavedCustomPresets(),
|
||||
);
|
||||
const [activePreset, setActivePreset] = useState(() => loadSavedActivePreset());
|
||||
const [activePreset, setActivePreset] = useState(() =>
|
||||
loadSavedActivePreset(),
|
||||
);
|
||||
const [savePresetOpen, setSavePresetOpen] = useState(false);
|
||||
const [presetNameDraft, setPresetNameDraft] = useState("");
|
||||
const presets = useMemo(
|
||||
|
|
@ -417,325 +427,354 @@ export function ChatSettingsPanel({
|
|||
<div className="flex-1 overflow-y-auto px-1.5">
|
||||
{/* mt-4 matches the Playground sidebar gap (SidebarHeader py-3 + SidebarGroup pt-1) */}
|
||||
<div className="mt-4 px-2 pb-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<Select value={activePreset} onValueChange={applyPreset}>
|
||||
<SelectTrigger className="h-8 flex-1 corner-squircle text-xs">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{presets.map((p) => (
|
||||
<SelectItem key={p.name} value={p.name}>
|
||||
{p.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<button
|
||||
type="button"
|
||||
onClick={openSavePresetDialog}
|
||||
className="flex h-8 items-center gap-1.5 rounded-md border px-2.5 text-xs text-muted-foreground transition-colors hover:bg-accent"
|
||||
title="Save preset"
|
||||
>
|
||||
<HugeiconsIcon icon={FloppyDiskIcon} className="size-3.5" />
|
||||
Save
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => deletePreset(activePreset)}
|
||||
disabled={isBuiltinPreset}
|
||||
className="flex h-8 items-center gap-1.5 rounded-md border px-2.5 text-xs text-muted-foreground transition-colors hover:bg-accent disabled:cursor-not-allowed disabled:opacity-50"
|
||||
title={
|
||||
isBuiltinPreset
|
||||
? "Built-in presets cannot be deleted"
|
||||
: "Delete selected preset"
|
||||
}
|
||||
>
|
||||
<HugeiconsIcon icon={Delete02Icon} className="size-3.5" />
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="px-2 pb-4">
|
||||
<label
|
||||
htmlFor="system-prompt"
|
||||
className="mb-1.5 block text-xs font-medium"
|
||||
<div className="flex items-center gap-2">
|
||||
<Select value={activePreset} onValueChange={applyPreset}>
|
||||
<SelectTrigger className="h-8 flex-1 corner-squircle text-xs">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{presets.map((p) => (
|
||||
<SelectItem key={p.name} value={p.name}>
|
||||
{p.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<button
|
||||
type="button"
|
||||
onClick={openSavePresetDialog}
|
||||
className="flex h-8 items-center gap-1.5 rounded-md border px-2.5 text-xs text-muted-foreground transition-colors hover:bg-accent"
|
||||
title="Save preset"
|
||||
>
|
||||
System Prompt
|
||||
</label>
|
||||
<Textarea
|
||||
id="system-prompt"
|
||||
value={params.systemPrompt}
|
||||
onChange={(e) => set("systemPrompt")(e.target.value)}
|
||||
placeholder="You are a helpful assistant..."
|
||||
className="min-h-20 text-xs corner-squircle"
|
||||
rows={3}
|
||||
/>
|
||||
<HugeiconsIcon icon={FloppyDiskIcon} className="size-3.5" />
|
||||
Save
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => deletePreset(activePreset)}
|
||||
disabled={isBuiltinPreset}
|
||||
className="flex h-8 items-center gap-1.5 rounded-md border px-2.5 text-xs text-muted-foreground transition-colors hover:bg-accent disabled:cursor-not-allowed disabled:opacity-50"
|
||||
title={
|
||||
isBuiltinPreset
|
||||
? "Built-in presets cannot be deleted"
|
||||
: "Delete selected preset"
|
||||
}
|
||||
>
|
||||
<HugeiconsIcon icon={Delete02Icon} className="size-3.5" />
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<CollapsibleSection icon={Settings02Icon} label="Model" defaultOpen={true}>
|
||||
<div className="flex flex-col gap-3 py-1">
|
||||
{isGguf && (
|
||||
<>
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium">Context Length</span>
|
||||
<Input
|
||||
type="number"
|
||||
value={typeof ctxDisplayValue === "number" ? ctxDisplayValue : (ggufContextLength ?? "")}
|
||||
placeholder="..."
|
||||
min={128}
|
||||
max={ctxMaxValue ?? undefined}
|
||||
step={1024}
|
||||
className="h-6 w-[100px] text-right text-xs tabular-nums"
|
||||
onChange={(e) => {
|
||||
const raw = e.target.value;
|
||||
if (raw === "") {
|
||||
setCustomContextLength(null);
|
||||
return;
|
||||
}
|
||||
const v = parseInt(raw, 10);
|
||||
if (!Number.isNaN(v) && v >= 0) {
|
||||
const maxCtx = ctxMaxValue ?? Infinity;
|
||||
const clamped = Math.min(v, maxCtx);
|
||||
setCustomContextLength(clamped === (ggufContextLength ?? 0) ? null : clamped);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<Slider
|
||||
min={1024}
|
||||
max={ctxMaxValue ?? 4096}
|
||||
<div className="px-2 pb-4">
|
||||
<label
|
||||
htmlFor="system-prompt"
|
||||
className="mb-1.5 block text-xs font-medium"
|
||||
>
|
||||
System Prompt
|
||||
</label>
|
||||
<Textarea
|
||||
id="system-prompt"
|
||||
value={params.systemPrompt}
|
||||
onChange={(e) => set("systemPrompt")(e.target.value)}
|
||||
placeholder="You are a helpful assistant..."
|
||||
className="min-h-20 text-xs corner-squircle"
|
||||
rows={3}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<CollapsibleSection
|
||||
icon={Settings02Icon}
|
||||
label="Model"
|
||||
defaultOpen={true}
|
||||
>
|
||||
<div className="flex flex-col gap-3 py-1">
|
||||
{isGguf && (
|
||||
<>
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium">Context Length</span>
|
||||
<Input
|
||||
type="number"
|
||||
value={
|
||||
typeof ctxDisplayValue === "number"
|
||||
? ctxDisplayValue
|
||||
: (ggufContextLength ?? "")
|
||||
}
|
||||
placeholder="..."
|
||||
min={128}
|
||||
max={ctxMaxValue ?? undefined}
|
||||
step={1024}
|
||||
value={[Math.min(typeof ctxDisplayValue === "number" ? ctxDisplayValue : (ggufContextLength ?? 4096), ctxMaxValue ?? 4096)]}
|
||||
onValueChange={([v]) => {
|
||||
setCustomContextLength(v === (ggufContextLength ?? 0) ? null : v);
|
||||
className="h-6 w-[100px] text-right text-xs tabular-nums"
|
||||
onChange={(e) => {
|
||||
const raw = e.target.value;
|
||||
if (raw === "") {
|
||||
setCustomContextLength(null);
|
||||
return;
|
||||
}
|
||||
const v = Number.parseInt(raw, 10);
|
||||
if (!Number.isNaN(v) && v >= 0) {
|
||||
const maxCtx =
|
||||
ctxMaxValue ?? Number.POSITIVE_INFINITY;
|
||||
const clamped = Math.min(v, maxCtx);
|
||||
setCustomContextLength(
|
||||
clamped === (ggufContextLength ?? 0)
|
||||
? null
|
||||
: clamped,
|
||||
);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-xs font-medium">KV Cache Dtype</div>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Quantize KV cache to reduce VRAM.
|
||||
</div>
|
||||
</div>
|
||||
<Select
|
||||
value={kvCacheDtype ?? "f16"}
|
||||
onValueChange={(v) => {
|
||||
setKvCacheDtype(v === "f16" ? null : v);
|
||||
}}
|
||||
>
|
||||
<SelectTrigger className="h-7 w-[90px] text-xs">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="f16">f16</SelectItem>
|
||||
<SelectItem value="bf16">bf16</SelectItem>
|
||||
<SelectItem value="q8_0">q8_0</SelectItem>
|
||||
<SelectItem value="q5_1">q5_1</SelectItem>
|
||||
<SelectItem value="q4_1">q4_1</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
{modelSettingsDirty && (
|
||||
<div className="flex flex-wrap gap-1.5 pt-1">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onReloadModel?.()}
|
||||
className="rounded-md bg-primary px-2.5 py-1 text-[11px] font-medium text-primary-foreground transition-colors hover:bg-primary/90"
|
||||
>
|
||||
Apply
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setCustomContextLength(null);
|
||||
setKvCacheDtype(loadedKvCacheDtype);
|
||||
}}
|
||||
className="rounded-md border px-2.5 py-1 text-[11px] font-medium text-muted-foreground transition-colors hover:bg-accent"
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{!isGguf && params.checkpoint && (
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-xs font-medium">Enable custom code</div>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Allow models with custom code (e.g. Nemotron). Only enable if sure.
|
||||
</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={params.trustRemoteCode ?? false}
|
||||
onCheckedChange={set("trustRemoteCode")}
|
||||
<Slider
|
||||
min={1024}
|
||||
max={ctxMaxValue ?? 4096}
|
||||
step={1024}
|
||||
value={[
|
||||
Math.min(
|
||||
typeof ctxDisplayValue === "number"
|
||||
? ctxDisplayValue
|
||||
: (ggufContextLength ?? 4096),
|
||||
ctxMaxValue ?? 4096,
|
||||
),
|
||||
]}
|
||||
onValueChange={([v]) => {
|
||||
setCustomContextLength(
|
||||
v === (ggufContextLength ?? 0) ? null : v,
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
<CollapsibleSection
|
||||
icon={SlidersHorizontalIcon}
|
||||
label="Sampling"
|
||||
defaultOpen={true}
|
||||
>
|
||||
<div className="flex flex-col gap-5">
|
||||
<ParamSlider
|
||||
label="Temperature"
|
||||
value={params.temperature}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.1}
|
||||
onChange={set("temperature")}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Top P"
|
||||
value={params.topP}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.05}
|
||||
onChange={set("topP")}
|
||||
displayValue={params.topP === 1 ? "Off" : undefined}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Top K"
|
||||
value={params.topK}
|
||||
min={0}
|
||||
max={100}
|
||||
step={1}
|
||||
onChange={set("topK")}
|
||||
displayValue={params.topK === 0 ? "Off" : undefined}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Min P"
|
||||
value={params.minP}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
onChange={set("minP")}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Repetition Penalty"
|
||||
value={params.repetitionPenalty}
|
||||
min={1}
|
||||
max={2}
|
||||
step={0.05}
|
||||
onChange={set("repetitionPenalty")}
|
||||
displayValue={params.repetitionPenalty === 1 ? "Off" : undefined}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Presence Penalty"
|
||||
value={params.presencePenalty}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.1}
|
||||
onChange={set("presencePenalty")}
|
||||
displayValue={params.presencePenalty === 0 ? "Off" : undefined}
|
||||
/>
|
||||
{!isGguf && (
|
||||
<ParamSlider
|
||||
label="Max Seq Length"
|
||||
value={params.maxSeqLength}
|
||||
min={128}
|
||||
max={32768}
|
||||
step={128}
|
||||
onChange={set("maxSeqLength")}
|
||||
/>
|
||||
)}
|
||||
<ParamSlider
|
||||
label="Max Tokens"
|
||||
value={params.maxTokens}
|
||||
min={64}
|
||||
max={isGguf && ggufContextLength ? ggufContextLength : 32768}
|
||||
step={64}
|
||||
onChange={set("maxTokens")}
|
||||
displayValue={
|
||||
isGguf && ggufContextLength && params.maxTokens >= ggufContextLength
|
||||
? "Max"
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
<CollapsibleSection icon={Wrench01Icon} label="Tools">
|
||||
<div className="flex flex-col gap-3 py-1">
|
||||
<AutoHealToolCallsToggle />
|
||||
<MaxToolCallsSlider />
|
||||
<ToolCallTimeoutSlider />
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
<CollapsibleSection icon={UserSettings01Icon} label="Preferences" defaultOpen={true}>
|
||||
<div className="flex flex-col gap-3 py-1">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-xs font-medium">KV Cache Dtype</div>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Quantize KV cache to reduce VRAM.
|
||||
</div>
|
||||
</div>
|
||||
<Select
|
||||
value={kvCacheDtype ?? "f16"}
|
||||
onValueChange={(v) => {
|
||||
setKvCacheDtype(v === "f16" ? null : v);
|
||||
}}
|
||||
>
|
||||
<SelectTrigger className="h-7 w-[90px] text-xs">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="f16">f16</SelectItem>
|
||||
<SelectItem value="bf16">bf16</SelectItem>
|
||||
<SelectItem value="q8_0">q8_0</SelectItem>
|
||||
<SelectItem value="q5_1">q5_1</SelectItem>
|
||||
<SelectItem value="q4_1">q4_1</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
{modelSettingsDirty && (
|
||||
<div className="flex flex-wrap gap-1.5 pt-1">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onReloadModel?.()}
|
||||
className="rounded-md bg-primary px-2.5 py-1 text-[11px] font-medium text-primary-foreground transition-colors hover:bg-primary/90"
|
||||
>
|
||||
Apply
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setCustomContextLength(null);
|
||||
setKvCacheDtype(loadedKvCacheDtype);
|
||||
}}
|
||||
className="rounded-md border px-2.5 py-1 text-[11px] font-medium text-muted-foreground transition-colors hover:bg-accent"
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{!isGguf && params.checkpoint && (
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-xs font-medium">Auto title</div>
|
||||
<div className="text-xs font-medium">Enable custom code</div>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Generate short title after reply.
|
||||
Allow models with custom code (e.g. Nemotron). Only enable
|
||||
if sure.
|
||||
</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={autoTitle}
|
||||
onCheckedChange={onAutoTitleChange}
|
||||
checked={params.trustRemoteCode ?? false}
|
||||
onCheckedChange={set("trustRemoteCode")}
|
||||
/>
|
||||
</div>
|
||||
<HfTokenField />
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
)}
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
<ChatTemplateSection onReloadModel={onReloadModel} />
|
||||
</div>
|
||||
<Dialog
|
||||
open={savePresetOpen}
|
||||
onOpenChange={(nextOpen) => {
|
||||
setSavePresetOpen(nextOpen);
|
||||
if (!nextOpen) {
|
||||
setPresetNameDraft("");
|
||||
}
|
||||
}}
|
||||
<CollapsibleSection
|
||||
icon={SlidersHorizontalIcon}
|
||||
label="Sampling"
|
||||
defaultOpen={true}
|
||||
>
|
||||
<DialogContent className="corner-squircle sm:max-w-sm">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Save Preset</DialogTitle>
|
||||
<DialogDescription>
|
||||
Enter a name for this inference preset.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<form
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
savePresetWithName(presetNameDraft);
|
||||
}}
|
||||
className="space-y-4"
|
||||
>
|
||||
<Input
|
||||
autoFocus={true}
|
||||
value={presetNameDraft}
|
||||
onChange={(event) => setPresetNameDraft(event.target.value)}
|
||||
placeholder="Preset name"
|
||||
maxLength={80}
|
||||
<div className="flex flex-col gap-5">
|
||||
<ParamSlider
|
||||
label="Temperature"
|
||||
value={params.temperature}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.1}
|
||||
onChange={set("temperature")}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Top P"
|
||||
value={params.topP}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.05}
|
||||
onChange={set("topP")}
|
||||
displayValue={params.topP === 1 ? "Off" : undefined}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Top K"
|
||||
value={params.topK}
|
||||
min={0}
|
||||
max={100}
|
||||
step={1}
|
||||
onChange={set("topK")}
|
||||
displayValue={params.topK === 0 ? "Off" : undefined}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Min P"
|
||||
value={params.minP}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
onChange={set("minP")}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Repetition Penalty"
|
||||
value={params.repetitionPenalty}
|
||||
min={1}
|
||||
max={2}
|
||||
step={0.05}
|
||||
onChange={set("repetitionPenalty")}
|
||||
displayValue={params.repetitionPenalty === 1 ? "Off" : undefined}
|
||||
/>
|
||||
<ParamSlider
|
||||
label="Presence Penalty"
|
||||
value={params.presencePenalty}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.1}
|
||||
onChange={set("presencePenalty")}
|
||||
displayValue={params.presencePenalty === 0 ? "Off" : undefined}
|
||||
/>
|
||||
{!isGguf && (
|
||||
<ParamSlider
|
||||
label="Max Seq Length"
|
||||
value={params.maxSeqLength}
|
||||
min={128}
|
||||
max={32768}
|
||||
step={128}
|
||||
onChange={set("maxSeqLength")}
|
||||
/>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
onClick={() => setSavePresetOpen(false)}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button type="submit" disabled={presetNameDraft.trim().length === 0}>
|
||||
Save
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</>
|
||||
)}
|
||||
<ParamSlider
|
||||
label="Max Tokens"
|
||||
value={params.maxTokens}
|
||||
min={64}
|
||||
max={isGguf && ggufContextLength ? ggufContextLength : 32768}
|
||||
step={64}
|
||||
onChange={set("maxTokens")}
|
||||
displayValue={
|
||||
isGguf &&
|
||||
ggufContextLength &&
|
||||
params.maxTokens >= ggufContextLength
|
||||
? "Max"
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
<CollapsibleSection icon={Wrench01Icon} label="Tools">
|
||||
<div className="flex flex-col gap-3 py-1">
|
||||
<AutoHealToolCallsToggle />
|
||||
<MaxToolCallsSlider />
|
||||
<ToolCallTimeoutSlider />
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
<CollapsibleSection
|
||||
icon={UserSettings01Icon}
|
||||
label="Preferences"
|
||||
defaultOpen={true}
|
||||
>
|
||||
<div className="flex flex-col gap-3 py-1">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-xs font-medium">Auto title</div>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Generate short title after reply.
|
||||
</div>
|
||||
</div>
|
||||
<Switch checked={autoTitle} onCheckedChange={onAutoTitleChange} />
|
||||
</div>
|
||||
<HfTokenField />
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
|
||||
<ChatTemplateSection onReloadModel={onReloadModel} />
|
||||
</div>
|
||||
<Dialog
|
||||
open={savePresetOpen}
|
||||
onOpenChange={(nextOpen) => {
|
||||
setSavePresetOpen(nextOpen);
|
||||
if (!nextOpen) {
|
||||
setPresetNameDraft("");
|
||||
}
|
||||
}}
|
||||
>
|
||||
<DialogContent className="corner-squircle sm:max-w-sm">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Save Preset</DialogTitle>
|
||||
<DialogDescription>
|
||||
Enter a name for this inference preset.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<form
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
savePresetWithName(presetNameDraft);
|
||||
}}
|
||||
className="space-y-4"
|
||||
>
|
||||
<Input
|
||||
autoFocus={true}
|
||||
value={presetNameDraft}
|
||||
onChange={(event) => setPresetNameDraft(event.target.value)}
|
||||
placeholder="Preset name"
|
||||
maxLength={80}
|
||||
/>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
onClick={() => setSavePresetOpen(false)}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
type="submit"
|
||||
disabled={presetNameDraft.trim().length === 0}
|
||||
>
|
||||
Save
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</>
|
||||
);
|
||||
|
||||
if (isMobile) {
|
||||
|
|
@ -763,7 +802,9 @@ export function ChatSettingsPanel({
|
|||
|
||||
function MaxToolCallsSlider() {
|
||||
const maxToolCalls = useChatRuntimeStore((s) => s.maxToolCallsPerMessage);
|
||||
const setMaxToolCalls = useChatRuntimeStore((s) => s.setMaxToolCallsPerMessage);
|
||||
const setMaxToolCalls = useChatRuntimeStore(
|
||||
(s) => s.setMaxToolCallsPerMessage,
|
||||
);
|
||||
|
||||
// Slider range 0-41; 41 maps to 9999 ("Max")
|
||||
const sliderValue = maxToolCalls >= 9999 ? 41 : Math.min(maxToolCalls, 40);
|
||||
|
|
@ -776,7 +817,9 @@ function MaxToolCallsSlider() {
|
|||
max={41}
|
||||
step={1}
|
||||
onChange={(v) => setMaxToolCalls(v >= 41 ? 9999 : v)}
|
||||
displayValue={sliderValue >= 41 ? "Max" : sliderValue === 0 ? "Off" : undefined}
|
||||
displayValue={
|
||||
sliderValue >= 41 ? "Max" : sliderValue === 0 ? "Off" : undefined
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
@ -810,7 +853,9 @@ function ToolCallTimeoutSlider() {
|
|||
|
||||
function AutoHealToolCallsToggle() {
|
||||
const autoHealToolCalls = useChatRuntimeStore((s) => s.autoHealToolCalls);
|
||||
const setAutoHealToolCalls = useChatRuntimeStore((s) => s.setAutoHealToolCalls);
|
||||
const setAutoHealToolCalls = useChatRuntimeStore(
|
||||
(s) => s.setAutoHealToolCalls,
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
|
|
|
|||
|
|
@ -909,7 +909,9 @@ export function ExportPage() {
|
|||
const source =
|
||||
model?.source === "hf_cache"
|
||||
? "HF cache"
|
||||
: "Local dir";
|
||||
: model?.source === "custom"
|
||||
? "Custom Folders"
|
||||
: "Local dir";
|
||||
return (
|
||||
<ComboboxItem key={id} value={id} className="gap-2">
|
||||
<span className="block min-w-0 flex-1 truncate">
|
||||
|
|
|
|||
|
|
@ -268,12 +268,12 @@ export function ModelSelectionStep() {
|
|||
<ComboboxItem
|
||||
key={id}
|
||||
value={id}
|
||||
className={`justify-between ${exceeds ? "opacity-50" : ""}`}
|
||||
className="justify-between"
|
||||
>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span
|
||||
className={`min-w-0 flex-1 truncate ${exceeds ? "line-through decoration-muted-foreground/50" : ""}`}
|
||||
className={`min-w-0 flex-1 truncate ${exceeds ? "!text-gray-500 dark:!text-gray-400" : ""}`}
|
||||
>
|
||||
{id}
|
||||
</span>
|
||||
|
|
@ -287,12 +287,12 @@ export function ModelSelectionStep() {
|
|||
</Tooltip>
|
||||
<span className="flex items-center gap-1.5 shrink-0">
|
||||
{fitStatus === "exceeds" && (
|
||||
<span className="text-[9px] font-medium text-red-400">
|
||||
<span className="text-[9px] font-medium !text-red-700 !bg-red-50 dark:!text-red-400 dark:!bg-red-950 px-1.5 py-0.5 rounded">
|
||||
OOM
|
||||
</span>
|
||||
)}
|
||||
{fitStatus === "tight" && (
|
||||
<span className="text-[9px] font-medium text-amber-400">
|
||||
<span className="text-[9px] font-medium !text-amber-400">
|
||||
TIGHT
|
||||
</span>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,16 @@ import {
|
|||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { MODEL_TYPE_TO_HF_TASK, PRIORITY_TRAINING_MODELS, applyPriorityOrdering } from "@/config/training";
|
||||
import {
|
||||
MODEL_TYPE_TO_HF_TASK,
|
||||
PRIORITY_TRAINING_MODELS,
|
||||
applyPriorityOrdering,
|
||||
} from "@/config/training";
|
||||
import {
|
||||
type LocalModelInfo,
|
||||
listLocalModels,
|
||||
useTrainingConfigStore,
|
||||
} from "@/features/training";
|
||||
import {
|
||||
useDebouncedValue,
|
||||
useGpuInfo,
|
||||
|
|
@ -38,15 +47,10 @@ import {
|
|||
} from "@/hooks";
|
||||
import { formatCompact } from "@/lib/utils";
|
||||
import {
|
||||
type TrainingMethod as VramTrainingMethod,
|
||||
type VramFitStatus,
|
||||
type TrainingMethod as VramTrainingMethod,
|
||||
buildModelVramMap,
|
||||
} from "@/lib/vram";
|
||||
import {
|
||||
listLocalModels,
|
||||
type LocalModelInfo,
|
||||
useTrainingConfigStore,
|
||||
} from "@/features/training";
|
||||
import type { TrainingMethod } from "@/types/training";
|
||||
import {
|
||||
ChipIcon,
|
||||
|
|
@ -150,7 +154,9 @@ export function ModelSection() {
|
|||
.catch((error) => {
|
||||
if (controller.signal.aborted) return;
|
||||
setLocalModelsError(
|
||||
error instanceof Error ? error.message : "Failed to load local models",
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Failed to load local models",
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
|
|
@ -241,7 +247,9 @@ export function ModelSection() {
|
|||
{ est: number; status: VramFitStatus | null; detail: string | null }
|
||||
>();
|
||||
for (const r of hfResults) {
|
||||
const detail = r.totalParams ? formatCompact(r.totalParams) : extractParamLabel(r.id);
|
||||
const detail = r.totalParams
|
||||
? formatCompact(r.totalParams)
|
||||
: extractParamLabel(r.id);
|
||||
const fit = fitMap.get(r.id);
|
||||
map.set(r.id, {
|
||||
est: fit?.est ?? 0,
|
||||
|
|
@ -271,363 +279,383 @@ export function ModelSection() {
|
|||
className="shadow-border ring-border"
|
||||
>
|
||||
<div className="grid min-w-0 gap-4 md:grid-cols-2 xl:grid-cols-4">
|
||||
<div data-tour="studio-local-model" className="flex min-w-0 flex-col gap-2">
|
||||
<div
|
||||
data-tour="studio-local-model"
|
||||
className="flex min-w-0 flex-col gap-2"
|
||||
>
|
||||
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
|
||||
Local Model
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<button
|
||||
type="button"
|
||||
className="text-foreground/70 hover:text-foreground"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={InformationCircleIcon}
|
||||
className="size-3"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
Path to a locally downloaded model or a custom HF repo.
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</span>
|
||||
<div ref={localComboboxAnchorRef} className="min-w-0">
|
||||
<Combobox
|
||||
items={localResultIds}
|
||||
filteredItems={localFilteredIds}
|
||||
filter={null}
|
||||
value={localModelInput || null}
|
||||
onValueChange={(id) => {
|
||||
const next = id ?? "";
|
||||
setLocalModelInput(next);
|
||||
if (next) setSelectedModel(next);
|
||||
}}
|
||||
onInputValueChange={setLocalModelInput}
|
||||
itemToStringValue={(id) => id}
|
||||
autoHighlight={true}
|
||||
>
|
||||
<ComboboxInput
|
||||
placeholder={
|
||||
isLoadingLocalModels
|
||||
? "Scanning local and cached models..."
|
||||
: "./models/my-model"
|
||||
}
|
||||
className="w-full bg-foreground text-background [&_input]:text-background [&_input]:placeholder:text-background/40 [&_svg]:text-background/50 hover:bg-foreground/90"
|
||||
onBlur={() => applyLocalModel(localModelInput)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key !== "Enter") return;
|
||||
event.preventDefault();
|
||||
applyLocalModel(localModelInput);
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<button
|
||||
type="button"
|
||||
className="text-foreground/70 hover:text-foreground"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={InformationCircleIcon}
|
||||
className="size-3"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
Path to a locally downloaded model or a custom HF repo.
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</span>
|
||||
<div ref={localComboboxAnchorRef} className="min-w-0">
|
||||
<Combobox
|
||||
items={localResultIds}
|
||||
filteredItems={localFilteredIds}
|
||||
filter={null}
|
||||
value={localModelInput || null}
|
||||
onValueChange={(id) => {
|
||||
const next = id ?? "";
|
||||
setLocalModelInput(next);
|
||||
if (next) setSelectedModel(next);
|
||||
}}
|
||||
onInputValueChange={setLocalModelInput}
|
||||
itemToStringValue={(id) => id}
|
||||
autoHighlight={true}
|
||||
>
|
||||
<InputGroupAddon>
|
||||
<HugeiconsIcon icon={FolderSearchIcon} className="size-4" />
|
||||
</InputGroupAddon>
|
||||
</ComboboxInput>
|
||||
<ComboboxContent
|
||||
anchor={localComboboxAnchorRef}
|
||||
className={DARK_COMBOBOX_CONTENT}
|
||||
>
|
||||
{isLoadingLocalModels ? (
|
||||
<div className="flex items-center justify-center gap-2 py-4 text-xs text-muted-foreground">
|
||||
<Spinner className="size-4" /> Scanning...
|
||||
</div>
|
||||
) : localModelsError ? (
|
||||
<div className="px-3 py-2 text-xs text-red-500">
|
||||
{localModelsError}
|
||||
</div>
|
||||
) : (
|
||||
<ComboboxEmpty>No local models found</ComboboxEmpty>
|
||||
)}
|
||||
<ComboboxList className="p-1">
|
||||
{(id: string) => {
|
||||
const model = localMetaById.get(id);
|
||||
const source =
|
||||
model?.source === "hf_cache"
|
||||
? "HF cache"
|
||||
: model?.source === "lmstudio"
|
||||
? "LM Studio"
|
||||
: "Local dir";
|
||||
return (
|
||||
<ComboboxItem key={id} value={id} className="gap-2">
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span className="block min-w-0 flex-1 truncate">
|
||||
{model?.display_name ?? id}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="left" className="max-w-xs break-all">
|
||||
{model?.path ?? id}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<span className="ml-auto shrink-0 text-[10px] text-muted-foreground">
|
||||
{source}
|
||||
</span>
|
||||
</ComboboxItem>
|
||||
);
|
||||
<ComboboxInput
|
||||
placeholder={
|
||||
isLoadingLocalModels
|
||||
? "Scanning local and cached models..."
|
||||
: "./models/my-model"
|
||||
}
|
||||
className="w-full bg-foreground text-background [&_input]:text-background [&_input]:placeholder:text-background/40 [&_svg]:text-background/50 hover:bg-foreground/90"
|
||||
onBlur={() => applyLocalModel(localModelInput)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key !== "Enter") return;
|
||||
event.preventDefault();
|
||||
applyLocalModel(localModelInput);
|
||||
}}
|
||||
</ComboboxList>
|
||||
</ComboboxContent>
|
||||
</Combobox>
|
||||
</div>
|
||||
{isLoadingLocalModels ? (
|
||||
<p className="text-[10px] text-muted-foreground">Scanning local models...</p>
|
||||
) : localModelsError ? (
|
||||
<p className="text-[10px] text-red-500">{localModelsError}</p>
|
||||
) : (
|
||||
<p className="text-[10px] text-muted-foreground">
|
||||
{trainableLocalModels.length > 0
|
||||
? `${trainableLocalModels.length} local/cached models found`
|
||||
: "No local models found. Enter path manually."}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div data-tour="studio-base-model" className="flex min-w-0 flex-col gap-2">
|
||||
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
|
||||
Hugging Face Model
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<button
|
||||
type="button"
|
||||
className="text-foreground/70 hover:text-foreground"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={InformationCircleIcon}
|
||||
className="size-3"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
Search Hugging Face models or pick from our recommended list.{" "}
|
||||
<a
|
||||
href="https://unsloth.ai/docs/get-started/fine-tuning-llms-guide/what-model-should-i-use"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-primary underline"
|
||||
<InputGroupAddon>
|
||||
<HugeiconsIcon icon={FolderSearchIcon} className="size-4" />
|
||||
</InputGroupAddon>
|
||||
</ComboboxInput>
|
||||
<ComboboxContent
|
||||
anchor={localComboboxAnchorRef}
|
||||
className={DARK_COMBOBOX_CONTENT}
|
||||
>
|
||||
Read more
|
||||
</a>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</span>
|
||||
<div
|
||||
ref={comboboxAnchorRef}
|
||||
className="min-w-0"
|
||||
onKeyDown={(event) => {
|
||||
if (event.key !== "Enter") return;
|
||||
if (!(event.target instanceof HTMLInputElement)) return;
|
||||
event.preventDefault();
|
||||
if (hfResults.length > 0) {
|
||||
handleModelSelect(hfResults[0].id);
|
||||
} else {
|
||||
const text = event.target.value.trim();
|
||||
if (text) handleModelSelect(text);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Combobox
|
||||
items={resultIds}
|
||||
filteredItems={resultIds}
|
||||
filter={null}
|
||||
value={selectedModel}
|
||||
onValueChange={handleModelSelect}
|
||||
onInputValueChange={handleInputChange}
|
||||
itemToStringValue={(id) => id}
|
||||
autoHighlight={true}
|
||||
>
|
||||
<ComboboxInput
|
||||
placeholder="Search models..."
|
||||
className="w-full leading-5"
|
||||
>
|
||||
<InputGroupAddon>
|
||||
<HugeiconsIcon icon={Search01Icon} className="size-4" />
|
||||
</InputGroupAddon>
|
||||
</ComboboxInput>
|
||||
<ComboboxContent anchor={comboboxAnchorRef}>
|
||||
{isLoading ? (
|
||||
<div className="flex items-center justify-center py-4 gap-2 text-xs text-muted-foreground">
|
||||
<Spinner className="size-4" /> Searching…
|
||||
</div>
|
||||
) : (
|
||||
<ComboboxEmpty>No models found</ComboboxEmpty>
|
||||
)}
|
||||
<div
|
||||
ref={scrollRef}
|
||||
className="max-h-64 overflow-y-auto overscroll-contain [scrollbar-width:thin]"
|
||||
>
|
||||
<ComboboxList className="p-1 !max-h-none !overflow-visible">
|
||||
{isLoadingLocalModels ? (
|
||||
<div className="flex items-center justify-center gap-2 py-4 text-xs text-muted-foreground">
|
||||
<Spinner className="size-4" /> Scanning...
|
||||
</div>
|
||||
) : localModelsError ? (
|
||||
<div className="px-3 py-2 text-xs text-red-500">
|
||||
{localModelsError}
|
||||
</div>
|
||||
) : (
|
||||
<ComboboxEmpty>No local models found</ComboboxEmpty>
|
||||
)}
|
||||
<ComboboxList className="p-1">
|
||||
{(id: string) => {
|
||||
const entry = vramMap.get(id);
|
||||
const detail = entry?.detail ?? null;
|
||||
const fitStatus = entry?.status ?? null;
|
||||
const vramEst = entry?.est ?? null;
|
||||
const exceeds = fitStatus === "exceeds";
|
||||
|
||||
const model = localMetaById.get(id);
|
||||
const source =
|
||||
model?.source === "hf_cache"
|
||||
? "HF cache"
|
||||
: model?.source === "lmstudio"
|
||||
? "LM Studio"
|
||||
: model?.source === "custom"
|
||||
? "Custom Folders"
|
||||
: "Local dir";
|
||||
return (
|
||||
<ComboboxItem
|
||||
key={id}
|
||||
value={id}
|
||||
className={`gap-2 ${exceeds ? "opacity-50" : ""}`}
|
||||
>
|
||||
<ComboboxItem key={id} value={id} className="gap-2">
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span className={`block min-w-0 flex-1 truncate ${exceeds ? "line-through decoration-muted-foreground/50" : ""}`}>
|
||||
{id}
|
||||
<span className="block min-w-0 flex-1 truncate">
|
||||
{model?.display_name ?? id}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent
|
||||
side="left"
|
||||
className="max-w-xs break-all"
|
||||
>
|
||||
{id}
|
||||
{vramEst != null && vramEst > 0 && gpu.available && (
|
||||
<span className="block text-[10px] mt-1">
|
||||
{exceeds
|
||||
? `Needs ~${vramEst}GB VRAM (GPU: ${gpu.memoryTotalGb}GB)`
|
||||
: fitStatus === "tight"
|
||||
? `~${vramEst}GB VRAM (tight fit on ${gpu.memoryTotalGb}GB)`
|
||||
: `~${vramEst}GB VRAM`}
|
||||
</span>
|
||||
)}
|
||||
{model?.path ?? id}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<span className="ml-auto flex items-center gap-1.5 shrink-0">
|
||||
{fitStatus === "exceeds" && (
|
||||
<span className="text-[9px] font-medium text-red-400">
|
||||
OOM
|
||||
</span>
|
||||
)}
|
||||
{fitStatus === "tight" && (
|
||||
<span className="text-[9px] font-medium text-amber-400">
|
||||
TIGHT
|
||||
</span>
|
||||
)}
|
||||
{detail && (
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{detail}
|
||||
</span>
|
||||
)}
|
||||
<span className="ml-auto shrink-0 text-[10px] text-muted-foreground">
|
||||
{source}
|
||||
</span>
|
||||
</ComboboxItem>
|
||||
);
|
||||
}}
|
||||
</ComboboxList>
|
||||
<div ref={sentinelRef} className="h-px" />
|
||||
{isLoadingMore && (
|
||||
<div className="flex items-center justify-center py-2">
|
||||
<Spinner className="size-3.5 text-muted-foreground" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</ComboboxContent>
|
||||
</Combobox>
|
||||
</ComboboxContent>
|
||||
</Combobox>
|
||||
</div>
|
||||
{isLoadingLocalModels ? (
|
||||
<p className="text-[10px] text-muted-foreground">
|
||||
Scanning local models...
|
||||
</p>
|
||||
) : localModelsError ? (
|
||||
<p className="text-[10px] text-red-500">{localModelsError}</p>
|
||||
) : (
|
||||
<p className="text-[10px] text-muted-foreground">
|
||||
{trainableLocalModels.length > 0
|
||||
? `${trainableLocalModels.length} local/cached models found`
|
||||
: "No local models found. Enter path manually."}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div data-tour="studio-method" className="flex min-w-0 flex-col gap-2">
|
||||
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
|
||||
Method
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<button
|
||||
type="button"
|
||||
className="text-foreground/70 hover:text-foreground"
|
||||
<div
|
||||
data-tour="studio-base-model"
|
||||
className="flex min-w-0 flex-col gap-2"
|
||||
>
|
||||
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
|
||||
Hugging Face Model
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<button
|
||||
type="button"
|
||||
className="text-foreground/70 hover:text-foreground"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={InformationCircleIcon}
|
||||
className="size-3"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
Search Hugging Face models or pick from our recommended list.{" "}
|
||||
<a
|
||||
href="https://unsloth.ai/docs/get-started/fine-tuning-llms-guide/what-model-should-i-use"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-primary underline"
|
||||
>
|
||||
Read more
|
||||
</a>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</span>
|
||||
<div
|
||||
ref={comboboxAnchorRef}
|
||||
className="min-w-0"
|
||||
onKeyDown={(event) => {
|
||||
if (event.key !== "Enter") return;
|
||||
if (!(event.target instanceof HTMLInputElement)) return;
|
||||
event.preventDefault();
|
||||
if (hfResults.length > 0) {
|
||||
handleModelSelect(hfResults[0].id);
|
||||
} else {
|
||||
const text = event.target.value.trim();
|
||||
if (text) handleModelSelect(text);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Combobox
|
||||
items={resultIds}
|
||||
filteredItems={resultIds}
|
||||
filter={null}
|
||||
value={selectedModel}
|
||||
onValueChange={handleModelSelect}
|
||||
onInputValueChange={handleInputChange}
|
||||
itemToStringValue={(id) => id}
|
||||
autoHighlight={true}
|
||||
>
|
||||
<ComboboxInput
|
||||
placeholder="Search models..."
|
||||
className="w-full leading-5"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={InformationCircleIcon}
|
||||
className="size-3"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className="max-w-xs">
|
||||
QLoRA uses 4-bit quantization for lowest VRAM. LoRA uses 16-bit.
|
||||
Full updates all weights.{" "}
|
||||
<InputGroupAddon>
|
||||
<HugeiconsIcon icon={Search01Icon} className="size-4" />
|
||||
</InputGroupAddon>
|
||||
</ComboboxInput>
|
||||
<ComboboxContent anchor={comboboxAnchorRef}>
|
||||
{isLoading ? (
|
||||
<div className="flex items-center justify-center py-4 gap-2 text-xs text-muted-foreground">
|
||||
<Spinner className="size-4" /> Searching…
|
||||
</div>
|
||||
) : (
|
||||
<ComboboxEmpty>No models found</ComboboxEmpty>
|
||||
)}
|
||||
<div
|
||||
ref={scrollRef}
|
||||
className="max-h-64 overflow-y-auto overscroll-contain [scrollbar-width:thin]"
|
||||
>
|
||||
<ComboboxList className="p-1 !max-h-none !overflow-visible">
|
||||
{(id: string) => {
|
||||
const entry = vramMap.get(id);
|
||||
const detail = entry?.detail ?? null;
|
||||
const fitStatus = entry?.status ?? null;
|
||||
const vramEst = entry?.est ?? null;
|
||||
const exceeds = fitStatus === "exceeds";
|
||||
|
||||
return (
|
||||
<ComboboxItem
|
||||
key={id}
|
||||
value={id}
|
||||
className="gap-2"
|
||||
>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span
|
||||
className={`block min-w-0 flex-1 truncate ${exceeds ? "!text-gray-500 dark:!text-gray-400" : ""}`}
|
||||
>
|
||||
{id}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent
|
||||
side="left"
|
||||
className="max-w-xs break-all"
|
||||
>
|
||||
{id}
|
||||
{vramEst != null &&
|
||||
vramEst > 0 &&
|
||||
gpu.available && (
|
||||
<span className="block text-[10px] mt-1">
|
||||
{exceeds
|
||||
? `Needs ~${vramEst}GB VRAM (GPU: ${gpu.memoryTotalGb}GB)`
|
||||
: fitStatus === "tight"
|
||||
? `~${vramEst}GB VRAM (tight fit on ${gpu.memoryTotalGb}GB)`
|
||||
: `~${vramEst}GB VRAM`}
|
||||
</span>
|
||||
)}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<span className="ml-auto flex items-center gap-1.5 shrink-0">
|
||||
{fitStatus === "exceeds" && (
|
||||
<span className="text-[9px] font-medium !text-red-700 !bg-red-50 dark:!text-red-400 dark:!bg-red-950 px-1.5 py-0.5 rounded">
|
||||
OOM
|
||||
</span>
|
||||
)}
|
||||
{fitStatus === "tight" && (
|
||||
<span className="text-[9px] font-medium !text-amber-400">
|
||||
TIGHT
|
||||
</span>
|
||||
)}
|
||||
{detail && (
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{detail}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</ComboboxItem>
|
||||
);
|
||||
}}
|
||||
</ComboboxList>
|
||||
<div ref={sentinelRef} className="h-px" />
|
||||
{isLoadingMore && (
|
||||
<div className="flex items-center justify-center py-2">
|
||||
<Spinner className="size-3.5 text-muted-foreground" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</ComboboxContent>
|
||||
</Combobox>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
data-tour="studio-method"
|
||||
className="flex min-w-0 flex-col gap-2"
|
||||
>
|
||||
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
|
||||
Method
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<button
|
||||
type="button"
|
||||
className="text-foreground/70 hover:text-foreground"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={InformationCircleIcon}
|
||||
className="size-3"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className="max-w-xs">
|
||||
QLoRA uses 4-bit quantization for lowest VRAM. LoRA uses
|
||||
16-bit. Full updates all weights.{" "}
|
||||
<a
|
||||
href="https://unsloth.ai/docs/get-started/fine-tuning-llms-guide/lora-hyperparameters-guide"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-primary underline"
|
||||
>
|
||||
Read more
|
||||
</a>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</span>
|
||||
<Select
|
||||
value={trainingMethod}
|
||||
onValueChange={(v) => setTrainingMethod(v as TrainingMethod)}
|
||||
>
|
||||
<SelectTrigger className={DARK_TRIGGER}>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent
|
||||
position="popper"
|
||||
className={`${DARK_CONTENT} w-[var(--radix-select-trigger-width)]`}
|
||||
>
|
||||
<SelectItem value="qlora">
|
||||
<span className="flex items-center gap-2">
|
||||
<span
|
||||
className={`size-2 shrink-0 rounded-full ${METHOD_DOTS.qlora}`}
|
||||
/>
|
||||
QLoRA (4-bit)
|
||||
</span>
|
||||
</SelectItem>
|
||||
<SelectItem value="lora">
|
||||
<span className="flex items-center gap-2">
|
||||
<span
|
||||
className={`size-2 shrink-0 rounded-full ${METHOD_DOTS.lora}`}
|
||||
/>
|
||||
LoRA (16-bit)
|
||||
</span>
|
||||
</SelectItem>
|
||||
<SelectItem value="full">
|
||||
<span className="flex items-center gap-2">
|
||||
<span
|
||||
className={`size-2 shrink-0 rounded-full ${METHOD_DOTS.full}`}
|
||||
/>
|
||||
Full Fine-tune
|
||||
</span>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
<div className="flex min-w-0 flex-col gap-2">
|
||||
<span className="text-xs font-medium text-muted-foreground">
|
||||
Hugging Face Token (Optional)
|
||||
</span>
|
||||
<InputGroup>
|
||||
<InputGroupAddon>
|
||||
<HugeiconsIcon icon={Key01Icon} className="size-4" />
|
||||
</InputGroupAddon>
|
||||
<InputGroupInput
|
||||
type="password"
|
||||
autoComplete="new-password"
|
||||
name="hf-token"
|
||||
placeholder="hf_..."
|
||||
value={hfToken}
|
||||
onChange={(e) => setHfToken(e.target.value)}
|
||||
/>
|
||||
</InputGroup>
|
||||
{(tokenValidationError ?? hfSearchError) && (
|
||||
<p className="text-xs text-destructive">
|
||||
{tokenValidationError ?? hfSearchError}
|
||||
{" — "}
|
||||
<a
|
||||
href="https://unsloth.ai/docs/get-started/fine-tuning-llms-guide/lora-hyperparameters-guide"
|
||||
href="https://huggingface.co/settings/tokens"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-primary underline"
|
||||
className="underline"
|
||||
>
|
||||
Read more
|
||||
Get or update token
|
||||
</a>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</span>
|
||||
<Select
|
||||
value={trainingMethod}
|
||||
onValueChange={(v) => setTrainingMethod(v as TrainingMethod)}
|
||||
>
|
||||
<SelectTrigger className={DARK_TRIGGER}>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent
|
||||
position="popper"
|
||||
className={`${DARK_CONTENT} w-[var(--radix-select-trigger-width)]`}
|
||||
>
|
||||
<SelectItem value="qlora">
|
||||
<span className="flex items-center gap-2">
|
||||
<span
|
||||
className={`size-2 shrink-0 rounded-full ${METHOD_DOTS.qlora}`}
|
||||
/>
|
||||
QLoRA (4-bit)
|
||||
</span>
|
||||
</SelectItem>
|
||||
<SelectItem value="lora">
|
||||
<span className="flex items-center gap-2">
|
||||
<span
|
||||
className={`size-2 shrink-0 rounded-full ${METHOD_DOTS.lora}`}
|
||||
/>
|
||||
LoRA (16-bit)
|
||||
</span>
|
||||
</SelectItem>
|
||||
<SelectItem value="full">
|
||||
<span className="flex items-center gap-2">
|
||||
<span
|
||||
className={`size-2 shrink-0 rounded-full ${METHOD_DOTS.full}`}
|
||||
/>
|
||||
Full Fine-tune
|
||||
</span>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
<div className="flex min-w-0 flex-col gap-2">
|
||||
<span className="text-xs font-medium text-muted-foreground">
|
||||
Hugging Face Token (Optional)
|
||||
</span>
|
||||
<InputGroup>
|
||||
<InputGroupAddon>
|
||||
<HugeiconsIcon icon={Key01Icon} className="size-4" />
|
||||
</InputGroupAddon>
|
||||
<InputGroupInput
|
||||
type="password"
|
||||
autoComplete="new-password"
|
||||
name="hf-token"
|
||||
placeholder="hf_..."
|
||||
value={hfToken}
|
||||
onChange={(e) => setHfToken(e.target.value)}
|
||||
/>
|
||||
</InputGroup>
|
||||
{(tokenValidationError ?? hfSearchError) && (
|
||||
<p className="text-xs text-destructive">
|
||||
{tokenValidationError ?? hfSearchError}
|
||||
{" — "}
|
||||
<a
|
||||
href="https://huggingface.co/settings/tokens"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="underline"
|
||||
>
|
||||
Get or update token
|
||||
</a>
|
||||
</p>
|
||||
)}
|
||||
{isCheckingToken && (
|
||||
<p className="text-xs text-muted-foreground">Checking token…</p>
|
||||
)}
|
||||
</div>
|
||||
</p>
|
||||
)}
|
||||
{isCheckingToken && (
|
||||
<p className="text-xs text-muted-foreground">Checking token…</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</SectionCard>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ export interface LocalModelInfo {
|
|||
id: string;
|
||||
display_name: string;
|
||||
path: string;
|
||||
source: "models_dir" | "hf_cache" | "lmstudio";
|
||||
source: "models_dir" | "hf_cache" | "lmstudio" | "custom";
|
||||
model_id?: string | null;
|
||||
updated_at?: number | null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -104,6 +104,11 @@ function makeMapModel(excludeGguf: boolean) {
|
|||
|
||||
/** Number of unsloth results to pull up-front before yielding general results. */
|
||||
const UNSLOTH_PREFETCH = 20;
|
||||
/** When the user searched for a specific publisher, show fewer unsloth results
|
||||
* before the pinned (original publisher) model. */
|
||||
const UNSLOTH_PINNED_PREFETCH = 4;
|
||||
/** Matches a valid "owner/repo" identifier (exactly two non-empty segments). */
|
||||
const PUBLISHER_RE = /^([^/\s]+)\/([^/\s]+)$/;
|
||||
|
||||
/**
|
||||
* Prime the hf-cache from a listModels result. For public (non-gated,
|
||||
|
|
@ -131,6 +136,7 @@ async function* mergedModelIterator(
|
|||
query: string,
|
||||
task?: PipelineType,
|
||||
accessToken?: string,
|
||||
pinnedId?: string,
|
||||
): AsyncGenerator<unknown> {
|
||||
const common = {
|
||||
additionalFields: ["safetensors", "tags"] as ("safetensors" | "tags")[],
|
||||
|
|
@ -148,6 +154,18 @@ async function* mergedModelIterator(
|
|||
...common,
|
||||
});
|
||||
|
||||
// Start pinned model lookup immediately so it can run in parallel with
|
||||
// the Phase 1 unsloth iteration instead of blocking Phase 2.
|
||||
const pinnedPromise = pinnedId
|
||||
? cachedModelInfo({
|
||||
name: pinnedId,
|
||||
additionalFields: ["safetensors", "tags"],
|
||||
...(accessToken ? { credentials: { accessToken } } : {}),
|
||||
}).catch(() => null)
|
||||
: null;
|
||||
|
||||
const limit = pinnedId ? UNSLOTH_PINNED_PREFETCH : UNSLOTH_PREFETCH;
|
||||
|
||||
// Phase 1: pull & yield unsloth models first
|
||||
const seen = new Set<string>();
|
||||
let count = 0;
|
||||
|
|
@ -159,10 +177,26 @@ async function* mergedModelIterator(
|
|||
}
|
||||
yield model;
|
||||
count++;
|
||||
if (count >= UNSLOTH_PREFETCH) break;
|
||||
if (count >= limit) break;
|
||||
}
|
||||
|
||||
// Phase 2: yield general results, skipping already-seen unsloth models
|
||||
// Phase 1b: yield the pinned (original publisher) model before general results
|
||||
if (pinnedId && !seen.has(pinnedId) && pinnedPromise) {
|
||||
const pinned = await pinnedPromise;
|
||||
if (pinned) {
|
||||
// Record both the raw input and the canonical name returned by HF
|
||||
// so phase 2 deduplication works even when casing differs
|
||||
// (e.g. user typed "OpenAI/gpt-oss-20b", HF returns "openai/gpt-oss-20b").
|
||||
seen.add(pinnedId);
|
||||
const canonicalName = (pinned as { name?: string }).name;
|
||||
if (canonicalName && canonicalName !== pinnedId) {
|
||||
seen.add(canonicalName);
|
||||
}
|
||||
yield pinned;
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: yield general results, skipping already-seen models
|
||||
for await (const model of generalIter) {
|
||||
const m = model as { name?: string };
|
||||
if (m.name && seen.has(m.name)) continue;
|
||||
|
|
@ -235,11 +269,24 @@ export function useHfModelSearch(
|
|||
) {
|
||||
const { task, accessToken, excludeGguf = false, priorityIds } = options ?? {};
|
||||
|
||||
// Parse publisher detection once and share between the iterator factory
|
||||
// and the secondary sort gate (avoids duplicating the regex + logic).
|
||||
const { isPublisherQuery, searchQuery, pinnedId, trimmed } = useMemo(() => {
|
||||
const t = query.trim();
|
||||
const m = PUBLISHER_RE.exec(t);
|
||||
const is = !!m && m[1].toLowerCase() !== "unsloth";
|
||||
return {
|
||||
isPublisherQuery: is,
|
||||
searchQuery: is ? m![2] : t,
|
||||
pinnedId: is ? t : undefined,
|
||||
trimmed: t,
|
||||
};
|
||||
}, [query]);
|
||||
|
||||
const createIter = useCallback(
|
||||
() => {
|
||||
const trimmed = query.trim();
|
||||
if (!trimmed) {
|
||||
// No query → show priority models first (with full metadata), then general unsloth listing
|
||||
// No query: show priority models first (with full metadata), then general unsloth listing
|
||||
if (priorityIds && priorityIds.length > 0) {
|
||||
return priorityThenListingIterator(priorityIds, task, accessToken) as AsyncGenerator<unknown>;
|
||||
}
|
||||
|
|
@ -250,24 +297,35 @@ export function useHfModelSearch(
|
|||
...(accessToken ? { credentials: { accessToken } } : {}),
|
||||
}) as AsyncGenerator<unknown>;
|
||||
}
|
||||
// Typed query: disable task filter so explicitly searched models still appear even if HF task metadata is wrong/missing.
|
||||
return mergedModelIterator(trimmed, undefined, accessToken) as AsyncGenerator<unknown>;
|
||||
// Typed query: disable task filter so explicitly searched models still
|
||||
// appear even if HF task metadata is wrong/missing.
|
||||
// If the query is a valid "owner/repo" identifier (exactly two non-empty,
|
||||
// slash-free, space-free segments), strip the org prefix so unsloth
|
||||
// variants surface, then pin the original publisher model after a small
|
||||
// batch of unsloth results. Queries for unsloth-owned models are left
|
||||
// as-is so they get the full 20-result prefetch and secondary sort.
|
||||
return mergedModelIterator(searchQuery, undefined, accessToken, pinnedId) as AsyncGenerator<unknown>;
|
||||
},
|
||||
[query, task, accessToken, priorityIds],
|
||||
[trimmed, searchQuery, pinnedId, task, accessToken, priorityIds],
|
||||
);
|
||||
|
||||
const mapModel = useMemo(() => makeMapModel(excludeGguf), [excludeGguf]);
|
||||
const search = useHfPaginatedSearch(createIter, mapModel);
|
||||
|
||||
// Secondary sort guarantee: unsloth models always float to the top
|
||||
// Secondary sort guarantee: unsloth models always float to the top.
|
||||
// Skip when the user searched for a specific non-unsloth publisher
|
||||
// (e.g. "openai/gpt-oss-20b") -- the iterator already handles the
|
||||
// pinned ordering in that case.
|
||||
const results = useMemo(
|
||||
() =>
|
||||
[...search.results].sort((a, b) => {
|
||||
const aFirst = a.id.startsWith("unsloth/") ? 0 : 1;
|
||||
const bFirst = b.id.startsWith("unsloth/") ? 0 : 1;
|
||||
return aFirst - bFirst;
|
||||
}),
|
||||
[search.results],
|
||||
isPublisherQuery
|
||||
? search.results
|
||||
: [...search.results].sort((a, b) => {
|
||||
const aFirst = a.id.startsWith("unsloth/") ? 0 : 1;
|
||||
const bFirst = b.id.startsWith("unsloth/") ? 0 : 1;
|
||||
return aFirst - bFirst;
|
||||
}),
|
||||
[search.results, isPublisherQuery],
|
||||
);
|
||||
|
||||
return { ...search, results };
|
||||
|
|
|
|||
|
|
@ -490,8 +490,7 @@ if (-not $HasNvidiaSmi) {
|
|||
if (-not $HasNvidiaSmi) {
|
||||
Write-Host ""
|
||||
step "gpu" "none (chat-only / GGUF)" "Yellow"
|
||||
Write-Host " Training and GPU inference require an NVIDIA GPU with drivers installed." -ForegroundColor Yellow
|
||||
Write-Host " https://www.nvidia.com/Download/index.aspx" -ForegroundColor Yellow
|
||||
substep "Training and GPU inference require an NVIDIA GPU with drivers installed." "Yellow"
|
||||
Write-Host ""
|
||||
} else {
|
||||
step "gpu" "NVIDIA GPU detected"
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2026.3.17"
|
||||
__version__ = "2026.3.18"
|
||||
|
||||
__all__ = [
|
||||
"SUPPORTS_BFLOAT16",
|
||||
|
|
@ -64,7 +64,8 @@ __all__ = [
|
|||
"patch_compiled_autograd",
|
||||
"process_vision_info",
|
||||
"unsloth_compile_transformers",
|
||||
"prefer_flex_attn_if_supported",
|
||||
"determine_attention_implementation",
|
||||
"_set_attn_impl",
|
||||
"patch_fast_lora",
|
||||
"validate_loftq_config",
|
||||
"RaiseUninitialized",
|
||||
|
|
@ -222,44 +223,74 @@ def apply_unsloth_gradient_checkpointing(
|
|||
return use_gradient_checkpointing
|
||||
|
||||
|
||||
def prefer_flex_attn_if_supported(model_class, config):
|
||||
if os.environ.get("UNSLOTH_ENABLE_FLEX_ATTENTION", "1") == "0":
|
||||
return None
|
||||
try:
|
||||
from transformers.utils.import_utils import is_torch_flex_attn_available
|
||||
# Models that don't work with flex_attention:
|
||||
# GPT-OSS: left padding issues cause incorrect outputs.
|
||||
# Mllama: BlockMask Q_LEN!=KV_LEN ValueError on decode.
|
||||
# NemotronH: hybrid Mamba-2 + Transformer, raises NotImplementedError.
|
||||
# Gemma3N: timm vision wrappers don't support flex_attention.
|
||||
_FLEX_EXCLUDED_MODELS = ("gpt_oss", "mllama", "nemotron_h")
|
||||
_EAGER_ONLY_PREFIXES = ("gemma3n",)
|
||||
|
||||
if not is_torch_flex_attn_available():
|
||||
return None
|
||||
if model_class is None or not getattr(
|
||||
model_class, "_supports_flex_attn", False
|
||||
):
|
||||
return None
|
||||
|
||||
attention_dropout = getattr(config, "attention_dropout", 0) or 0
|
||||
if attention_dropout > 0:
|
||||
return None
|
||||
# GPT-OSS, Mllama and Gemma3N use eager/sdpa attention during
|
||||
# inference since flex attention returns incorrect results or errors out.
|
||||
# GPT-OSS: left padding issues cause incorrect outputs.
|
||||
# Mllama: _update_causal_mask uses make_flex_block_causal_mask which
|
||||
# creates BlockMask with Q_LEN=KV_LEN=total_seq_len, but during
|
||||
# decode q_len=1, causing ValueError. Needs transformers update.
|
||||
# Gemma3N: timm vision wrappers (eg Gemma3nVisionConfig) do not
|
||||
# support flex_attention.
|
||||
# NemotronH: hybrid Mamba-2 + Transformer model that does not
|
||||
# support flex_attention (raises NotImplementedError from transformers).
|
||||
model_type = getattr(config, "model_type", "") if config else ""
|
||||
if model_type in ("gpt_oss", "mllama", "nemotron_h") or str(
|
||||
model_type
|
||||
).startswith("gemma3n"):
|
||||
return None
|
||||
if config is not None:
|
||||
setattr(config, "_attn_implementation", "flex_attention")
|
||||
if hasattr(config, "attn_implementation"):
|
||||
setattr(config, "attn_implementation", "flex_attention")
|
||||
return "flex_attention"
|
||||
except Exception:
|
||||
return None
|
||||
def _is_flex_excluded(model_type):
|
||||
return model_type in _FLEX_EXCLUDED_MODELS
|
||||
|
||||
|
||||
def _is_eager_only(model_type):
|
||||
return any(model_type.startswith(p) for p in _EAGER_ONLY_PREFIXES)
|
||||
|
||||
|
||||
def _set_attn_impl(config, impl):
|
||||
"""Helper function to set attention implementation on config and return it."""
|
||||
if config is not None:
|
||||
setattr(config, "_attn_implementation", impl)
|
||||
if hasattr(config, "attn_implementation"):
|
||||
setattr(config, "attn_implementation", impl)
|
||||
return impl
|
||||
|
||||
|
||||
def determine_attention_implementation(model_class, config):
|
||||
model_type = getattr(config, "model_type", "").lower()
|
||||
|
||||
# Eager-only models (e.g. gemma3n timm vision towers)
|
||||
if _is_eager_only(model_type):
|
||||
_set_attn_impl(config, "eager")
|
||||
return "eager"
|
||||
|
||||
# Flash Attention 2
|
||||
if HAS_FLASH_ATTENTION and model_class is not None:
|
||||
supports_fa2 = getattr(model_class, "_supports_flash_attn_2", False) or getattr(
|
||||
model_class, "_supports_flash_attn", False
|
||||
)
|
||||
if supports_fa2:
|
||||
_set_attn_impl(config, "flash_attention_2")
|
||||
return "flash_attention_2"
|
||||
|
||||
# Flex Attention
|
||||
if os.environ.get("UNSLOTH_ENABLE_FLEX_ATTENTION", "1") != "0":
|
||||
try:
|
||||
from transformers.utils.import_utils import is_torch_flex_attn_available
|
||||
|
||||
if (
|
||||
is_torch_flex_attn_available()
|
||||
and model_class is not None
|
||||
and getattr(model_class, "_supports_flex_attn", False)
|
||||
and not _is_flex_excluded(model_type)
|
||||
):
|
||||
attention_dropout = getattr(config, "attention_dropout", 0) or 0
|
||||
if attention_dropout == 0:
|
||||
_set_attn_impl(config, "flex_attention")
|
||||
return "flex_attention"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# SDPA
|
||||
if model_class is not None and getattr(model_class, "_supports_sdpa", False):
|
||||
_set_attn_impl(config, "sdpa")
|
||||
return "sdpa"
|
||||
|
||||
_set_attn_impl(config, "eager")
|
||||
return "eager"
|
||||
|
||||
|
||||
def _run_temporary_patches(phase):
|
||||
|
|
|
|||
|
|
@ -2341,8 +2341,8 @@ class FastLlamaModel:
|
|||
model_function = MODEL_FOR_CAUSAL_LM_MAPPING[model_config.__class__]
|
||||
IS_FALCON_H1 = model_config.model_type.startswith("falcon_h1")
|
||||
|
||||
preferred_attn_impl = (
|
||||
prefer_flex_attn_if_supported(model_function, model_config) or "eager"
|
||||
preferred_attn_impl = determine_attention_implementation(
|
||||
model_function, model_config
|
||||
)
|
||||
|
||||
has_rope_scaling = False
|
||||
|
|
|
|||
|
|
@ -597,8 +597,6 @@ class FastBaseModel:
|
|||
custom_datatype = None
|
||||
correct_dtype = None
|
||||
|
||||
# Stop SDPA for some archs like Pixtral / Mistral3
|
||||
flex_attn_impl = None
|
||||
if auto_config is None:
|
||||
auto_config = AutoConfig.from_pretrained(
|
||||
model_name,
|
||||
|
|
@ -609,7 +607,14 @@ class FastBaseModel:
|
|||
model_class = auto_model._model_mapping[auto_config.__class__]
|
||||
except Exception:
|
||||
model_class = None
|
||||
flex_attn_impl = prefer_flex_attn_if_supported(model_class, auto_config)
|
||||
if model_class is None:
|
||||
# When model_class cannot be resolved (remote-code or unmapped
|
||||
# configs), preserve the old fallback of sdpa when supported.
|
||||
attn_impl = _set_attn_impl(
|
||||
auto_config, "sdpa" if supports_sdpa else "eager"
|
||||
)
|
||||
else:
|
||||
attn_impl = determine_attention_implementation(model_class, auto_config)
|
||||
|
||||
# Handle FP8 models: get_model_name has already redirected this to BF16 sibling if the model ships with
|
||||
# FP8 weights. We just need to update it here for sanity.
|
||||
|
|
@ -620,21 +625,15 @@ class FastBaseModel:
|
|||
except Exception:
|
||||
model_class = None
|
||||
|
||||
model_type = str(getattr(auto_config, "model_type", "")).lower()
|
||||
if model_type.startswith("gemma3n"):
|
||||
# Gemma3N variants initialize timm-based vision towers which do
|
||||
# not support flex_attention, so default to eager unless overridden.
|
||||
default_attn_impl = "eager"
|
||||
else:
|
||||
default_attn_impl = "flex_attention" if flex_attn_impl else "sdpa"
|
||||
if not ("attn_implementation" in kwargs):
|
||||
kwargs["attn_implementation"] = default_attn_impl
|
||||
kwargs["attn_implementation"] = attn_impl
|
||||
if not supports_sdpa and kwargs.get("attn_implementation") == "sdpa":
|
||||
if os.environ.get("UNSLOTH_ENABLE_FLEX_ATTENTION", "0") == "0":
|
||||
print(
|
||||
f"Unsloth: {model_type_arch.title()} does not support SDPA - switching to fast eager."
|
||||
)
|
||||
print(
|
||||
f"Unsloth: {model_type_arch.title()} does not support SDPA - switching to fast eager."
|
||||
)
|
||||
del kwargs["attn_implementation"]
|
||||
# Re-stamp config so it stays consistent with the actual impl
|
||||
_set_attn_impl(auto_config, "eager")
|
||||
|
||||
bnb_config = None
|
||||
user_quantization_config = kwargs.get("quantization_config", None)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue