Studio: add RAG with hybrid search, reranker, chat integration

Backend (studio/backend/):
- core/rag/: parsers (PDF/TXT/MD/DOCX/HTML via pypdf/python-docx/bs4),
  recursive token-aware chunker, embeddings singleton via
  FastSentenceTransformer.from_pretrained(for_inference=True), Qdrant
  local vector store, bm25s lexical index, RRF hybrid retrieval,
  spawn-subprocess ingestion job with SSE progress, optional
  CrossEncoder reranker (off-by-default).
- routes/rag.py: KB CRUD, doc upload (KB + per-thread), doc list/delete,
  ingestion SSE, hybrid+rerank search, thread-index list/clear.
- routes/chat_history.py: purge thread RAG artifacts on thread delete
  and clear-all (rag_documents has no FK cascade to chat_threads so
  uploads work on un-persisted threads).
- studio.db gains 4 RAG tables; storage_roots gains rag_*() helpers.
- auth/authentication.py: get_current_subject_sse accepts ?token=... so
  EventSource can stream ingestion progress.

Frontend (studio/frontend/):
- features/rag/: api client, Zustand store, hooks, dropzone, KB list,
  doc rows, ingestion-progress, thread-index list components.
- Settings dialog gains a Knowledge Bases tab (master/detail + thread
  documents list); /knowledge-bases deep-links to it.
- features/chat/: per-thread ragSource/enableRerank/ragTopK state in
  chat-runtime-store; Retrieval section in chat-settings-sheet with KB
  DropdownMenu (active highlight + per-row trash), thread doc list with
  Clear-thread-index button, RAG Top K slider, reranker toggle;
  chat-adapter retrieves before /v1/chat/completions and injects hits
  as a system block; shared-composer + button routes documents into
  pendingDocs (auto-uploads, send blocked while indexing).

tests/python/test_rag_bm25.py

@@ -0,0 +1,69 @@
+"""BM25 index lifecycle tests (skipped if bm25s is unavailable)."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
+if str(STUDIO_BACKEND) not in sys.path:
+    sys.path.insert(0, str(STUDIO_BACKEND))
+
+pytest.importorskip("bm25s")
+
+
+@pytest.fixture
+def isolated_bm25_root(tmp_path, monkeypatch):
+    from utils.paths import storage_roots
+
+    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+    # Reset module-level cache between tests.
+    from core.rag import bm25
+
+    bm25._cache.clear()
+    return tmp_path
+
+
+def test_bm25_index_search_roundtrip(isolated_bm25_root):
+    from core.rag import bm25
+
+    scope = "kb_test"
+    chunks = [
+        {"id": "c1", "text": "the quick brown fox jumps over the lazy dog"},
+        {"id": "c2", "text": "machine learning models predict outputs from inputs"},
+        {"id": "c3", "text": "fox terriers are small dogs"},
+    ]
+    bm25.rebuild_index(scope, chunks)
+    results = bm25.search(scope, "fox", k = 3)
+    ids = [cid for cid, _ in results]
+    assert "c1" in ids
+    assert "c3" in ids
+
+
+def test_bm25_empty_returns_empty(isolated_bm25_root):
+    from core.rag import bm25
+
+    assert bm25.search("kb_nonexistent", "anything", k = 5) == []
+
+
+def test_bm25_delete_scope(isolated_bm25_root):
+    from core.rag import bm25
+
+    scope = "kb_del"
+    chunks = [{"id": "a", "text": "alpha beta gamma"}]
+    bm25.rebuild_index(scope, chunks)
+    assert bm25.search(scope, "alpha", k = 1)
+    bm25.delete_scope(scope)
+    assert bm25.search(scope, "alpha", k = 1) == []
+
+
+def test_bm25_rebuild_replaces_old_corpus(isolated_bm25_root):
+    from core.rag import bm25
+
+    scope = "kb_replace"
+    bm25.rebuild_index(scope, [{"id": "old", "text": "alpha beta"}])
+    bm25.rebuild_index(scope, [{"id": "new", "text": "gamma delta"}])
+    results = bm25.search(scope, "alpha", k = 5)
+    ids = [cid for cid, _ in results]
+    assert "old" not in ids

tests/python/test_rag_chunking.py

@@ -0,0 +1,84 @@
+"""Unit tests for RAG chunking — pure-python, no external deps."""
+
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
+if str(STUDIO_BACKEND) not in sys.path:
+    sys.path.insert(0, str(STUDIO_BACKEND))
+
+from core.rag.chunking import Chunk, chunk_pages
+from core.rag.parsers import ParsedPage
+
+
+def _wc_counter(text: str) -> int:
+    return max(1, len(text.split()))
+
+
+def test_chunk_pages_splits_long_text():
+    text = "Lorem ipsum dolor sit amet. " * 200
+    chunks = chunk_pages(
+        [ParsedPage(text = text)],
+        max_tokens = 50,
+        overlap_tokens = 5,
+        token_counter = _wc_counter,
+    )
+    assert len(chunks) > 1
+    for chunk in chunks:
+        assert _wc_counter(chunk.text) <= 55  # max + small slack from atomic split granularity
+
+
+def test_chunk_pages_short_text_is_one_chunk():
+    text = "Just a short sentence."
+    chunks = chunk_pages(
+        [ParsedPage(text = text)],
+        max_tokens = 50,
+        overlap_tokens = 5,
+        token_counter = _wc_counter,
+    )
+    assert len(chunks) == 1
+    assert chunks[0].text == text
+
+
+def test_chunk_pages_preserves_page_numbers():
+    chunks = chunk_pages(
+        [
+            ParsedPage(text = "Page one content here.", page_number = 1),
+            ParsedPage(text = "Page two content here.", page_number = 2),
+        ],
+        max_tokens = 50,
+        overlap_tokens = 0,
+        token_counter = _wc_counter,
+    )
+    page_numbers = {c.page_number for c in chunks}
+    assert page_numbers == {1, 2}
+
+
+def test_chunk_pages_no_empty_chunks():
+    text = "\n\n\n\n\nReal content\n\n\n\n\n"
+    chunks = chunk_pages(
+        [ParsedPage(text = text)],
+        max_tokens = 50,
+        overlap_tokens = 0,
+        token_counter = _wc_counter,
+    )
+    for chunk in chunks:
+        assert chunk.text.strip()
+
+
+def test_chunk_pages_overlap_produces_repeated_tokens():
+    # Build a list of unique numbered sentences so we can detect overlap.
+    sentences = [f"sentence-{i}" for i in range(40)]
+    text = " ".join(sentences)
+    chunks = chunk_pages(
+        [ParsedPage(text = text)],
+        max_tokens = 10,
+        overlap_tokens = 4,
+        token_counter = _wc_counter,
+    )
+    if len(chunks) >= 2:
+        first_tail_words = set(chunks[0].text.split()[-4:])
+        second_head_words = set(chunks[1].text.split()[:4])
+        # At least one word should appear in both
+        assert first_tail_words & second_head_words

tests/python/test_rag_parsers.py

@@ -0,0 +1,89 @@
+"""Document parser tests — each format skipped if its lib is unavailable."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
+if str(STUDIO_BACKEND) not in sys.path:
+    sys.path.insert(0, str(STUDIO_BACKEND))
+
+
+def test_text_parser_utf8(tmp_path):
+    from core.rag.parsers import parse
+
+    file = tmp_path / "sample.txt"
+    file.write_text("hello world\n\nsecond paragraph", encoding = "utf-8")
+    pages = parse(file)
+    assert len(pages) == 1
+    assert "hello world" in pages[0].text
+    assert "second paragraph" in pages[0].text
+
+
+def test_markdown_parser_treated_as_text(tmp_path):
+    from core.rag.parsers import parse
+
+    file = tmp_path / "sample.md"
+    file.write_text("# Title\n\nBody text with **emphasis**.", encoding = "utf-8")
+    pages = parse(file)
+    assert pages and "Title" in pages[0].text
+
+
+def test_unsupported_format_raises(tmp_path):
+    from core.rag.parsers import UnsupportedFormatError, parse
+
+    file = tmp_path / "weird.xyz"
+    file.write_text("nope")
+    with pytest.raises(UnsupportedFormatError):
+        parse(file)
+
+
+def test_html_parser_strips_scripts(tmp_path):
+    pytest.importorskip("bs4")
+    pytest.importorskip("lxml")
+    from core.rag.parsers import parse
+
+    file = tmp_path / "sample.html"
+    file.write_text(
+        "<html><body><script>alert(1)</script><p>visible text</p></body></html>",
+        encoding = "utf-8",
+    )
+    pages = parse(file)
+    assert pages
+    assert "visible text" in pages[0].text
+    assert "alert" not in pages[0].text
+
+
+def test_pdf_parser_extracts_pages(tmp_path):
+    pypdf = pytest.importorskip("pypdf")
+    from pypdf import PdfWriter
+
+    file = tmp_path / "tiny.pdf"
+    writer = PdfWriter()
+    writer.add_blank_page(width = 72, height = 72)
+    with open(file, "wb") as f:
+        writer.write(f)
+    from core.rag.parsers import parse
+
+    # blank page yields no extractable text — should return [] without error
+    pages = parse(file)
+    assert isinstance(pages, list)
+
+
+def test_docx_parser_extracts_paragraphs(tmp_path):
+    docx = pytest.importorskip("docx")
+    from docx import Document
+
+    file = tmp_path / "sample.docx"
+    doc = Document()
+    doc.add_paragraph("First paragraph here.")
+    doc.add_paragraph("Second paragraph here.")
+    doc.save(str(file))
+    from core.rag.parsers import parse
+
+    pages = parse(file)
+    assert pages
+    assert "First paragraph" in pages[0].text
+    assert "Second paragraph" in pages[0].text

tests/python/test_rag_reranker.py

@@ -0,0 +1,57 @@
+"""Reranker tests — skipped if sentence_transformers is unavailable.
+
+These tests load a real CrossEncoder, so they're slow and gated under
+the ``server`` marker so a default ``pytest`` run skips them. Force
+with ``pytest -m server``.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
+if str(STUDIO_BACKEND) not in sys.path:
+    sys.path.insert(0, str(STUDIO_BACKEND))
+
+pytest.importorskip("sentence_transformers")
+
+
+def test_rerank_empty_returns_empty():
+    from core.rag.reranker import rerank
+
+    assert rerank("anything", []) == []
+
+
+@pytest.mark.server
+def test_rerank_reorders_by_relevance(monkeypatch):
+    """Hide the relevant chunk at the back of the input and check it bubbles up."""
+    monkeypatch.setenv("UNSLOTH_RAG_RERANKER_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2")
+    from core.rag.reranker import rerank, unload
+    from core.rag.retrieval import Hit
+
+    pairs = [
+        (Hit("noise1", 0.0), "Cats are small carnivorous mammals."),
+        (Hit("noise2", 0.0), "The Eiffel Tower is in Paris, France."),
+        (Hit("noise3", 0.0), "Python is a programming language."),
+        (Hit("answer", 0.0), "The speed of light in vacuum is approximately 299792458 meters per second."),
+    ]
+    try:
+        ranked = rerank("How fast does light travel?", pairs, top_k = 2)
+        assert ranked
+        assert ranked[0].chunk_id == "answer"
+    finally:
+        unload()
+
+
+@pytest.mark.server
+def test_unload_clears_singleton(monkeypatch):
+    monkeypatch.setenv("UNSLOTH_RAG_RERANKER_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2")
+    from core.rag import reranker
+    from core.rag.retrieval import Hit
+
+    reranker.rerank("q", [(Hit("a", 0.0), "some text")])
+    assert reranker._model is not None
+    reranker.unload()
+    assert reranker._model is None

tests/python/test_rag_retrieval.py

@@ -0,0 +1,46 @@
+"""Unit tests for RAG RRF fusion — no external deps."""
+
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
+if str(STUDIO_BACKEND) not in sys.path:
+    sys.path.insert(0, str(STUDIO_BACKEND))
+
+from core.rag.retrieval import Hit, _rrf_fuse
+
+
+def test_rrf_fuses_two_rankings():
+    bm25 = [Hit("a", 10.0), Hit("b", 8.0), Hit("c", 5.0)]
+    dense = [Hit("c", 0.9), Hit("b", 0.8), Hit("d", 0.5)]
+    fused = _rrf_fuse([bm25, dense], rrf_k = 60, top_k = 3)
+    ids = [h.chunk_id for h in fused]
+    # b appears at rank 2 in both -> highest fused score
+    assert ids[0] == "b"
+    assert set(ids) == {"a", "b", "c"} or set(ids) == {"b", "c", "a"}
+
+
+def test_rrf_top_k_limits_output():
+    rankings = [
+        [Hit(f"r1_{i}", 0.0) for i in range(20)],
+        [Hit(f"r2_{i}", 0.0) for i in range(20)],
+    ]
+    fused = _rrf_fuse(rankings, rrf_k = 60, top_k = 5)
+    assert len(fused) == 5
+
+
+def test_rrf_unique_ranking():
+    # Single ranking — fused order matches input order.
+    ranking = [Hit("x", 0.0), Hit("y", 0.0), Hit("z", 0.0)]
+    fused = _rrf_fuse([ranking], rrf_k = 60, top_k = 3)
+    assert [h.chunk_id for h in fused] == ["x", "y", "z"]
+
+
+def test_rrf_preserves_payload_from_first_ranking():
+    a = Hit("a", 1.0, document_id = "doc1", chunk_index = 5)
+    b = Hit("a", 2.0, document_id = "doc2", chunk_index = 7)
+    fused = _rrf_fuse([[a], [b]], rrf_k = 60, top_k = 1)
+    # First sighting wins for payload (deterministic)
+    assert fused[0].document_id == "doc1"
+    assert fused[0].chunk_index == 5

tests/python/test_rag_vector_store.py

@@ -0,0 +1,84 @@
+"""Qdrant local-mode vector store tests (skipped if qdrant-client is unavailable)."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
+if str(STUDIO_BACKEND) not in sys.path:
+    sys.path.insert(0, str(STUDIO_BACKEND))
+
+pytest.importorskip("qdrant_client")
+
+
+@pytest.fixture
+def isolated_qdrant(tmp_path, monkeypatch):
+    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+    from core.rag import vector_store
+
+    # Reset client cache so the fixture's tmp path is used.
+    vector_store._client = None
+    yield tmp_path
+    vector_store._client = None
+
+
+def test_ensure_and_upsert_and_search(isolated_qdrant):
+    from core.rag import vector_store
+
+    scope = "kb_test"
+    vector_store.ensure_collection(scope, dim = 4)
+    points = [
+        {
+            "id": "p1",
+            "vector": [1.0, 0.0, 0.0, 0.0],
+            "payload": {"document_id": "doc1", "chunk_index": 0, "text": "first"},
+        },
+        {
+            "id": "p2",
+            "vector": [0.0, 1.0, 0.0, 0.0],
+            "payload": {"document_id": "doc1", "chunk_index": 1, "text": "second"},
+        },
+    ]
+    vector_store.upsert_chunks(scope, points)
+    results = vector_store.search(scope, [1.0, 0.0, 0.0, 0.0], top_k = 2)
+    assert results
+    assert results[0]["chunk_id"] == "p1"
+
+
+def test_delete_scope_removes_collection(isolated_qdrant):
+    from core.rag import vector_store
+
+    scope = "kb_to_delete"
+    vector_store.ensure_collection(scope, dim = 3)
+    assert vector_store.collection_exists(scope)
+    vector_store.delete_scope(scope)
+    assert not vector_store.collection_exists(scope)
+
+
+def test_delete_document_removes_only_its_points(isolated_qdrant):
+    from core.rag import vector_store
+
+    scope = "kb_doc_del"
+    vector_store.ensure_collection(scope, dim = 3)
+    vector_store.upsert_chunks(
+        scope,
+        [
+            {
+                "id": "a",
+                "vector": [1.0, 0.0, 0.0],
+                "payload": {"document_id": "keep", "chunk_index": 0},
+            },
+            {
+                "id": "b",
+                "vector": [0.0, 1.0, 0.0],
+                "payload": {"document_id": "drop", "chunk_index": 0},
+            },
+        ],
+    )
+    vector_store.delete_document(scope, "drop")
+    results = vector_store.search(scope, [0.0, 1.0, 0.0], top_k = 5)
+    doc_ids = {r["payload"]["document_id"] for r in results}
+    assert "drop" not in doc_ids
+    assert "keep" in doc_ids