Studio: add RAG with hybrid search, reranker, chat integration

Backend (studio/backend/): - core/rag/: parsers (PDF/TXT/MD/DOCX/HTML via pypdf/python-docx/bs4), recursive token-aware chunker, embeddings singleton via FastSentenceTransformer.from_pretrained(for_inference=True), Qdrant local vector store, bm25s lexical index, RRF hybrid retrieval, spawn-subprocess ingestion job with SSE progress, optional CrossEncoder reranker (off-by-default). - routes/rag.py: KB CRUD, doc upload (KB + per-thread), doc list/delete, ingestion SSE, hybrid+rerank search, thread-index list/clear. - routes/chat_history.py: purge thread RAG artifacts on thread delete and clear-all (rag_documents has no FK cascade to chat_threads so uploads work on un-persisted threads). - studio.db gains 4 RAG tables; storage_roots gains rag_*() helpers. - auth/authentication.py: get_current_subject_sse accepts ?token=... so EventSource can stream ingestion progress. Frontend (studio/frontend/): - features/rag/: api client, Zustand store, hooks, dropzone, KB list, doc rows, ingestion-progress, thread-index list components. - Settings dialog gains a Knowledge Bases tab (master/detail + thread documents list); /knowledge-bases deep-links to it. - features/chat/: per-thread ragSource/enableRerank/ragTopK state in chat-runtime-store; Retrieval section in chat-settings-sheet with KB DropdownMenu (active highlight + per-row trash), thread doc list with Clear-thread-index button, RAG Top K slider, reranker toggle; chat-adapter retrieves before /v1/chat/completions and injects hits as a system block; shared-composer + button routes documents into pendingDocs (auto-uploads, send blocked while indexing).
2026-05-23 18:39:58 +04:00 · 2026-05-23 18:39:58 +04:00 · 92994e8b83
commit 92994e8b83
parent 83b20976f7
51 changed files with 4527 additions and 6 deletions
--- a/tests/python/test_rag_parsers.py
+++ b/tests/python/test_rag_parsers.py
@ -0,0 +1,89 @@
+"""Document parser tests — each format skipped if its lib is unavailable."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
+if str(STUDIO_BACKEND) not in sys.path:
+    sys.path.insert(0, str(STUDIO_BACKEND))
+
+
+def test_text_parser_utf8(tmp_path):
+    from core.rag.parsers import parse
+
+    file = tmp_path / "sample.txt"
+    file.write_text("hello world\n\nsecond paragraph", encoding = "utf-8")
+    pages = parse(file)
+    assert len(pages) == 1
+    assert "hello world" in pages[0].text
+    assert "second paragraph" in pages[0].text
+
+
+def test_markdown_parser_treated_as_text(tmp_path):
+    from core.rag.parsers import parse
+
+    file = tmp_path / "sample.md"
+    file.write_text("# Title\n\nBody text with **emphasis**.", encoding = "utf-8")
+    pages = parse(file)
+    assert pages and "Title" in pages[0].text
+
+
+def test_unsupported_format_raises(tmp_path):
+    from core.rag.parsers import UnsupportedFormatError, parse
+
+    file = tmp_path / "weird.xyz"
+    file.write_text("nope")
+    with pytest.raises(UnsupportedFormatError):
+        parse(file)
+
+
+def test_html_parser_strips_scripts(tmp_path):
+    pytest.importorskip("bs4")
+    pytest.importorskip("lxml")
+    from core.rag.parsers import parse
+
+    file = tmp_path / "sample.html"
+    file.write_text(
+        "<html><body><script>alert(1)</script><p>visible text</p></body></html>",
+        encoding = "utf-8",
+    )
+    pages = parse(file)
+    assert pages
+    assert "visible text" in pages[0].text
+    assert "alert" not in pages[0].text
+
+
+def test_pdf_parser_extracts_pages(tmp_path):
+    pypdf = pytest.importorskip("pypdf")
+    from pypdf import PdfWriter
+
+    file = tmp_path / "tiny.pdf"
+    writer = PdfWriter()
+    writer.add_blank_page(width = 72, height = 72)
+    with open(file, "wb") as f:
+        writer.write(f)
+    from core.rag.parsers import parse
+
+    # blank page yields no extractable text — should return [] without error
+    pages = parse(file)
+    assert isinstance(pages, list)
+
+
+def test_docx_parser_extracts_paragraphs(tmp_path):
+    docx = pytest.importorskip("docx")
+    from docx import Document
+
+    file = tmp_path / "sample.docx"
+    doc = Document()
+    doc.add_paragraph("First paragraph here.")
+    doc.add_paragraph("Second paragraph here.")
+    doc.save(str(file))
+    from core.rag.parsers import parse
+
+    pages = parse(file)
+    assert pages
+    assert "First paragraph" in pages[0].text
+    assert "Second paragraph" in pages[0].text