92994e8b83
Backend (studio/backend/): - core/rag/: parsers (PDF/TXT/MD/DOCX/HTML via pypdf/python-docx/bs4), recursive token-aware chunker, embeddings singleton via FastSentenceTransformer.from_pretrained(for_inference=True), Qdrant local vector store, bm25s lexical index, RRF hybrid retrieval, spawn-subprocess ingestion job with SSE progress, optional CrossEncoder reranker (off-by-default). - routes/rag.py: KB CRUD, doc upload (KB + per-thread), doc list/delete, ingestion SSE, hybrid+rerank search, thread-index list/clear. - routes/chat_history.py: purge thread RAG artifacts on thread delete and clear-all (rag_documents has no FK cascade to chat_threads so uploads work on un-persisted threads). - studio.db gains 4 RAG tables; storage_roots gains rag_*() helpers. - auth/authentication.py: get_current_subject_sse accepts ?token=... so EventSource can stream ingestion progress. Frontend (studio/frontend/): - features/rag/: api client, Zustand store, hooks, dropzone, KB list, doc rows, ingestion-progress, thread-index list components. - Settings dialog gains a Knowledge Bases tab (master/detail + thread documents list); /knowledge-bases deep-links to it. - features/chat/: per-thread ragSource/enableRerank/ragTopK state in chat-runtime-store; Retrieval section in chat-settings-sheet with KB DropdownMenu (active highlight + per-row trash), thread doc list with Clear-thread-index button, RAG Top K slider, reranker toggle; chat-adapter retrieves before /v1/chat/completions and injects hits as a system block; shared-composer + button routes documents into pendingDocs (auto-uploads, send blocked while indexing).
84 lines
2.5 KiB
Python
84 lines
2.5 KiB
Python
"""Qdrant local-mode vector store tests (skipped if qdrant-client is unavailable)."""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
STUDIO_BACKEND = REPO_ROOT / "studio" / "backend"
|
|
if str(STUDIO_BACKEND) not in sys.path:
|
|
sys.path.insert(0, str(STUDIO_BACKEND))
|
|
|
|
pytest.importorskip("qdrant_client")
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_qdrant(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
|
|
from core.rag import vector_store
|
|
|
|
# Reset client cache so the fixture's tmp path is used.
|
|
vector_store._client = None
|
|
yield tmp_path
|
|
vector_store._client = None
|
|
|
|
|
|
def test_ensure_and_upsert_and_search(isolated_qdrant):
|
|
from core.rag import vector_store
|
|
|
|
scope = "kb_test"
|
|
vector_store.ensure_collection(scope, dim = 4)
|
|
points = [
|
|
{
|
|
"id": "p1",
|
|
"vector": [1.0, 0.0, 0.0, 0.0],
|
|
"payload": {"document_id": "doc1", "chunk_index": 0, "text": "first"},
|
|
},
|
|
{
|
|
"id": "p2",
|
|
"vector": [0.0, 1.0, 0.0, 0.0],
|
|
"payload": {"document_id": "doc1", "chunk_index": 1, "text": "second"},
|
|
},
|
|
]
|
|
vector_store.upsert_chunks(scope, points)
|
|
results = vector_store.search(scope, [1.0, 0.0, 0.0, 0.0], top_k = 2)
|
|
assert results
|
|
assert results[0]["chunk_id"] == "p1"
|
|
|
|
|
|
def test_delete_scope_removes_collection(isolated_qdrant):
|
|
from core.rag import vector_store
|
|
|
|
scope = "kb_to_delete"
|
|
vector_store.ensure_collection(scope, dim = 3)
|
|
assert vector_store.collection_exists(scope)
|
|
vector_store.delete_scope(scope)
|
|
assert not vector_store.collection_exists(scope)
|
|
|
|
|
|
def test_delete_document_removes_only_its_points(isolated_qdrant):
|
|
from core.rag import vector_store
|
|
|
|
scope = "kb_doc_del"
|
|
vector_store.ensure_collection(scope, dim = 3)
|
|
vector_store.upsert_chunks(
|
|
scope,
|
|
[
|
|
{
|
|
"id": "a",
|
|
"vector": [1.0, 0.0, 0.0],
|
|
"payload": {"document_id": "keep", "chunk_index": 0},
|
|
},
|
|
{
|
|
"id": "b",
|
|
"vector": [0.0, 1.0, 0.0],
|
|
"payload": {"document_id": "drop", "chunk_index": 0},
|
|
},
|
|
],
|
|
)
|
|
vector_store.delete_document(scope, "drop")
|
|
results = vector_store.search(scope, [0.0, 1.0, 0.0], top_k = 5)
|
|
doc_ids = {r["payload"]["document_id"] for r in results}
|
|
assert "drop" not in doc_ids
|
|
assert "keep" in doc_ids
|