68114fd223
When a KB has mode = 'multimodal', ingestion extracts images alongside
text and embeds both into a shared 512-d vector space via BGE-VL-base.
Image hits become first-class search results — useful for slides,
reports, and diagrams where text-only retrieval loses ~30-50% of the
content.
Backend
- embeddings.py: new encode_images(image_bytes_list) — opens bytes via
PIL and routes to the SentenceTransformer (BGE-VL accepts PIL images
in the same encode call as text).
- ingestion.py: _subprocess_worker gains document_id arg and a new
_stream_image_chunks() helper. For multimodal KBs the standard text
chunking runs first, then images are saved to
rag_uploads_root() / 'images' / <document_id> / img-NNNN.<ext> and
embedded; for each image with an adjacent caption, both an
'image'-kind chunk (vector = encoded image) and a 'caption'-kind
chunk (vector = encoded caption text) are streamed back with a
shared pair_group field.
- ingestion.py parent: _insert_chunks_and_collect_for_bm25 now reads
kind / image_path / pair_group from the subprocess message,
populates the new rag_chunks columns, and runs a second pass that
sets linked_chunk_id for each image ↔ caption pair. BM25 indexes
text + caption chunks only — image chunks have no tokenisable body.
- retrieval.py: Hit gains a `kind` field plumbed through bm25, dense,
RRF, and rerank paths.
- reranker.py: image-kind hits skip CrossEncoder rerank (text-only
model) but are appended back in their original relative position
rather than dropped.
- routes/rag.py: new GET /api/rag/images/{document_id}/{filename}
static-file route with realpath containment check. SearchHit gains
`kind` and `image_url` fields so the chat UI can render image
thumbnails alongside text hits. KB-doc upload threads kind/mode
through to ingestion.
Frontend
- rag-api.ts: SearchHit gains optional `kind` and `image_url`.
- kb-create-dialog.tsx: new Mode select (Text / Multimodal) alongside
the existing Chunking strategy select. The forbidden
(multimodal + late) combo is enforced in the UI — each side
disables the conflicting option on the other side with a tooltip
explaining why. Embedding-model placeholder cycles through the
three valid defaults (bge-small / nomic / BGE-VL).
- kb-list.tsx + chat-settings-sheet.tsx: 🖼️ MM badge alongside the
⚡ Late one so multimodal KBs are obvious at a glance.
Tests
- test_rag_multimodal.py: parser returns images when want_images=True
and skips them when False; _validate_mode_combo rejects the
forbidden (multimodal, late) pair with 400; RAG_EMBEDDER_MATRIX
contains the three valid combos and excludes the forbidden one;
image URL construction shape is verified. A server-marked test
loads BGE-VL-base end-to-end and confirms image + text vectors
share the same dimension.
Phase 3 of the plan is now feature-complete on the backend; the
remaining items (re-ingest UX for changing strategy on existing KBs)
are tracked under "Backfill UX" and can land separately.
116 lines
3.1 KiB
Python
116 lines
3.1 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""High-level retrieval surface for RAG: BM25, dense, and RRF hybrid.
|
|
|
|
Reciprocal Rank Fusion is parameter-light: each candidate's fused score
|
|
is the sum of ``1 / (rrf_k + rank)`` across rankers. It avoids the
|
|
need to calibrate score scales between BM25 (raw, unbounded) and cosine
|
|
similarity (-1..1).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from utils.rag.config import (
|
|
RAG_RRF_K,
|
|
RAG_TOP_K_BM25,
|
|
RAG_TOP_K_DENSE,
|
|
RAG_TOP_K_HYBRID,
|
|
)
|
|
|
|
from . import bm25, embeddings, vector_store
|
|
|
|
|
|
@dataclass(frozen = True)
|
|
class Hit:
|
|
chunk_id: str
|
|
score: float
|
|
document_id: str | None = None
|
|
chunk_index: int | None = None
|
|
kind: str = "text"
|
|
|
|
|
|
def retrieve_bm25(scope: str, query: str, k: int | None = None) -> list[Hit]:
|
|
limit = k or RAG_TOP_K_BM25
|
|
return [Hit(chunk_id = cid, score = s) for cid, s in bm25.search(scope, query, limit)]
|
|
|
|
|
|
def retrieve_dense(
|
|
scope: str,
|
|
query: str,
|
|
k: int | None = None,
|
|
*,
|
|
document_ids: list[str] | None = None,
|
|
) -> list[Hit]:
|
|
limit = k or RAG_TOP_K_DENSE
|
|
vector = embeddings.encode([query], normalize = True)[0].tolist()
|
|
raw = vector_store.search(
|
|
scope,
|
|
query_vector = vector,
|
|
top_k = limit,
|
|
document_ids = document_ids,
|
|
)
|
|
out: list[Hit] = []
|
|
for r in raw:
|
|
payload = r["payload"]
|
|
out.append(
|
|
Hit(
|
|
chunk_id = r["chunk_id"],
|
|
score = r["score"],
|
|
document_id = payload.get("document_id"),
|
|
chunk_index = payload.get("chunk_index"),
|
|
kind = payload.get("kind", "text"),
|
|
)
|
|
)
|
|
return out
|
|
|
|
|
|
def _rrf_fuse(
|
|
rankings: list[list[Hit]],
|
|
*,
|
|
rrf_k: int,
|
|
top_k: int,
|
|
) -> list[Hit]:
|
|
fused: dict[str, float] = {}
|
|
seen: dict[str, Hit] = {}
|
|
for ranking in rankings:
|
|
for rank, hit in enumerate(ranking):
|
|
fused[hit.chunk_id] = fused.get(hit.chunk_id, 0.0) + 1.0 / (rrf_k + rank + 1)
|
|
if hit.chunk_id not in seen:
|
|
seen[hit.chunk_id] = hit
|
|
ordered = sorted(fused.items(), key = lambda kv: kv[1], reverse = True)[:top_k]
|
|
return [
|
|
Hit(
|
|
chunk_id = cid,
|
|
score = score,
|
|
document_id = seen[cid].document_id,
|
|
chunk_index = seen[cid].chunk_index,
|
|
kind = seen[cid].kind,
|
|
)
|
|
for cid, score in ordered
|
|
]
|
|
|
|
|
|
def retrieve_hybrid(
|
|
scope: str,
|
|
query: str,
|
|
*,
|
|
k: int | None = None,
|
|
k_bm25: int | None = None,
|
|
k_dense: int | None = None,
|
|
document_ids: list[str] | None = None,
|
|
) -> list[Hit]:
|
|
bm25_hits = retrieve_bm25(scope, query, k_bm25 or RAG_TOP_K_BM25)
|
|
dense_hits = retrieve_dense(
|
|
scope,
|
|
query,
|
|
k_dense or RAG_TOP_K_DENSE,
|
|
document_ids = document_ids,
|
|
)
|
|
return _rrf_fuse(
|
|
[bm25_hits, dense_hits],
|
|
rrf_k = RAG_RRF_K,
|
|
top_k = k or RAG_TOP_K_HYBRID,
|
|
)
|