From f522545b65c670b06e9819fe2b04bb8d5d779c03 Mon Sep 17 00:00:00 2001
From: Roland Tannous <rolandtannous@gravityq.ai>
Date: Tue, 26 May 2026 22:07:21 +0400
Subject: [PATCH] Studio: strip pymupdf4llm picture-text markers; fail ingest
 cleanly on FK error

---
 studio/backend/core/rag/ingestion.py   | 24 +++++++++++++++++-------
 studio/backend/core/rag/parsers/pdf.py | 16 +++++++++++++++-
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.py
index 3965b40e0f..faf55fccb8 100644
--- a/studio/backend/core/rag/ingestion.py
+++ b/studio/backend/core/rag/ingestion.py
@@ -12,6 +12,7 @@ from __future__ import annotations
 import logging
 import multiprocessing as mp
 import queue as queue_module
+import sqlite3
 import threading
 import time
 from pathlib import Path
@@ -564,13 +565,22 @@ def _pump(
                     embedding_dim = len(msg["vectors"][0]) if msg["vectors"] else None
                     if embedding_dim is not None:
                         vector_store.ensure_collection(state.scope, embedding_dim)
-                bm25_rows = _insert_chunks_and_collect_for_bm25(
-                    state.document_id,
-                    state.scope,
-                    int(msg["first_index"]),
-                    msg["chunks"],
-                    msg["vectors"],
-                )
+                try:
+                    bm25_rows = _insert_chunks_and_collect_for_bm25(
+                        state.document_id,
+                        state.scope,
+                        int(msg["first_index"]),
+                        msg["chunks"],
+                        msg["vectors"],
+                    )
+                except sqlite3.IntegrityError as exc:
+                    # rag_documents row was deleted mid-ingest (user removed
+                    # the chip / cleared the index). Fail the job cleanly
+                    # rather than crashing the pump thread.
+                    final_error = (
+                        f"document was removed before ingestion finished ({exc})"
+                    )
+                    break
                 bm25_buffer.extend(bm25_rows)
             elif mtype == "complete":
                 final_status = "completed"
diff --git a/studio/backend/core/rag/parsers/pdf.py b/studio/backend/core/rag/parsers/pdf.py
index eeb46e2ce7..b9e6696a15 100644
--- a/studio/backend/core/rag/parsers/pdf.py
+++ b/studio/backend/core/rag/parsers/pdf.py
@@ -6,12 +6,26 @@
 from __future__ import annotations
 
 import logging
+import re
 from pathlib import Path
 
 from . import ParsedImage, ParsedPage, ParseResult
 
 logger = logging.getLogger(__name__)
 
+# pymupdf4llm wraps OCR'd vector-graphics text with these markers even when
+# `ignore_images=True`. Strip the whole block — the VLM captioner produces
+# a proper description for the figure, and the marker text just pollutes
+# the chunked body / shows up verbatim in citations.
+_PICTURE_TEXT_BLOCK_RE = re.compile(
+    r"-{3,}\s*Start of picture text\s*-{3,}.*?-{3,}\s*End of picture text\s*-{3,}",
+    re.DOTALL | re.IGNORECASE,
+)
+
+
+def _strip_picture_text_markers(md: str) -> str:
+    return _PICTURE_TEXT_BLOCK_RE.sub("", md)
+
 
 def _extract_with_pymupdf(path: Path, want_images: bool) -> ParseResult:
     import pymupdf
@@ -32,7 +46,7 @@ def _extract_with_pymupdf(path: Path, want_images: bool) -> ParseResult:
             except Exception:
                 # pymupdf4llm can choke on a single page; fall back to plain text.
                 md = doc[page_index].get_text("text") or ""
-            md = md.strip()
+            md = _strip_picture_text_markers(md).strip()
             if md:
                 pages.append(ParsedPage(text = md, page_number = page_index + 1))