Studio: strip pymupdf4llm picture-text markers; fail ingest cleanly on FK error

studio/backend/core/rag/ingestion.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 import logging
 import multiprocessing as mp
 import queue as queue_module
+import sqlite3
 import threading
 import time
 from pathlib import Path
@@ -564,13 +565,22 @@ def _pump(
                     embedding_dim = len(msg["vectors"][0]) if msg["vectors"] else None
                     if embedding_dim is not None:
                         vector_store.ensure_collection(state.scope, embedding_dim)
-                bm25_rows = _insert_chunks_and_collect_for_bm25(
-                    state.document_id,
-                    state.scope,
-                    int(msg["first_index"]),
-                    msg["chunks"],
-                    msg["vectors"],
-                )
+                try:
+                    bm25_rows = _insert_chunks_and_collect_for_bm25(
+                        state.document_id,
+                        state.scope,
+                        int(msg["first_index"]),
+                        msg["chunks"],
+                        msg["vectors"],
+                    )
+                except sqlite3.IntegrityError as exc:
+                    # rag_documents row was deleted mid-ingest (user removed
+                    # the chip / cleared the index). Fail the job cleanly
+                    # rather than crashing the pump thread.
+                    final_error = (
+                        f"document was removed before ingestion finished ({exc})"
+                    )
+                    break
                 bm25_buffer.extend(bm25_rows)
             elif mtype == "complete":
                 final_status = "completed"

studio/backend/core/rag/parsers/pdf.py

@@ -6,12 +6,26 @@
 from __future__ import annotations
 
 import logging
+import re
 from pathlib import Path
 
 from . import ParsedImage, ParsedPage, ParseResult
 
 logger = logging.getLogger(__name__)
 
+# pymupdf4llm wraps OCR'd vector-graphics text with these markers even when
+# `ignore_images=True`. Strip the whole block — the VLM captioner produces
+# a proper description for the figure, and the marker text just pollutes
+# the chunked body / shows up verbatim in citations.
+_PICTURE_TEXT_BLOCK_RE = re.compile(
+    r"-{3,}\s*Start of picture text\s*-{3,}.*?-{3,}\s*End of picture text\s*-{3,}",
+    re.DOTALL | re.IGNORECASE,
+)
+
+
+def _strip_picture_text_markers(md: str) -> str:
+    return _PICTURE_TEXT_BLOCK_RE.sub("", md)
+
 
 def _extract_with_pymupdf(path: Path, want_images: bool) -> ParseResult:
     import pymupdf
@@ -32,7 +46,7 @@ def _extract_with_pymupdf(path: Path, want_images: bool) -> ParseResult:
             except Exception:
                 # pymupdf4llm can choke on a single page; fall back to plain text.
                 md = doc[page_index].get_text("text") or ""
-            md = md.strip()
+            md = _strip_picture_text_markers(md).strip()
             if md:
                 pages.append(ParsedPage(text = md, page_number = page_index + 1))