fix(rag): skip hidden and junk directories when indexing (#5633)

* fix(rag): skip hidden and junk directories when indexing (#5559)

index_personal_documents walked the whole tree with no pruning, so
pointing RAG at a real-world folder silently swept in .obsidian/ plugin
JS, .git/ internals, node_modules/, and __pycache__/ — multiplying
indexing time and polluting retrieval with junk chunks.

Prune hidden directories and well-known junk directories from the walk,
and skip hidden files. The explicitly passed root is exempt, so a user
who deliberately indexes a hidden directory still gets its contents.

* fix(rag): prune hidden/junk dirs in the keyword index too, via a shared helper

The #5559 fix pruned only VectorRAG.index_personal_documents (the vector index).
The parallel keyword index built by PersonalDocsManager.refresh_index ->
load_personal_index walked the same tree unpruned, so .obsidian/, .git/,
node_modules/ etc. still swept into keyword retrieval and the file listing —
the 'end-to-end' guarantee was only half true.

Single-source the pruning policy in src/index_walk (prune_index_dirs +
is_indexable_file) and use it from both walkers so they cannot drift again.
The junk-dir match is now case-insensitive, so a Node_Modules on a
case-insensitive filesystem is pruned too.

Tests: keyword-path regressions covering hidden/junk dirs, hidden files, junk
at depth (not just top level), case-insensitive junk, and the explicit-hidden-
root exemption. The existing vector tests still pass against the shared helper.

tests/test_personal_index_hidden_dirs.py

@@ -0,0 +1,63 @@
+"""Regression guard for #5559 — the KEYWORD index (load_personal_index, which
+PersonalDocsManager.refresh_index builds from) must skip hidden dirs, hidden
+files, and junk dirs at ANY depth, the same as the vector index. Both walkers
+share one pruning helper (src/index_walk) so they cannot drift again.
+"""
+import os
+
+os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
+
+from src.personal_docs import load_personal_index
+
+
+def _write(path, content="real content"):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+
+
+def _indexed(root):
+    return {rec["name"] for rec in load_personal_index(str(root))}
+
+
+def test_keyword_index_skips_hidden_and_junk_dirs(tmp_path):
+    _write(tmp_path / "note.md")
+    _write(tmp_path / "sub" / "deeper.md")
+    _write(tmp_path / ".obsidian" / "workspace.json")
+    _write(tmp_path / ".git" / "hooks.md")
+    _write(tmp_path / "node_modules" / "lib" / "readme.md")
+    _write(tmp_path / "__pycache__" / "cached.txt")
+    _write(tmp_path / "venv" / "lib" / "site.txt")
+    assert _indexed(tmp_path) == {"note.md", os.path.join("sub", "deeper.md")}
+
+
+def test_keyword_index_skips_hidden_files(tmp_path):
+    _write(tmp_path / "visible.md")
+    _write(tmp_path / ".hidden.md")
+    _write(tmp_path / "sub" / ".secret.txt")
+    assert _indexed(tmp_path) == {"visible.md"}
+
+
+def test_keyword_index_prunes_junk_at_depth(tmp_path):
+    """Pruning must apply at every level, not just the first (the vector test's
+    fixtures only nested one level under the root)."""
+    _write(tmp_path / "a" / "b" / "keep.md")
+    _write(tmp_path / "a" / "b" / "node_modules" / "dep.md")
+    _write(tmp_path / "a" / ".obsidian" / "deep.json")
+    assert _indexed(tmp_path) == {os.path.join("a", "b", "keep.md")}
+
+
+def test_keyword_index_junk_match_is_case_insensitive(tmp_path):
+    """A case-variant junk dir must still be pruned (macOS default FS is
+    case-insensitive, so `Node_Modules` and `node_modules` are the same dir)."""
+    _write(tmp_path / "keep.md")
+    _write(tmp_path / "Node_Modules" / "dep.md")
+    assert _indexed(tmp_path) == {"keep.md"}
+
+
+def test_keyword_index_explicit_hidden_root_still_indexed(tmp_path):
+    """Children-only pruning: pointing indexing at a hidden dir gets its
+    contents, minus nested hidden/junk."""
+    root = tmp_path / ".notes"
+    _write(root / "idea.md")
+    _write(root / ".obsidian" / "plugin.json")
+    assert {rec["name"] for rec in load_personal_index(str(root))} == {"idea.md"}

tests/test_rag_index_hidden_dirs.py

@@ -0,0 +1,78 @@
+"""Regression guard for #5559 — directory indexing must skip hidden directories,
+hidden files, and well-known junk directories.
+
+VectorRAG.index_personal_documents walked the whole tree with no pruning, so
+pointing RAG at a real-world folder (an Obsidian vault, a git repo) swept in
+`.obsidian/` plugin JavaScript, `.git/` internals, `node_modules/`, etc. The
+junk multiplied indexing time and polluted retrieval.
+
+These tests are hermetic — no chromadb; VectorRAG is created via __new__ (skip
+Chroma connect) with add_document stubbed to record which files get indexed.
+"""
+import os
+
+os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
+
+import src.rag_vector as rag_vector
+
+
+def _make_rag(recorded_sources):
+    rag = rag_vector.VectorRAG.__new__(rag_vector.VectorRAG)  # skip Chroma connect
+
+    def _record(text, metadata):
+        recorded_sources.add(metadata["source"])
+        return True
+
+    rag.add_document = _record
+    return rag
+
+
+def _write(path, content="some real content"):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+
+
+def test_index_skips_hidden_and_junk_directories(tmp_path):
+    _write(tmp_path / "note.md")
+    _write(tmp_path / "sub" / "deeper.md")
+    _write(tmp_path / ".obsidian" / "plugins" / "plugin.js")
+    _write(tmp_path / ".git" / "hooks.js")
+    _write(tmp_path / "node_modules" / "lib" / "index.js")
+    _write(tmp_path / "__pycache__" / "cached.py")
+    _write(tmp_path / "venv" / "lib" / "site.py")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    result = rag.index_personal_documents(str(tmp_path))
+
+    assert result["success"] is True
+    indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded}
+    assert indexed == {"note.md", os.path.join("sub", "deeper.md")}
+
+
+def test_index_skips_hidden_files(tmp_path):
+    _write(tmp_path / "visible.md")
+    _write(tmp_path / ".hidden.md")
+    _write(tmp_path / "sub" / ".secret.txt")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    rag.index_personal_documents(str(tmp_path))
+
+    indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded}
+    assert indexed == {"visible.md"}
+
+
+def test_explicitly_passed_hidden_root_is_still_indexed(tmp_path):
+    """Pruning applies to children only — a user who deliberately points RAG at
+    a hidden directory gets its contents, minus nested hidden/junk dirs."""
+    root = tmp_path / ".notes"
+    _write(root / "idea.md")
+    _write(root / ".obsidian" / "plugin.js")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    rag.index_personal_documents(str(root))
+
+    indexed = {os.path.relpath(p, str(root)) for p in recorded}
+    assert indexed == {"idea.md"}