From c7c1b7717e99395889a17817923b3def34799bf7 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira Date: Sun, 26 Jul 2026 12:44:34 +0100 Subject: [PATCH 01/43] docs(discovery): publish provisional feature baseline --- docs/discovery/BASELINE-STATUS.md | 62 + docs/discovery/README.md | 21 + docs/discovery/audit-method.md | 80 + docs/discovery/domains/agent.md | 153 + docs/discovery/domains/calendar.md | 45 + docs/discovery/domains/chat.md | 197 + docs/discovery/domains/contact.md | 24 + docs/discovery/domains/cookbook.md | 85 + docs/discovery/domains/document.md | 108 + docs/discovery/domains/email.md | 87 + docs/discovery/domains/frontend.md | 149 + docs/discovery/domains/media.md | 170 + docs/discovery/domains/memory.md | 24 + docs/discovery/domains/model.md | 177 + docs/discovery/domains/note.md | 24 + docs/discovery/domains/platform.md | 213 + docs/discovery/domains/research.md | 86 + docs/discovery/domains/security.md | 154 + docs/discovery/domains/skill.md | 24 + docs/discovery/feature-catalog.json | 4797 +++++++++++++++++ docs/discovery/feature-catalog.md | 85 + .../discovery/references/source-provenance.md | 28 + .../tools/test_validate_discovery_evidence.py | 224 + .../tools/validate_discovery_consistency.py | 592 ++ .../tools/validate_discovery_docs.py | 172 + .../tools/validate_discovery_evidence.py | 1413 +++++ 26 files changed, 9194 insertions(+) create mode 100644 docs/discovery/BASELINE-STATUS.md create mode 100644 docs/discovery/README.md create mode 100644 docs/discovery/audit-method.md create mode 100644 docs/discovery/domains/agent.md create mode 100644 docs/discovery/domains/calendar.md create mode 100644 docs/discovery/domains/chat.md create mode 100644 docs/discovery/domains/contact.md create mode 100644 docs/discovery/domains/cookbook.md create mode 100644 docs/discovery/domains/document.md create mode 100644 docs/discovery/domains/email.md create mode 100644 docs/discovery/domains/frontend.md create mode 100644 docs/discovery/domains/media.md create mode 100644 docs/discovery/domains/memory.md create mode 100644 docs/discovery/domains/model.md create mode 100644 docs/discovery/domains/note.md create mode 100644 docs/discovery/domains/platform.md create mode 100644 docs/discovery/domains/research.md create mode 100644 docs/discovery/domains/security.md create mode 100644 docs/discovery/domains/skill.md create mode 100644 docs/discovery/feature-catalog.json create mode 100644 docs/discovery/feature-catalog.md create mode 100644 docs/discovery/references/source-provenance.md create mode 100644 docs/discovery/tools/test_validate_discovery_evidence.py create mode 100755 docs/discovery/tools/validate_discovery_consistency.py create mode 100755 docs/discovery/tools/validate_discovery_docs.py create mode 100644 docs/discovery/tools/validate_discovery_evidence.py diff --git a/docs/discovery/BASELINE-STATUS.md b/docs/discovery/BASELINE-STATUS.md new file mode 100644 index 000000000..7cb6ec391 --- /dev/null +++ b/docs/discovery/BASELINE-STATUS.md @@ -0,0 +1,62 @@ +# Discovery Baseline Status + +## Purpose and scope + +This package is a commit-pinned discovery baseline and feature index for the `discovery` branch at frozen commit `d8a2059df8e53bc7275c45339849d14c8651e73c`. It inventories **79 feature records** across **16 domains** to help maintainers locate likely implementation areas and identify validation gaps. It is not an authoritative architecture reference or a runtime-certification record. + +The canonical inventory is [`feature-catalog.json`](feature-catalog.json); [`feature-catalog.md`](feature-catalog.md) and the files in [`domains/`](domains/) are derived reading views. See [`audit-method.md`](audit-method.md) for the status and maturity definitions. + +## What maintainers may use now + +- Use the catalog and domain views as a frozen discovery index, including their feature IDs, stated scope, likely source locations, and declared runtime prerequisites. +- Treat a catalog status such as `verified` as meaning implementation was identified during discovery. It does **not** mean every evidence locator, line range, test claim, or runtime behaviour has passed semantic validation. +- Use the structural checks to confirm package shape and cross-view consistency; use the evidence validator to assess whether individual evidence assertions are semantically supported. + +## Validation snapshot + +Structural validation passes: the catalog has 79 records, the 16 domain views match it, and the package structural validators pass. All 11 focused evidence-validator tests pass. + +The evidence validator found **170 evidence entries**: **82 valid**, **66 invalid**, **6 ambiguous**, and **16 unsupported**. Its non-zero exit is expected while those semantic evidence defects remain. + +Structural validity checks the documentation schema, record counts, derived-view consistency, file existence, line-range bounds, links, and public-safety rules. Semantic evidence validity additionally checks whether the cited locator exists, falls within its cited range, uses a supported parser, and actually supports the feature claim. Passing the former does not establish the latter. + +## E2 review decisions + +E2 means directly relevant automated test evidence supports the feature claim; a test file’s existence alone is insufficient. The generated evidence report was used to reassess all ten E2 records. + +| Feature | Decision | Reason | +|---|---|---| +| `CHAT-001` | Retain E2 | Two cited tests directly exercise documented streaming-related behaviour. | +| `MODEL-006` | Retain E2 | The cited device-flow test exercises the Copilot start/poll contract and verification URI. | +| `MODEL-007` | Retain E2 | The cited device-flow test exercises the ChatGPT subscription contract and verification URI. | +| `RESEARCH-003` | Demote to E1 | The cited test covers ranking, not provider connectivity or dispatch; the route-to-provider implementation path was identified. | +| `DOCUMENT-002` | Demote to E1 | The cited marker test does not support form processing or rendering; the document route does call the PDF form handlers. | +| `EMAIL-001` | Demote to E1 | The cited health test is narrower than setup, SMTP, and polling; application setup invokes the email router and its poller. | +| `SECURITY-002` | Demote to E1 | Relevant vault-password tests exist, but the cited test locator is fabricated; the application mounts the vault route implementation. | +| `SECURITY-004` | Demote to E1 | Relevant injection tests exist, but the cited locator names are fabricated; callers use the documented context guard. | +| `SECURITY-005` | Demote to E1 | Relevant guard tests exist, but the cited locator names are fabricated; route code calls the documented URL guard. | +| `PLATFORM-009` | Demote to E0 | The manifests and diagnostic script establish discovered operational artifacts, not a traced application path or directly relevant automated test. | + +The current maturity distribution is **E0: 68**, **E1: 8**, **E2: 3**, **E3: 0**, **E4: 0**. Runtime validation is still pending where the catalog says it requires external services, interactive authentication, specialised hardware, or host Docker GPU support. + +## Known discovery caveats + +- `AGENT-004` includes a legacy no-op activity-log shim rather than active assistant-log behaviour. +- `FRONTEND-007` points to a missing `static/backgrounds.html` target; its existing variant pages do not make that route functional. +- `RESEARCH-003` retains a compatibility module that aliases the canonical search implementation, and `DOCUMENT-002` separates optional PDF viewing from form handling. +- Secret-storage and vault-command handling are distinct implementation areas; this index does not make an end-to-end security guarantee. + +## Recommended next documentation work + +Repair semantic evidence selectively while architecture and operations documentation is written, beginning with the seven E2 demotions and maintainer-owned feature descriptions. Do not wait for the complete evidence queue before documenting the system. Record controlled runtime observations when external services, credentials, hardware, or Docker GPU access are available, and label unsupported claims explicitly. + +## Validation commands + +```bash +PYTHONDONTWRITEBYTECODE=1 python3 docs/discovery/tools/validate_discovery_docs.py --repo-root . +PYTHONDONTWRITEBYTECODE=1 python3 docs/discovery/tools/validate_discovery_consistency.py +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest docs/discovery/tools/test_validate_discovery_evidence.py -v +PYTHONDONTWRITEBYTECODE=1 python3 docs/discovery/tools/validate_discovery_evidence.py --repo-root . --catalog docs/discovery/feature-catalog.json --output-dir +``` + +Supply a local report directory outside `docs/discovery/` for the final command so generated reports are not added to the package. diff --git a/docs/discovery/README.md b/docs/discovery/README.md new file mode 100644 index 000000000..1a397da74 --- /dev/null +++ b/docs/discovery/README.md @@ -0,0 +1,21 @@ +# Odysseus Discovery Package + +## Provisional discovery baseline + +This is a commit-pinned discovery baseline and feature index for the `discovery` branch at `d8a2059df8e53bc7275c45339849d14c8651e73c`. It contains 79 feature records across 16 domains. It is **not** an authoritative architecture reference, a runtime certification, or a claim that every evidence citation is semantically valid. + +Read [`BASELINE-STATUS.md`](BASELINE-STATUS.md) first for the publication status, evidence-validation totals, E2 decisions, known caveats, and the recommended next documentation work. + +## Package contents + +- [`feature-catalog.json`](feature-catalog.json) is the canonical machine-readable catalog. +- [`feature-catalog.md`](feature-catalog.md) and [`domains/`](domains/) are derived reading views. +- [`audit-method.md`](audit-method.md) defines feature status and evidence maturity. +- [`references/source-provenance.md`](references/source-provenance.md) records the frozen snapshot. +- [`tools/`](tools/) contains the structural, consistency, and evidence validators. + +A feature status such as `verified` means implementation was identified during discovery. It does not mean every evidence locator, line range, test claim, or runtime behaviour has passed semantic validation. + +## Validation + +Run the commands in [`BASELINE-STATUS.md`](BASELINE-STATUS.md#validation-commands). Structural checks and semantic evidence checks have different purposes; see that status document for the current results and interpretation. diff --git a/docs/discovery/audit-method.md b/docs/discovery/audit-method.md new file mode 100644 index 000000000..b2c3bd1f1 --- /dev/null +++ b/docs/discovery/audit-method.md @@ -0,0 +1,80 @@ +# Odysseus Functional Audit Methodology + +## Purpose + +This document specifies the methodology and evidence standards for the read-only discovery audit of **Odysseus**. + +## Snapshot Baseline + +- **Repository**: `odysseus-dev/odysseus` +- **Audit Target Branch**: `discovery` +- **Frozen Commit SHA**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Snapshot Date**: `2026-07-23T14:49:02Z` +- **Audit Mode**: Read-Only inventory & documentation review + +## Rules of Engagement + +1. **No Code Mutations**: Application code and tests outside `docs/discovery/` remain untouched. +2. **No External Operations**: No GitHub issues, PRs, comments, labels, or branch mutations. +3. **Zero Inferred Success**: Documentation claims require empirical evidence of implementation and reachability. Filenames, README descriptions, and docstrings alone do not constitute proof. +4. **Strict Status Categorization**: All capabilities are assigned exactly one authorized status: + - `verified`: Implemented, reachable, and supported by code evidence. + - `partial`: Partially implemented or missing full frontend/backend connection. + - `disabled`: Gated off by default feature flags or configuration. + - `experimental`: Active but requiring non-standard hardware or runtimes. + - `legacy`: Obsolete feature retained for backwards compatibility. + - `dead-code-candidate`: Code exists but is unreachable from UI or API routes. + - `unverified`: Implementation present but untestable without external secrets or hardware. + +## Evidence Maturity Scale + +Evidence maturity is evaluated independently from catalog feature status. A feature status such as `verified` records that implementation was identified during discovery; it is not a statement that every evidence locator, test claim, line range, or runtime behaviour has passed semantic validation. + +- **E0 - Discovered**: Candidate identified in documentation, route declaration, or source file. +- **E1 - Code-path traced**: Frontend/API entry point connected through services and data handlers. +- **E2 - Test-backed**: At least one directly relevant automated test assertion supports the feature claim. A test file's existence, an unrelated assertion, or an invalid test locator does not establish E2. +- **E3 - Runtime-validated**: Maintainer reproduced behavior in a recorded local environment. +- **E4 - Maintainer-accepted**: Maintainers accepted the feature description and support status. + +## Audit Workflow + +```mermaid +flowchart TD + P0["Phase 0: Snapshot Isolation
(Commit d8a2059)"] --> P1["Phase 1: Codebase Discovery
(Routes, Services, Static JS, Specs)"] + P1 --> P2["Phase 2: Feature Reachability & Verification
(Route matching, FE entrypoints, tests)"] + P2 --> P3["Phase 3: Catalog & Evidence Compilation
(File paths, symbols, exact line ranges)"] + P3 --> P4["Phase 4: Quality & Integrity Audit
(100% path existence check, schema validation)"] +``` + +### Phase 0: Snapshot Isolation +The audit is pinned to git commit `d8a2059df8e53bc7275c45339849d14c8651e73c`. All file paths, symbol declarations, and line ranges map strictly to this commit. + +### Phase 1: Codebase Discovery +All top-level and nested directories were traversed, including: +- Backend Entry Points (`app.py`, `routes/`, `routes/*/*.py`, `companion/`) +- Core Framework (`core/database.py`, `core/session_manager.py`, `core/auth.py`) +- Business Logic Services (`src/`, `services/`, `mcp_servers/`) +- Frontend Assets (`static/app.js`, `static/js/`, `static/index.html`) +- Test Suites (`tests/`, `tests/cli/`, `tests/streaming/`) +- Operations & Docker (`Dockerfile`, `docker-compose*.yml`, `scripts/`) + +### Phase 2: Verification Protocol +For each feature candidate, the following table was evaluated: +- **User Reachability**: Frontend UI element, modal, route, or CLI script. +- **API Entrypoint**: FastAPI `@router` declaration or WebSocket/SSE handler. +- **Backend Execution**: Concrete Python module method, service, or tool call. +- **Data Persistence**: Disk file, SQLite table, or vector collection. +- **Test Coverage**: Automated test file executing assertions against the component. + +### Phase 3: Evidence Linking Standard +Every feature entry in `feature-catalog.json` contains a structured `evidence` list with: +- `path`: Relative path from repository root. +- `symbol`: Route, class, function, or element symbol name. +- `line_range`: Inclusive line range (e.g. `L120-L250`). +- `explanation`: Short factual statement proving reachability or implementation. + +### Phase 4: Quality Check & Schema Constraints +Before finalization: +1. Every evidence file path is validated against the checkout. +2. Every Markdown entry matches `feature-catalog.json`. +3. Recommendation language is separated from empirical factual observations. diff --git a/docs/discovery/domains/agent.md b/docs/discovery/domains/agent.md new file mode 100644 index 000000000..82574ddc3 --- /dev/null +++ b/docs/discovery/domains/agent.md @@ -0,0 +1,153 @@ +# Agent + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## AGENT-001 — Autonomous Agent Loop & Tool Execution Engine + +- **Domain**: `agent` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Executes multi-step agent reasoning loops, tool invocation parsing, and automated response generation. + +### Evidence summary + +- `src/agent_loop.py` — `run_agent_loop` — Core loop evaluating model tool requests and executing handlers. +- `src/tool_execution.py` — `execute_tool_call` — Dispatches tool invocation requests to underlying tool handlers. + +### Unknowns + +- Infinite tool loop if termination condition fails. + +## AGENT-002 — Scheduled Tasks & Event Bus Dispatcher + +- **Domain**: `agent` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Schedules background recurring or delayed tasks, emits event bus triggers, and executes automated flows. + +### Evidence summary + +- `routes/task_routes.py` — `@router.get('')` — Fetches active scheduled tasks. +- `src/task_scheduler.py` — `TaskScheduler` — Async task scheduler dispatching cron and delay triggers. + +### Unknowns + +- Task execution failure handling on system restart. + +## AGENT-003 — Webhook Event Subscriptions & Trigger Processing + +- **Domain**: `agent` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Manages incoming/outgoing webhook subscriptions, endpoint authentication tokens, and event triggers. + +### Evidence summary + +- `routes/webhook_routes.py` — `@router.get('/webhooks')` — Returns list of registered webhooks. +- `src/webhook_manager.py` — `WebhookManager` — Handles payload delivery and signature verification. + +### Unknowns + +- SSRF risks when contacting external webhook URLs if unvalidated. + +## AGENT-004 — Assistant Settings, Task Check-Ins & Background Job Monitor + +- **Domain**: `agent` +- **Status**: `partial` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Manages per-user assistant sessions and scheduled check-in settings, drains background job completions, and retains a legacy no-op activity logging shim. + +### Evidence summary + +- `routes/assistant_routes.py` — `setup_assistant_routes` — Active assistant session, settings, manual check-in, run-status and timezone-list endpoints, including the owner-scoping guards. +- `src/bg_monitor.py` — `_drain_agent` — Runs the agent loop headless against a session to produce the background-job follow-up turn. +- `src/bg_monitor.py` — `_run_followup` — Drains completed background jobs and auto-continues the owning session, deferring while a live turn is in progress. +- `src/assistant_log.py` — `log_to_assistant` — Legacy no-op activity logging shim retained for existing callers; documented as inactive rather than as current behaviour. + +### Unknowns + +- Route `/api/assistant/logs` cited in legacy docs is absent from assistant router. +- Existing unit test `tests/cli/test_logs_cli_resolve_nonstring.py` tests CLI target-name resolution logic, not active assistant routes or bg_monitor execution loop. + +## AGENT-005 — Model Context Protocol (MCP) Server Integration + +- **Domain**: `agent` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Integrates external MCP servers over stdio/SSE to expand agent capabilities dynamically. + +### Evidence summary + +- `routes/mcp_routes.py` — `setup_mcp_routes` — Exposes management endpoints for external MCP servers. +- `src/mcp_manager.py` — `McpManager` — Manages MCP server subprocess lifecycles. + +### Unknowns + +- Subprocess leaks if external MCP server process fails to terminate clean. + +## AGENT-006 — AI Interaction Tools & Pipeline Orchestration + +- **Domain**: `agent` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides specialized AI interaction tools for agent self-debugging, debate, and multi-model collaboration. + +### Evidence summary + +- `src/ai_interaction.py` — `init_ai_interaction_tools` — Registers specialized multi-agent interaction primitives. +- `src/builtin_actions.py` — `execute_builtin_action` — Executes pre-built action intent sequences. + +### Unknowns + +- High API token consumption during extended agent debates. + +## AGENT-007 — Subprocess & Background Job Execution Tools + +- **Domain**: `agent` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides sandboxed bash/shell tool execution capabilities with output streaming and background tracking. + +### Evidence summary + +- `src/agent_tools/subprocess_tools.py` — `run_command` — Executes shell commands in background/foreground. +- `src/bg_jobs.py` — `JobManager` — Tracks async background subprocess tasks. + +### Unknowns + +- Arbitrary shell command execution permissions if sandbox confinement fails. diff --git a/docs/discovery/domains/calendar.md b/docs/discovery/domains/calendar.md new file mode 100644 index 000000000..4a2dad8d3 --- /dev/null +++ b/docs/discovery/domains/calendar.md @@ -0,0 +1,45 @@ +# Calendar + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## CALENDAR-001 — CalDAV Calendar Synchronization & Account Setup + +- **Domain**: `calendar` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires a controlled external CalDAV server. + +### Purpose + +Connects to remote CalDAV servers (Apple iCloud, Nextcloud, Google) to sync calendar event feeds. + +### Evidence summary + +- `routes/calendar_routes.py` — `setup_calendar_routes` — Exposes CalDAV setup and manual sync trigger routes. +- `src/caldav_sync.py` — `CalDavSync` — Fetches and parses remote iCalendar VEVENT objects. + +### Unknowns + +- Invalid SSL certificates on self-hosted CalDAV servers. + +## CALENDAR-002 — Calendar Event Operations & iCalendar Parsing + +- **Domain**: `calendar` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Creates, updates, deletes, and displays calendar events with timezone conversion and reminder notifications. + +### Evidence summary + +- `routes/calendar_routes.py` — `@router.get('/events')` — Fetches calendar events for requested date window. +- `src/tools/calendar.py` — `CalendarTool` — Agent tool for creating and modifying calendar entries. + +### Unknowns + +- Recurring RRULE event expansion calculation bugs across leap years. diff --git a/docs/discovery/domains/chat.md b/docs/discovery/domains/chat.md new file mode 100644 index 000000000..8ff904c6b --- /dev/null +++ b/docs/discovery/domains/chat.md @@ -0,0 +1,197 @@ +# Chat + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## CHAT-001 — Core Chat Streaming & SSE Message Generation + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E2` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires access to a live LLM provider endpoint (OpenAI API key or local Ollama server). + +### Purpose + +Handles real-time Server-Sent Events (SSE) chat streaming, token rendering, and model response generation. + +### Evidence summary + +- `routes/chat_routes.py` — `chat_stream` — POST /api/chat_stream SSE endpoint; builds the shared chat context, then dispatches to the chat-mode or agent-mode streaming path. +- `routes/chat_helpers.py` — `build_chat_context` — Shared context builder invoked by chat_stream; runs message preprocessing and assembles the memory/RAG/web context preface. +- `src/chat_handler.py` — `ChatHandler.preprocess_message` — Message preprocessing (attachments, URLs, tool preprocessing) reached from build_chat_context via routes/chat_helpers.py:preprocess. +- `src/chat_processor.py` — `ChatProcessor.build_context_preface` — Builds the retrieval and web-source context preface injected into the streamed request. +- `src/llm_core.py` — `stream_llm_with_fallback` — Chat-mode streaming dispatcher called from chat_stream; wraps stream_llm with an ordered provider fallback chain. +- `src/llm_core.py` — `stream_llm` — Per-request streaming entry wrapped by stream_llm_with_fallback; acquires the local model slot and delegates to _stream_llm_inner. +- `src/agent_loop.py` — `stream_agent_loop` — Agent-mode streaming path called from chat_stream when the request selects agent mode. +- `tests/test_chat_metrics.py` — `test_stream_llm_passes_through_llamacpp_timings` — Inspected unit test asserting stream_llm forwards backend generation timings into the emitted metrics chunk. +- `tests/test_resend_message_nondestructive.py` — `test_resend_message_does_not_truncate_by_default` — Inspected unit test asserting the frontend resend path does not truncate prior conversation turns. + +### Unknowns + +- Stream interruption on connection drops requires retry logic. + +## CHAT-002 — Session Management & Conversation State + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Manages session creation, listing, switching, renaming, and persistence of conversation metadata. + +### Evidence summary + +- `routes/session_routes.py` — `@router.get('/api/sessions')` — Lists active sessions filtered by user owner scope. +- `core/session_manager.py` — `SessionManager` — Provides thread-safe session storage operations. + +### Unknowns + +- Concurrent file writes to sessions.json under high load. + +## CHAT-003 — Chat History & Message Editing/Truncation + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides history retrieval, message content updating, message deletion, and history branch truncation. + +### Evidence summary + +- `routes/history/history_routes.py` — `@router.get('/api/history/{session_id}')` — Fetches message history timeline for a session. +- `routes/history_routes.py` — `_sys.modules[__name__] = _canonical` — Backward-compatibility shim module. + +### Unknowns + +- Truncating messages re-indexes context window and clears cached tool calls. + +## CHAT-004 — File & Multimodal Attachment Handling + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Handles uploading, mime validation, image preview, vision encoding, and file attachments in chat messages. + +### Evidence summary + +- `routes/upload_routes.py` — `@router.post('')` — Accepts multi-part file uploads and generates vision metadata. +- `src/upload_handler.py` — `UploadHandler.save_file` — Validates upload size and atomicity on disk. + +### Unknowns + +- Large file uploads may consume server disk space if cleanup task fails. + +## CHAT-005 — Chat Message Search + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Enables full-text keyword search across stored chat messages and sessions. + +### Evidence summary + +- `routes/search_routes.py` — `setup_search_routes` — Registers chat message search endpoint. +- `src/session_search.py` — `search_sessions` — Executes query matching against session transcripts. + +### Unknowns + +- Full table scans on un-indexed text columns for very large databases. + +## CHAT-006 — System Prompts & Preset Management + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides creation, selection, and customization of system prompt presets for chat sessions. + +### Evidence summary + +- `routes/preset_routes.py` — `setup_preset_routes` — API routes for listing and modifying system prompt presets. +- `src/preset_manager.py` — `PresetManager` — Disk-backed manager for prompt presets. + +### Unknowns + +- Invalid JSON syntax in user presets file can corrupt preset loading. + +## CHAT-007 — Emoji Rendering & Twemoji SVG Proxy + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Proxies Twemoji SVG icons locally to render flat SVG emojis in message text without external CDN dependencies. + +### Evidence summary + +- `routes/emoji_routes.py` — `setup_emoji_routes` — Serves locally cached Twemoji SVGs. + +### Unknowns + +- First request fetches SVG from remote CDN before caching locally. + +## CHAT-008 — Input History Recall (Arrow Up) + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Allows users to cycle through previously sent prompt messages in the chat composer input using Arrow-Up/Down keys. + +### Evidence summary + +- `static/js/composerArrowUpRecall.js` — `initComposerRecall` — Listens for ArrowUp keypress on composer textarea. + +### Unknowns + +- Client-side browser storage limits. + +## CHAT-009 — Context Window Compaction & Truncation + +- **Domain**: `chat` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Compacts session transcript history when prompt size exceeds context limits using summarization. + +### Evidence summary + +- `routes/history/history_routes.py` — `@router.post('/api/session/{session_id}/compact')` — Triggers context summarization and compaction. +- `src/context_compactor.py` — `compact_context` — Executes context token pruning and summary generation. + +### Unknowns + +- Aggressive compaction may discard subtle user instructions. diff --git a/docs/discovery/domains/contact.md b/docs/discovery/domains/contact.md new file mode 100644 index 000000000..47122b20a --- /dev/null +++ b/docs/discovery/domains/contact.md @@ -0,0 +1,24 @@ +# Contact + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## CONTACT-001 — CardDAV Contact Management & Address Book Integration + +- **Domain**: `contact` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Connects to CardDAV servers, imports VCard contacts, and provides contact lookup for email/calendar autocomplete. + +### Evidence summary + +- `routes/contacts/contacts_routes.py` — `@router.get('/list')` — Returns contact list filtered by search query. +- `src/tools/contacts.py` — `ContactsTool` — Agent tool for querying user address book contacts. + +### Unknowns + +- VCard 3.0 vs 4.0 property parsing mismatches. diff --git a/docs/discovery/domains/cookbook.md b/docs/discovery/domains/cookbook.md new file mode 100644 index 000000000..6d22bccef --- /dev/null +++ b/docs/discovery/domains/cookbook.md @@ -0,0 +1,85 @@ +# Cookbook + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## COOKBOOK-001 — Local Model Download & Recipe Lifecycle Management + +- **Domain**: `cookbook` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Downloads HuggingFace models, configures execution parameters, and manages local GGUF/MLX model servers. + +### Evidence summary + +- `routes/cookbook_routes.py` — `setup_cookbook_routes` — Exposes model downloading and process serving endpoints. +- `static/js/cookbook.js` — `initCookbook` — UI manager for local model library. + +### Unknowns + +- Disk space exhaustion during multi-gigabyte GGUF weights downloads. + +## COOKBOOK-002 — Hardware Model Fitting ('What Fits?') Analysis Engine + +- **Domain**: `cookbook` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Calculates RAM/VRAM requirements, quantized size, and context overhead to determine model compatibility. + +### Evidence summary + +- `routes/hwfit_routes.py` — `setup_hwfit_routes` — Calculates hardware model compatibility. +- `services/hwfit/fit.py` — `calculate_fit` — Performs parameter and memory fit calculations. + +### Unknowns + +- Inaccurate VRAM estimation for non-standard KV-cache quantization. + +## COOKBOOK-003 — HuggingFace & MLX Model Discovery Services + +- **Domain**: `cookbook` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Searches HuggingFace Hub and local MLX model repositories for compatible GGUF and MLX weights. + +### Evidence summary + +- `services/hwfit/hf_discovery.py` — `search_hf_models` — Queries HuggingFace API for model tags and files. + +### Unknowns + +- HuggingFace API rate limits when searching without an API token. + +## COOKBOOK-004 — Host Docker Access for Inference Container Runtimes + +- **Domain**: `cookbook` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires Docker access and supported physical GPU hardware. + +### Purpose + +Detects and connects to host Docker engine to launch containerized Ollama, vLLM, or SGLang runtimes. + +### Evidence summary + +- `src/host_docker_access.py` — `HostDockerAccess` — Interacts with host docker daemon. + +### Unknowns + +- Permission denied accessing docker socket on non-root setups. diff --git a/docs/discovery/domains/document.md b/docs/discovery/domains/document.md new file mode 100644 index 000000000..ca8ece487 --- /dev/null +++ b/docs/discovery/domains/document.md @@ -0,0 +1,108 @@ +# Document + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## DOCUMENT-001 — Document & Canvas Artifact Management + +- **Domain**: `document` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Renders dynamic canvas documents, handles live editing, markdown preview, and side-by-side artifact display. + +### Evidence summary + +- `routes/document_routes.py` — `setup_document_routes` — Registers document artifact CRUD routes. +- `static/js/document.js` — `initDocumentView` — Renders interactive canvas document panel. + +### Unknowns + +- Concurrent edits on the same document artifact. + +## DOCUMENT-002 — PDF Form Processing & High-Fidelity Rendering + +- **Domain**: `document` +- **Status**: `verified` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires optional PyMuPDF (`fitz`) or pypdf runtime dependency. + +### Purpose + +Extracts form fields from PDF files, fills dynamic values, and generates PDF previews. + +### Evidence summary + +- `src/pdf_runtime.py` — `load_pymupdf_for_pdf_viewer` — Loads optional PyMuPDF runtime for PDF viewing. +- `src/pdf_forms.py` — `extract_form_fields` — Handles PDF form field extraction and filling. +- `tests/test_document_pdf_marker.py` — `test_marker_removed_without_eating_following_text` — Tests PDF text extraction wrapper stripping without content corruption. + +### Unknowns + +- Complex XFA PDF forms may not extract cleanly with standard pdf parsers. + +## DOCUMENT-003 — Personal Document Indexing & RAG Retrieval + +- **Domain**: `document` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Indexes local user documents (PDF, DOCX, TXT) into ChromaDB for semantic vector retrieval. + +### Evidence summary + +- `routes/personal_routes.py` — `setup_personal_routes` — Personal document RAG indexing and search API endpoints. +- `src/personal_docs.py` — `PersonalDocsManager` — Handles file text chunking and vector storage. + +### Unknowns + +- Slow vector embedding indexing step for massive multi-thousand page documents. + +## DOCUMENT-004 — Document Conversion & Text Extraction Engine + +- **Domain**: `document` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Converts office formats (.docx, .xlsx, .pptx) and HTML into clean Markdown text representations. + +### Evidence summary + +- `src/markitdown_runtime.py` — `convert_to_markdown` — Converts binary office documents into structured Markdown text. + +### Unknowns + +- Formatting loss when parsing legacy binary doc/xls files. + +## DOCUMENT-005 — Document Library UI Navigation + +- **Domain**: `document` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides dedicated UI view for browsing, filtering, and organizing saved user documents. + +### Evidence summary + +- `static/js/documentLibrary.js` — `initDocumentLibrary` — Renders document library navigation grid. +- `app.py` — `serve_library` — Serves SPA shell for /library route. + +### Unknowns + +- Large folder trees may cause initial DOM render slowdown. diff --git a/docs/discovery/domains/email.md b/docs/discovery/domains/email.md new file mode 100644 index 000000000..12b28ca43 --- /dev/null +++ b/docs/discovery/domains/email.md @@ -0,0 +1,87 @@ +# Email + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## EMAIL-001 — Email Account Setup, IMAP/SMTP Connection & Polling + +- **Domain**: `email` +- **Status**: `verified` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires a controlled live IMAP account and network access. + +### Purpose + +Configures IMAP/SMTP email accounts, validates TLS certificates, and polls background inbox updates. + +### Evidence summary + +- `routes/email_routes.py` — `setup_email_routes` — Sets up email account management and synchronization routes. +- `routes/email_pollers.py` — `_start_poller` — Background poller for email inbox synchronization. +- `tests/test_service_health_email.py` — `test_email_ok_all_connect` — Tests IMAP connection health probing and status reporting. + +### Unknowns + +- Account lockouts if bad credentials are repeatedly polled. + +## EMAIL-002 — Email Searching, Threading & Message Operations + +- **Domain**: `email` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Parses email headers, folds signatures, groups messages into threads, and executes full-text email search. + +### Evidence summary + +- `routes/email_routes.py` — `@router.get('/search')` — Executes search across cached email headers and text. +- `src/email_thread_parser.py` — `parse_email_thread` — Builds conversation tree from Message-ID and In-Reply-To headers. + +### Unknowns + +- Malformed MIME email structures failing HTML sanitization. + +## EMAIL-003 — Email Composition, Draft Management & Sending + +- **Domain**: `email` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires a controlled live SMTP account and network access. + +### Purpose + +Creates, saves, and dispatches HTML/plaintext email messages via SMTP. + +### Evidence summary + +- `routes/email_routes.py` — `@router.post('/send')` — Sends email message via user SMTP credentials. + +### Unknowns + +- SMTP connection drop mid-send causing unsent mail state. + +## EMAIL-004 — Email MCP Server & Codex Integration Bridge + +- **Domain**: `email` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Exposes constrained email reading and draft capabilities to external Codex / MCP agents with scope checks. + +### Evidence summary + +- `mcp_servers/email_server.py` — `EmailMcpServer` — MCP server exposing email tools over stdio/SSE. +- `routes/codex_routes.py` — `setup_codex_routes` — Bridge endpoints for external Codex plugin integration. + +### Unknowns + +- Unauthorized mail sending if token scopes are improperly scoped. diff --git a/docs/discovery/domains/frontend.md b/docs/discovery/domains/frontend.md new file mode 100644 index 000000000..6f9891783 --- /dev/null +++ b/docs/discovery/domains/frontend.md @@ -0,0 +1,149 @@ +# Frontend + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## FRONTEND-001 — Single Page Application Shell & Client Router + +- **Domain**: `frontend` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Main HTML5 SPA shell, DOM lifecycle initializers, tab navigation, and deep-link route handlers. + +### Evidence summary + +- `static/index.html` — `index.html` — Main SPA entry point containing modal roots and CSS bundles. +- `app.py` — `serve_index` — Serves index.html with dynamically generated CSP nonces. + +### Unknowns + +- Stale browser static cache if asset hashing is omitted during deployment. + +## FRONTEND-002 — Dynamic Theme, Color System & Custom Fonts + +- **Domain**: `frontend` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Supports dark/light themes, custom CSS variables, color picker controls, and user font uploads. + +### Evidence summary + +- `static/js/theme.js` — `applyTheme` — Applies custom HSL theme variables to DOM document root. +- `routes/font_routes.py` — `setup_font_routes` — Allows uploading and serving custom WOFF2 font files. + +### Unknowns + +- Flash of unstyled content (FOUC) on slow connections. + +## FRONTEND-003 — Window Manager, Tile Layout & Modal Control System + +- **Domain**: `frontend` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Manages draggable tool windows, snapped multi-tile viewports, modal dialog Z-ordering, and ESC key stacks. + +### Evidence summary + +- `static/js/modalManager.js` — `ModalManager` — Controls modal open/close transitions and focus trapping. +- `static/js/tileManager.js` — `TileManager` — Handles viewport split-pane grid arrangements. + +### Unknowns + +- Overlap artifacts when opening many simultaneous tool floating windows. + +## FRONTEND-004 — Global Keyboard Shortcuts & Accessibility Controls + +- **Domain**: `frontend` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides configurable hotkeys (Ctrl+K search, Esc close, Alt+1-9 tabs) and high-contrast accessibility options. + +### Evidence summary + +- `static/js/keyboard-shortcuts.js` — `initShortcuts` — Binds global keydown handlers for system shortcuts. +- `static/js/a11y.js` — `initA11y` — Applies ARIA roles and dyslexic font toggles. + +### Unknowns + +- Browser keybinding collisions with browser default hotkeys. + +## FRONTEND-005 — Markdown, LaTeX & Code Block Streaming Renderer + +- **Domain**: `frontend` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Parses incoming SSE markdown streams, renders KaTeX math formulas, syntax-highlighted code, and interactive runners. + +### Evidence summary + +- `static/js/markdown.js` — `renderMarkdown` — Converts markdown prose to HTML nodes with syntax highlighting. +- `static/js/streamingSegmenter.js` — `Segmenter` — Parses un-closed markdown fences during live stream. + +### Unknowns + +- DOM thrashing if streaming segmenter updates UI too frequently. + +## FRONTEND-006 — Interactive Tour & Guided Onboarding System + +- **Domain**: `frontend` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Presents interactive step-by-step feature tours and UI tooltip hints for new users. + +### Evidence summary + +- `static/js/tourHints.js` — `startTour` — Renders guided feature tour overlays over target UI elements. + +### Unknowns + +- Tour step misalignment if window is resized mid-tour. + +## FRONTEND-007 — Background Effects Prototyping Sandbox + +- **Domain**: `frontend` +- **Status**: `dead-code-candidate` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Standalone sandbox page for prototyping visual background animations, waves, and whirlpool effects. + +### Evidence summary + +- `app.py` — `serve_backgrounds` — Serves visual background sandbox HTML page route. +- `static/wave-variants.html` — `wave-variants.html` — Interactive background effect prototyping sandbox variant. + +### Unknowns + +- Route `/backgrounds` in app.py L918 attempts to serve `static/backgrounds.html` which is missing from disk; variant templates `wave-variants.html` and `whirlpool-variants.html` exist. diff --git a/docs/discovery/domains/media.md b/docs/discovery/domains/media.md new file mode 100644 index 000000000..cc0622ceb --- /dev/null +++ b/docs/discovery/domains/media.md @@ -0,0 +1,170 @@ +# Media + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## MEDIA-001 — Gallery Image Library & Album Operations + +- **Domain**: `media` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Organizes images into custom albums, provides grid browsing, tagging, and album metadata management. + +### Evidence summary + +- `routes/gallery/gallery_routes.py` — `@router.get('/api/gallery/library')` — Fetches image library list with tag filters. +- `static/js/gallery.js` — `initGallery` — Main gallery grid renderer and uploader. + +### Unknowns + +- Thumbnail generation overhead for high-resolution RAW camera images. + +## MEDIA-002 — Image Processing, AI Upscaling & Style Transfer + +- **Domain**: `media` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Executes local image enhancement, background removal, face sharpening, and AI upscaling operations. + +### Evidence summary + +- `routes/gallery/gallery_routes.py` — `@router.post('/api/gallery/ai-upscale')` — Runs RealESRGAN image upscaling. +- `routes/gallery/gallery_routes.py` — `@router.post('/api/image/remove-bg')` — Executes background removal pass. + +### Unknowns + +- High GPU memory allocation when upscaling 4K images. + +## MEDIA-003 — Interactive Image Canvas Editor & Persisted Drafts + +- **Domain**: `media` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides full multi-layer raster canvas editor, brush tools, transforms, masks, and draft project persistence. + +### Evidence summary + +- `routes/editor_draft_routes.py` — `setup_editor_draft_routes` — API routes for saving and loading canvas project drafts. +- `static/js/editor/history-panel.js` — `HistoryManager` — Canvas undo/redo stack manager. + +### Unknowns + +- Browser memory leak if multi-gigabyte layer undo buffers are kept indefinitely. + +## MEDIA-004 — Text-to-Speech (TTS) Synthesis Service + +- **Domain**: `media` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Synthesizes spoken audio from text using local Kokoro, EdgeTTS, or OpenAI TTS engines. + +### Evidence summary + +- `routes/tts_routes.py` — `@router.post('/synthesize')` — Synthesizes TTS audio clip. +- `services/tts/tts_service.py` — `TTSService` — Provider abstraction layer for audio speech generation. + +### Unknowns + +- Audio synthesis latency on CPU-only hardware setups. + +## MEDIA-005 — Speech-to-Text (STT) Audio Transcription Service + +- **Domain**: `media` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Transcribes user audio recordings into text using faster-whisper or local speech models. + +### Evidence summary + +- `routes/stt_routes.py` — `@router.post('/transcribe')` — Accepts multipart audio file and returns transcription text. +- `services/stt/stt_service.py` — `STTService` — Whisper audio transcription engine wrapper. + +### Unknowns + +- Missing ffmpeg system dependency prevents audio format decoding. + +## MEDIA-006 — Digital Signature Stamp Storage & Placement + +- **Domain**: `media` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Stores transparent PNG user signatures and stamps for placement onto PDF forms and documents. + +### Evidence summary + +- `routes/signature_routes.py` — `setup_signature_routes` — CRUD endpoints for managing user signature PNG stamps. + +### Unknowns + +- Cross-site scripting if signature image titles contain unescaped user input. + +## MEDIA-007 — Generated Image Artifact Route & MCP Integration + +- **Domain**: `media` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Serves generated AI artwork artifacts and integrates with image generation MCP server. + +### Evidence summary + +- `app.py` — `serve_generated_image` — Serves generated image artifacts with cache headers. +- `src/generated_images.py` — `resolve_generated_image_path` — Confines requested image path within artifacts directory. + +### Unknowns + +- Path traversal vulnerability if filename parameter is un-sanitized. + +## MEDIA-008 — Native MLX Image Bridge (macOS Apple Silicon) + +- **Domain**: `media` +- **Status**: `experimental` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires Apple Silicon, macOS tooling, and the compiled MLX bridge. + +### Purpose + +Native Apple Swift bridge for hardware-accelerated diffusion and MLX image colorization on macOS. + +### Evidence summary + +- `swift/odysseus-mlx-image-bridge/Package.swift` — `Package` — Swift package manifest for native MLX image bridge. +- `scripts/mlx_image_server.py` — `main` — Python daemon wrapping native Swift MLX binary. + +### Unknowns + +- Binary build requires Xcode command line tools build step (`build-macos-app.sh`). diff --git a/docs/discovery/domains/memory.md b/docs/discovery/domains/memory.md new file mode 100644 index 000000000..22b9f8b78 --- /dev/null +++ b/docs/discovery/domains/memory.md @@ -0,0 +1,24 @@ +# Memory + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## MEMORY-001 — Persistent Long-Term Memory & Vector Indexing + +- **Domain**: `memory` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Extracts facts, user preferences, and temporal memories from chat sessions into vector/relational storage. + +### Evidence summary + +- `routes/memory/memory_routes.py` — `@router.get('')` — Fetches long-term user memory timeline. +- `services/memory/memory_extractor.py` — `MemoryExtractor` — LLM-driven fact extraction from conversation transcripts. + +### Unknowns + +- Conflicting memory facts extracted from contradictory user prompts. diff --git a/docs/discovery/domains/model.md b/docs/discovery/domains/model.md new file mode 100644 index 000000000..1855c0d3f --- /dev/null +++ b/docs/discovery/domains/model.md @@ -0,0 +1,177 @@ +# Model + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## MODEL-001 — Multi-Provider LLM Model Discovery & Metadata Management + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Discovers models from OpenAI, Anthropic, Ollama, vLLM, LMStudio, OpenRouter, and Google AI Studio endpoints. + +### Evidence summary + +- `routes/model_routes.py` — `@router.get('/api/models')` — Returns unified list of available models across providers. +- `src/model_discovery.py` — `ModelDiscovery.discover_all` — Queries connected provider endpoints for available model IDs. + +### Unknowns + +- Remote endpoint timeouts may slow down full discovery refresh. + +## MODEL-002 — Model Capability & Context Limits Detection + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Detects vision, tool calling, reasoning, and context window limits for connected model endpoints. + +### Evidence summary + +- `src/model_capabilities.py` — `get_model_capabilities` — Maps model names to vision and tool support flags. +- `src/endpoint_resolver.py` — `resolve_endpoint_headers` — Resolves auth headers and target URLs for model endpoints. + +### Unknowns + +- Incorrect context limit metadata for unlisted custom fine-tunes. + +## MODEL-003 — LLM Core Provider Communication & Fallback Routing + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Provider dispatch, header injection and fallback advancement are only observable against a reachable LLM provider endpoint; not exercised in this documentation pass. + +### Purpose + +Manages HTTP request dispatching, authorization header injection, and fallback provider routing for LLM calls. + +### Evidence summary + +- `src/llm_core.py` — `llm_call_async` — Non-streaming provider request dispatcher: resolves the endpoint, injects authorization headers and executes the HTTP call. +- `src/llm_core.py` — `llm_call_async_with_fallback` — Ordered fallback wrapper that retries llm_call_async across the configured candidate endpoints. +- `src/llm_core.py` — `stream_llm_with_fallback` — Ordered fallback wrapper for the streaming path; advances to the next candidate when a provider yields an empty completion. + +### Unknowns + +- Unexpected API changes in upstream third-party model providers. + +## MODEL-004 — Model Selection & Display Ordering Preferences + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Allows pinning, sorting, and hiding specific models in the UI selection dropdown. + +### Evidence summary + +- `routes/model_routes.py` — `@router.post('/order')` — Saves custom model display order preference. + +### Unknowns + +- Stale model IDs in custom order lists after model endpoints are removed. + +## MODEL-005 — Side-by-Side Model Comparison (A/B Testing) + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Enables dual-model side-by-side response evaluation, arena scoring, and latency comparison. + +### Evidence summary + +- `routes/compare/compare_routes.py` — `@router.post('/start')` — Starts a parallel dual-model comparison stream. +- `static/js/compare/index.js` — `initCompareView` — Renders side-by-side model chat panes. + +### Unknowns + +- High memory and network usage when streaming two model responses simultaneously. + +## MODEL-006 — GitHub Copilot Device Flow Authentication + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E2` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires an interactive GitHub Copilot OAuth device-flow account. + +### Purpose + +Authenticates with GitHub Copilot via OAuth device flow to use Copilot models directly. + +### Evidence summary + +- `routes/copilot_routes.py` — `setup_copilot_routes` — Builds the Copilot device-flow router at prefix /api/copilot, wiring _start_device_flow and _poll_device_flow. +- `routes/device_flow.py` — `create_device_flow_router` — Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix. +- `src/copilot.py` — `request_device_code` — Issues the GitHub device-code request that begins the Copilot OAuth device flow. +- `src/copilot.py` — `poll_access_token` — Polls GitHub for the access token once the user has authorized the device code. +- `tests/test_provider_device_flow_js.py` — `test_copilot_success_uses_complete_verification_uri` — Inspected unit test asserting the Copilot device-flow runner surfaces the complete verification URI returned by the backend. + +### Unknowns + +- Token expiration requires manual device re-authentication. + +## MODEL-007 — ChatGPT Subscription Device Flow Authentication + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E2` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires an interactive ChatGPT subscription OAuth flow. + +### Purpose + +Authenticates with ChatGPT Pro/Plus subscription tokens via device login flow. + +### Evidence summary + +- `routes/chatgpt_subscription_routes.py` — `setup_chatgpt_subscription_routes` — Builds the ChatGPT subscription device-flow router at prefix /api/chatgpt-subscription. +- `routes/device_flow.py` — `create_device_flow_router` — Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix. +- `src/chatgpt_subscription.py` — `request_device_code` — Issues the ChatGPT device-authorization request that begins the subscription OAuth device flow. +- `src/chatgpt_subscription.py` — `poll_device_auth` — Polls the ChatGPT device-authorization endpoint for completion using the stored device_auth_id and user_code. +- `tests/test_provider_device_flow_js.py` — `test_chatgpt_success_uses_plain_verification_uri` — Inspected unit test asserting the ChatGPT device-flow runner uses the plain verification URI rather than the Copilot complete-URI form. + +### Unknowns + +- Changes in OpenAI auth endpoint security challenges. + +## MODEL-008 — Embedding Model Lane & Vector Provider Setup + +- **Domain**: `model` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Configures local sentence-transformers, FastEmbed, or remote OpenAI embedding model lanes. + +### Evidence summary + +- `routes/embedding_routes.py` — `setup_embedding_routes` — Provides embedding provider configuration endpoints. +- `src/embeddings.py` — `EmbeddingManager` — Generates dense vector embeddings for RAG and memory. + +### Unknowns + +- First-time download of heavy PyTorch model weights on CPU-only machines. diff --git a/docs/discovery/domains/note.md b/docs/discovery/domains/note.md new file mode 100644 index 000000000..e0c197428 --- /dev/null +++ b/docs/discovery/domains/note.md @@ -0,0 +1,24 @@ +# Note + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## NOTE-001 — Interactive Notes & Checklist Management + +- **Domain**: `note` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides Google Keep-style notes, rich markdown text, checklist items, pinning, color tags, and reminders. + +### Evidence summary + +- `routes/note/note_routes.py` — `@router.get('')` — Lists all user notes with pin and archive states. +- `static/js/notes.js` — `initNotesView` — Main interactive notes grid and modal manager. + +### Unknowns + +- Concurrent edits on note item checkboxes. diff --git a/docs/discovery/domains/platform.md b/docs/discovery/domains/platform.md new file mode 100644 index 000000000..037edddde --- /dev/null +++ b/docs/discovery/domains/platform.md @@ -0,0 +1,213 @@ +# Platform + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## PLATFORM-001 — Application Initialization & Lifespan Management + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Orchestrates server startup, database table migration, background daemon initialization, and clean shutdown. + +### Evidence summary + +- `app.py` — `_lifespan` — FastAPI lifespan context manager executing startup tasks. +- `src/app_initializer.py` — `initialize_app` — Initializes app directories, DB schemas, and logging. + +### Unknowns + +- Un-handled exceptions during startup halt application launch. + +## PLATFORM-002 — System Health, Readiness & Version Monitoring APIs + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Exposes Liveness (/api/health), Readiness (/api/ready), App Version (/api/version), and Client Perf APIs. + +### Evidence summary + +- `app.py` — `readiness_check` — Performs system component integrity check. +- `src/readiness.py` — `check_readiness` — Checks database, storage, and key paths for read/write access. + +### Unknowns + +- Readiness check delays if verifying connectivity to offline remote endpoints. + +## PLATFORM-003 — Database Schema, Migrations & SQLite Persistence + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Defines core relational tables (users, tokens, tasks, sessions) and executes automated SQLite schema upgrades. + +### Evidence summary + +- `core/database.py` — `init_db` — Creates ORM tables and establishes connection pool. +- `scripts/update_database.py` — `run_migrations` — Applies missing schema columns and indices. + +### Unknowns + +- SQLite file lock contention under high concurrent write loads. + +## PLATFORM-004 — User Data Export & Import Backup Infrastructure + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Exports complete user workspace state (sessions, memory, skills, notes, presets) into a zip archive. + +### Evidence summary + +- `routes/backup_routes.py` — `setup_backup_routes` — Handles workspace data export and import upload unpack. +- `docs/backup-restore.md` — `Documentation` — Backup and restore operational documentation. + +### Unknowns + +- Corrupt archive files causing partial data restore. + +## PLATFORM-005 — File Cleanup & Storage Maintenance Engine + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Scans data directories for orphaned files, old uploads, temporary vision images, and frees disk space. + +### Evidence summary + +- `routes/cleanup/cleanup_routes.py` — `@router.get('/preview')` — Previews reclaimable disk space across storage directories. +- `src/cleanup_service.py` — `CleanupService` — Executes filesystem purge of orphaned asset files. + +### Unknowns + +- Deletes files uploaded in active sessions if retention window is set too short. + +## PLATFORM-006 — System Health & RAG Diagnostic Suite + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Executes real-time integrity diagnostics across ChromaDB, SearXNG, local models, and network interfaces. + +### Evidence summary + +- `routes/diagnostics_routes.py` — `setup_diagnostics_routes` — Runs subsystem health check suite. +- `src/service_health.py` — `collect_health_status` — Inspects vector database, email, search, and local provider status. + +### Unknowns + +- Diagnostic timeout if external search provider is unreachable. + +## PLATFORM-007 — Desktop CLI Utilities & Shell Integration Tools + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides command-line interface tools (`odysseus`, `odysseus-mcp`, `odysseus-mail`) for terminal usage. + +### Evidence summary + +- `scripts/_lib/cli.py` — `main` — Shared CLI framework for terminal helper commands. +- `scripts/odysseus` — `odysseus` — Main terminal launcher script. + +### Unknowns + +- Outdated CLI scripts if backend API schemas change. + +## PLATFORM-008 — Desktop Companion App Integration + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides API routes and pairing mechanisms for the native macOS/desktop menu bar companion app. + +### Evidence summary + +- `companion/routes.py` — `setup_companion_routes` — Endpoints for pairing and status sync with desktop companion. +- `companion/pairing.py` — `PairingManager` — Generates and validates companion pairing codes. + +### Unknowns + +- Pairing code expiration timing window. + +## PLATFORM-009 — Docker Containerization & GPU Hardware Manifests + +- **Domain**: `platform` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires Docker GPU pass-through and compatible host drivers. + +### Purpose + +Provides multi-stage Dockerfile and Docker Compose manifests for CPU, NVIDIA CUDA, and AMD ROCm GPUs. + +### Evidence summary + +- `Dockerfile` — `multi-stage-build` — Multi-stage container build environment. +- `docker-compose.gpu-nvidia.yml` — `nvidia-gpu-manifest` — NVIDIA GPU pass-through container specification. +- `scripts/check-docker-gpu.sh` — `check-docker-gpu` — Automated diagnostic test script for host NVIDIA GPU passthrough. + +### Unknowns + +- Driver version incompatibility with host NVIDIA/AMD kernel drivers. + +## PLATFORM-010 — Legacy FAISS Vector Index Migration Script + +- **Domain**: `platform` +- **Status**: `legacy` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Legacy utility script to migrate older FAISS vector indices into ChromaDB. + +### Evidence summary + +- `scripts/migrate_faiss_to_chroma.py` — `migrate_faiss` — Reads FAISS vector index files and writes to ChromaDB collection. + +### Unknowns + +- Superseded by native ChromaDB vector index pipeline. diff --git a/docs/discovery/domains/research.md b/docs/discovery/domains/research.md new file mode 100644 index 000000000..2df5a4b75 --- /dev/null +++ b/docs/discovery/domains/research.md @@ -0,0 +1,86 @@ +# Research + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## RESEARCH-001 — Deep Research Execution Engine & SSE Progress Streaming + +- **Domain**: `research` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Executes multi-step recursive deep research tasks, web page scraping, synthesis, and streams live progress. + +### Evidence summary + +- `routes/research/research_routes.py` — `@router.post('/api/research/start')` — Initiates deep research job. +- `src/deep_research.py` — `DeepResearchEngine` — Recursive search and summary crawler. + +### Unknowns + +- High memory consumption when parsing multi-megabyte HTML target pages. + +## RESEARCH-002 — Research Library, Detail View & Image Controls + +- **Domain**: `research` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Stores completed research reports, generated diagrams, reference links, and manages image visibility. + +### Evidence summary + +- `routes/research/research_routes.py` — `@router.get('/api/research/library')` — Returns all saved research reports. + +### Unknowns + +- Orphaned report files if storage directory is modified out-of-band. + +## RESEARCH-003 — Web Search Engine Integration (SearXNG & Multi-Provider) + +- **Domain**: `research` +- **Status**: `verified` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires an active SearXNG instance or external search API provider. + +### Purpose + +Queries SearXNG, DuckDuckGo, or Google Search instances to retrieve web search snippets. + +### Evidence summary + +- `routes/search_routes.py` — `setup_search_routes` — Defines /api/search, /api/search/config, and /api/search/query endpoints. +- `src/search/core.py` — `SearchEngine` — Compatibility module aliasing services.search.core. +- `tests/test_search_ranking.py` — `test_news_queries_prefer_news_sources_over_sports_and_social_results` — Tests search result domain ranking and scoring. + +### Unknowns + +- Search provider IP throttling or rate-limiting. + +## RESEARCH-004 — Research Result Peeking & Topic Spinoff Generation + +- **Domain**: `research` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Extracts preliminary research snippets and spawns child research sessions focused on specific sub-topics. + +### Evidence summary + +- `routes/research/research_routes.py` — `@router.post('/api/research/spinoff/{session_id}')` — Spawns child research session for specific query. + +### Unknowns + +- Deep recursion tree depth when spawning multiple nested spinoffs. diff --git a/docs/discovery/domains/security.md b/docs/discovery/domains/security.md new file mode 100644 index 000000000..2d07dae22 --- /dev/null +++ b/docs/discovery/domains/security.md @@ -0,0 +1,154 @@ +# Security + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## SECURITY-001 — Authentication, Session Cookies & User Management + +- **Domain**: `security` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Handles bcrypt password hashing, session cookie issuance, authentication enforcement, and user administration. + +### Evidence summary + +- `routes/auth_routes.py` — `@router.post('/login')` — Authenticates credentials and sets session cookie. +- `core/auth.py` — `AuthManager` — Handles user creation, password verification, and session tokens. + +### Unknowns + +- Cookie session hijack if deployed over unencrypted HTTP without HTTPS cookie flags. + +## SECURITY-002 — System Vault Encrypted Secret Storage + +- **Domain**: `security` +- **Status**: `verified` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: pending — Requires installed Bitwarden CLI (`bw`) executable. + +### Purpose + +Encrypts API keys, passphrases, and third-party secrets on disk using AES-GCM / PBKDF2 key derivation. + +### Evidence summary + +- `routes/vault_routes.py` — `setup_vault_routes` — Admin routes for vault configuration, login, unlock, lock, and logout. +- `src/secret_storage.py` — `SecretStorage` — Fernet symmetric key DB secret encryption. +- `tests/test_vault_password_not_in_argv.py` — `test_bw_password_not_in_argv` — Verifies master password is fed via stdin and never appears in process argv. + +### Unknowns + +- Loss of vault master passphrase renders all encrypted secrets permanently unrecoverable. + +## SECURITY-003 — API Token Management & Scope Access Control + +- **Domain**: `security` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Generates scoped API bearer tokens (read/write/admin) for external tool and script authentication. + +### Evidence summary + +- `routes/api_token_routes.py` — `setup_api_token_routes` — Exposes API token creation, scope assignment, and revocation. +- `core/database.py` — `ApiToken` — SQLAlchemy ORM schema for API tokens and permissions. + +### Unknowns + +- Leaked API bearer tokens with excessive permission scopes. + +## SECURITY-004 — Prompt Security & Injection Defense Engine + +- **Domain**: `security` +- **Status**: `verified` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Scans system prompts and external inputs for prompt injection attempts, jailbreaks, and sensitive data leaks. + +### Evidence summary + +- `src/prompt_security.py` — `untrusted_context_message` — Wraps untrusted context with guard delimiters and sets metadata.trusted = False. +- `src/tool_security.py` — `NON_ADMIN_BLOCKED_TOOLS` — Enforces tool execution safety for non-admin user roles. +- `tests/test_skill_index_prompt_injection.py` — `test_skill_index` — Verifies skill index descriptions cannot leak into trusted system prompts. +- `tests/test_tool_output_prompt_injection.py` — `test_tool_output` — Tool output injection guards. + +### Unknowns + +- False positives blocking legitimate complex coding or security prompts. + +## SECURITY-005 — URL & Path Confinement Security Guards + +- **Domain**: `security` +- **Status**: `verified` +- **Evidence Maturity**: `E1` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Prevents SSRF attacks and path traversal by validating target IP addresses and resolving symlinks. + +### Evidence summary + +- `src/url_safety.py` — `check_outbound_url` — Rejects non-HTTP(S) schemes, link-local, cloud metadata SSRF addresses. +- `src/url_security.py` — `validate_public_http_url` — Validates public-facing endpoints. +- `tests/test_url_safety.py` — `test_url_safety` — Scheme validation, cloud metadata SSRF rejection, IP classification. +- `tests/test_tool_path_confinement.py` — `test_path_confinement` — Path traversal checks. +- `tests/test_workspace_confine.py` — `test_workspace_confine` — Workspace confinement checks. + +### Unknowns + +- DNS rebinding attacks if IP address is re-resolved post-validation. + +## SECURITY-006 — HTTP Security Headers Middleware + +- **Domain**: `security` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Injects standard OWASP HTTP security headers (CSP, HSTS, X-Content-Type-Options, X-Frame-Options). + +### Evidence summary + +- `core/middleware.py` — `SecurityHeadersMiddleware` — Sets strict security headers and CSP nonces on HTTP responses. + +### Unknowns + +- Strict Content Security Policy (CSP) blocking third-party embedded web resources. + +## SECURITY-007 — Admin System Data Wipe ('Danger Zone') + +- **Domain**: `security` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Provides administrative reset operations to wipe sessions, cache, uploaded files, or factory reset state. + +### Evidence summary + +- `routes/admin_wipe/admin_wipe_routes.py` — `@router.delete('/wipe/{kind}')` — Executes systemic data wipe based on requested scope. + +### Unknowns + +- Accidental catastrophic data loss if triggered without user confirmation. diff --git a/docs/discovery/domains/skill.md b/docs/discovery/domains/skill.md new file mode 100644 index 000000000..43fb72180 --- /dev/null +++ b/docs/discovery/domains/skill.md @@ -0,0 +1,24 @@ +# Skill + +Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. + +## SKILL-001 — Dynamic Skill Management & Code Execution Engine + +- **Domain**: `skill` +- **Status**: `verified` +- **Evidence Maturity**: `E0` +- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. + +### Purpose + +Allows users to create, import, edit, test, and execute custom Python/Markdown skills dynamically. + +### Evidence summary + +- `routes/skills_routes.py` — `setup_skills_routes` — Exposes CRUD and remote import routes for user skills. +- `services/memory/skills.py` — `SkillsManager` — Handles skill storage, parsing, and execution. + +### Unknowns + +- Arbitrary code execution risks if skill import URL is untrusted. diff --git a/docs/discovery/feature-catalog.json b/docs/discovery/feature-catalog.json new file mode 100644 index 000000000..93cdc6fd0 --- /dev/null +++ b/docs/discovery/feature-catalog.json @@ -0,0 +1,4797 @@ +[ + { + "id": "CHAT-001", + "domain": "chat", + "name": "Core Chat Streaming & SSE Message Generation", + "purpose": "Handles real-time Server-Sent Events (SSE) chat streaming, token rendering, and model response generation.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chatStream.js", + "static/js/chat.js", + "static/js/streamingRenderer.js" + ], + "backend_entrypoints": [ + "routes/chat_routes.py:chat_stream", + "routes/chat_helpers.py:build_chat_context", + "src/chat_handler.py:ChatHandler.preprocess_message", + "src/chat_processor.py:ChatProcessor.build_context_preface", + "src/llm_core.py:stream_llm_with_fallback", + "src/agent_loop.py:stream_agent_loop" + ], + "routes": [ + "/api/chat_stream", + "/api/chat" + ], + "configuration": [ + "OPENAI_API_KEY", + "OLLAMA_BASE_URL", + "REQUEST_TIMEOUT" + ], + "persistence": [ + "SESSIONS_FILE", + "DATA_DIR/sessions/" + ], + "dependencies": [ + "fastapi", + "starlette.responses.StreamingResponse", + "httpx" + ], + "tests": [ + "tests/test_chat_metrics.py", + "tests/test_resend_message_nondestructive.py" + ], + "documentation": [ + "docs/chat.webm", + "README.md" + ], + "risks": [ + "Stream interruption on connection drops requires retry logic." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/chat_routes.py", + "symbol": "chat_stream", + "line_range": "L702-L1870", + "explanation": "POST /api/chat_stream SSE endpoint; builds the shared chat context, then dispatches to the chat-mode or agent-mode streaming path." + }, + { + "path": "routes/chat_helpers.py", + "symbol": "build_chat_context", + "line_range": "L669-L863", + "explanation": "Shared context builder invoked by chat_stream; runs message preprocessing and assembles the memory/RAG/web context preface." + }, + { + "path": "src/chat_handler.py", + "symbol": "ChatHandler.preprocess_message", + "line_range": "L123-L315", + "explanation": "Message preprocessing (attachments, URLs, tool preprocessing) reached from build_chat_context via routes/chat_helpers.py:preprocess." + }, + { + "path": "src/chat_processor.py", + "symbol": "ChatProcessor.build_context_preface", + "line_range": "L263-L493", + "explanation": "Builds the retrieval and web-source context preface injected into the streamed request." + }, + { + "path": "src/llm_core.py", + "symbol": "stream_llm_with_fallback", + "line_range": "L2794-L2895", + "explanation": "Chat-mode streaming dispatcher called from chat_stream; wraps stream_llm with an ordered provider fallback chain." + }, + { + "path": "src/llm_core.py", + "symbol": "stream_llm", + "line_range": "L2131-L2151", + "explanation": "Per-request streaming entry wrapped by stream_llm_with_fallback; acquires the local model slot and delegates to _stream_llm_inner." + }, + { + "path": "src/agent_loop.py", + "symbol": "stream_agent_loop", + "line_range": "L3079-L5248", + "explanation": "Agent-mode streaming path called from chat_stream when the request selects agent mode." + }, + { + "path": "tests/test_chat_metrics.py", + "symbol": "test_stream_llm_passes_through_llamacpp_timings", + "line_range": "L129-L136", + "explanation": "Inspected unit test asserting stream_llm forwards backend generation timings into the emitted metrics chunk." + }, + { + "path": "tests/test_resend_message_nondestructive.py", + "symbol": "test_resend_message_does_not_truncate_by_default", + "line_range": "L23-L36", + "explanation": "Inspected unit test asserting the frontend resend path does not truncate prior conversation turns." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E2", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires access to a live LLM provider endpoint (OpenAI API key or local Ollama server)." + } + }, + { + "id": "CHAT-002", + "domain": "chat", + "name": "Session Management & Conversation State", + "purpose": "Manages session creation, listing, switching, renaming, and persistence of conversation metadata.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/sessions.js", + "static/js/sidebar-layout.js" + ], + "backend_entrypoints": [ + "routes/session_routes.py:setup_session_routes", + "core/session_manager.py:SessionManager" + ], + "routes": [ + "/api/sessions", + "/api/sessions/{session_id}" + ], + "configuration": [ + "SESSIONS_FILE", + "DATA_DIR/odysseus.db" + ], + "persistence": [ + "DATA_DIR/sessions.json", + "SQLite session table" + ], + "dependencies": [ + "sqlite3", + "pydantic" + ], + "tests": [ + "tests/test_session_manager.py", + "tests/test_session_routes_utcnow.py" + ], + "documentation": [ + "docs/setup.md" + ], + "risks": [ + "Concurrent file writes to sessions.json under high load." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/session_routes.py", + "symbol": "@router.get('/api/sessions')", + "line_range": "L150-L210", + "explanation": "Lists active sessions filtered by user owner scope." + }, + { + "path": "core/session_manager.py", + "symbol": "SessionManager", + "line_range": "L30-L150", + "explanation": "Provides thread-safe session storage operations." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CHAT-003", + "domain": "chat", + "name": "Chat History & Message Editing/Truncation", + "purpose": "Provides history retrieval, message content updating, message deletion, and history branch truncation.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chat.js", + "static/js/sessions.js" + ], + "backend_entrypoints": [ + "routes/history/history_routes.py:setup_history_routes" + ], + "routes": [ + "/api/history/{session_id}", + "/api/session/{session_id}/truncate", + "/api/session/{session_id}/edit-message" + ], + "configuration": [ + "MAX_HISTORY_MESSAGES" + ], + "persistence": [ + "DATA_DIR/sessions/" + ], + "dependencies": [ + "sqlite3", + "fastapi" + ], + "tests": [ + "tests/test_history_db_fallback_hidden.py", + "tests/test_truncate_message_count_regression.py" + ], + "documentation": [ + "docs/attachments.md" + ], + "risks": [ + "Truncating messages re-indexes context window and clears cached tool calls." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/history/history_routes.py", + "symbol": "@router.get('/api/history/{session_id}')", + "line_range": "L178-L230", + "explanation": "Fetches message history timeline for a session." + }, + { + "path": "routes/history_routes.py", + "symbol": "_sys.modules[__name__] = _canonical", + "line_range": "L1-L17", + "explanation": "Backward-compatibility shim module." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CHAT-004", + "domain": "chat", + "name": "File & Multimodal Attachment Handling", + "purpose": "Handles uploading, mime validation, image preview, vision encoding, and file attachments in chat messages.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/fileHandler.js", + "static/js/chat.js" + ], + "backend_entrypoints": [ + "routes/upload_routes.py:setup_upload_routes", + "src/upload_handler.py:UploadHandler" + ], + "routes": [ + "/api/upload", + "/api/upload/{file_id}/vision", + "/api/upload/cleanup" + ], + "configuration": [ + "MAX_UPLOAD_SIZE_MB", + "ALLOWED_UPLOAD_EXTENSIONS" + ], + "persistence": [ + "DATA_DIR/uploads/" + ], + "dependencies": [ + "pillow", + "python-magic" + ], + "tests": [ + "tests/test_upload_handler_atomicity.py", + "tests/test_upload_routes_owner_scope.py" + ], + "documentation": [ + "docs/attachments.md" + ], + "risks": [ + "Large file uploads may consume server disk space if cleanup task fails." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/upload_routes.py", + "symbol": "@router.post('')", + "line_range": "L257-L310", + "explanation": "Accepts multi-part file uploads and generates vision metadata." + }, + { + "path": "src/upload_handler.py", + "symbol": "UploadHandler.save_file", + "line_range": "L50-L140", + "explanation": "Validates upload size and atomicity on disk." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CHAT-005", + "domain": "chat", + "name": "Chat Message Search", + "purpose": "Enables full-text keyword search across stored chat messages and sessions.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/search-chat.js", + "static/js/search.js" + ], + "backend_entrypoints": [ + "routes/search_routes.py:setup_search_routes", + "src/session_search.py:search_sessions" + ], + "routes": [ + "/api/search/chat" + ], + "configuration": [ + "SEARCH_INDEX_CACHE_TTL" + ], + "persistence": [ + "SQLite FTS / session indices" + ], + "dependencies": [ + "sqlite3" + ], + "tests": [ + "tests/test_session_search.py", + "tests/test_session_search_batch_fetch.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Full table scans on un-indexed text columns for very large databases." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/search_routes.py", + "symbol": "setup_search_routes", + "line_range": "L30-L80", + "explanation": "Registers chat message search endpoint." + }, + { + "path": "src/session_search.py", + "symbol": "search_sessions", + "line_range": "L20-L90", + "explanation": "Executes query matching against session transcripts." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CHAT-006", + "domain": "chat", + "name": "System Prompts & Preset Management", + "purpose": "Provides creation, selection, and customization of system prompt presets for chat sessions.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/presets.js", + "static/js/settings.js" + ], + "backend_entrypoints": [ + "routes/preset_routes.py:setup_preset_routes", + "src/preset_manager.py:PresetManager" + ], + "routes": [ + "/api/presets", + "/api/presets/{preset_id}" + ], + "configuration": [ + "PRESETS_FILE" + ], + "persistence": [ + "DATA_DIR/presets.json" + ], + "dependencies": [ + "pydantic", + "json" + ], + "tests": [ + "tests/cli/test_preset_cli_store.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Invalid JSON syntax in user presets file can corrupt preset loading." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/preset_routes.py", + "symbol": "setup_preset_routes", + "line_range": "L30-L110", + "explanation": "API routes for listing and modifying system prompt presets." + }, + { + "path": "src/preset_manager.py", + "symbol": "PresetManager", + "line_range": "L15-L100", + "explanation": "Disk-backed manager for prompt presets." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CHAT-007", + "domain": "chat", + "name": "Emoji Rendering & Twemoji SVG Proxy", + "purpose": "Proxies Twemoji SVG icons locally to render flat SVG emojis in message text without external CDN dependencies.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/emojiPicker.js", + "static/js/emojiShortcodes.js" + ], + "backend_entrypoints": [ + "routes/emoji_routes.py:setup_emoji_routes" + ], + "routes": [ + "/api/emoji/{code}.svg" + ], + "configuration": [ + "EMOJI_CACHE_DIR" + ], + "persistence": [ + "DATA_DIR/emoji_cache/" + ], + "dependencies": [ + "httpx", + "fastapi.responses.Response" + ], + "tests": [ + "tests/test_censor_pref_js.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "First request fetches SVG from remote CDN before caching locally." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/emoji_routes.py", + "symbol": "setup_emoji_routes", + "line_range": "L20-L100", + "explanation": "Serves locally cached Twemoji SVGs." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CHAT-008", + "domain": "chat", + "name": "Input History Recall (Arrow Up)", + "purpose": "Allows users to cycle through previously sent prompt messages in the chat composer input using Arrow-Up/Down keys.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/composerArrowUpRecall.js", + "static/js/chat.js" + ], + "backend_entrypoints": [ + "None (Pure client-side state feature)" + ], + "routes": [], + "configuration": [ + "localStorage key: chat_composer_history" + ], + "persistence": [ + "Browser localStorage" + ], + "dependencies": [ + "Vanilla JavaScript DOM keyboard listeners" + ], + "tests": [], + "documentation": [ + "README.md" + ], + "risks": [ + "Client-side browser storage limits." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/js/composerArrowUpRecall.js", + "symbol": "initComposerRecall", + "line_range": "L1-L80", + "explanation": "Listens for ArrowUp keypress on composer textarea." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CHAT-009", + "domain": "chat", + "name": "Context Window Compaction & Truncation", + "purpose": "Compacts session transcript history when prompt size exceeds context limits using summarization.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chat.js" + ], + "backend_entrypoints": [ + "routes/history/history_routes.py:compact_session", + "src/context_compactor.py:compact_context" + ], + "routes": [ + "/api/session/{session_id}/compact", + "/api/session/{session_id}/context" + ], + "configuration": [ + "MAX_CONTEXT_TOKENS", + "COMPACTION_THRESHOLD" + ], + "persistence": [ + "DATA_DIR/sessions/" + ], + "dependencies": [ + "tiktoken", + "fastapi" + ], + "tests": [ + "tests/test_context_compactor.py" + ], + "documentation": [ + "specs/architecture-runtime-inventory.md" + ], + "risks": [ + "Aggressive compaction may discard subtle user instructions." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/history/history_routes.py", + "symbol": "@router.post('/api/session/{session_id}/compact')", + "line_range": "L751-L790", + "explanation": "Triggers context summarization and compaction." + }, + { + "path": "src/context_compactor.py", + "symbol": "compact_context", + "line_range": "L25-L110", + "explanation": "Executes context token pruning and summary generation." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "AGENT-001", + "domain": "agent", + "name": "Autonomous Agent Loop & Tool Execution Engine", + "purpose": "Executes multi-step agent reasoning loops, tool invocation parsing, and automated response generation.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chatStream.js" + ], + "backend_entrypoints": [ + "src/agent_loop.py:run_agent_loop", + "src/tool_execution.py:execute_tool_call" + ], + "routes": [ + "/api/chat/stream" + ], + "configuration": [ + "MAX_AGENT_STEPS", + "ENABLE_TOOL_EXECUTION" + ], + "persistence": [ + "Session transcript tool calls" + ], + "dependencies": [ + "asyncio", + "pydantic" + ], + "tests": [ + "tests/test_tool_policy.py", + "tests/test_unknown_tool_calls.py" + ], + "documentation": [ + "specs/architecture-runtime-inventory.md" + ], + "risks": [ + "Infinite tool loop if termination condition fails." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/agent_loop.py", + "symbol": "run_agent_loop", + "line_range": "L40-L210", + "explanation": "Core loop evaluating model tool requests and executing handlers." + }, + { + "path": "src/tool_execution.py", + "symbol": "execute_tool_call", + "line_range": "L30-L150", + "explanation": "Dispatches tool invocation requests to underlying tool handlers." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "AGENT-002", + "domain": "agent", + "name": "Scheduled Tasks & Event Bus Dispatcher", + "purpose": "Schedules background recurring or delayed tasks, emits event bus triggers, and executes automated flows.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/tasks.js" + ], + "backend_entrypoints": [ + "routes/task_routes.py:setup_task_routes", + "src/task_scheduler.py:TaskScheduler", + "src/event_bus.py" + ], + "routes": [ + "/api/tasks", + "/api/tasks/{task_id}/run", + "/api/tasks/{task_id}/pause", + "/api/tasks/{task_id}/resume" + ], + "configuration": [ + "TASK_SCHEDULER_INTERVAL" + ], + "persistence": [ + "DATA_DIR/tasks.db" + ], + "dependencies": [ + "apscheduler", + "sqlite3" + ], + "tests": [ + "tests/test_task_scheduler_cancel.py", + "tests/test_task_chain_owner_scope.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Task execution failure handling on system restart." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/task_routes.py", + "symbol": "@router.get('')", + "line_range": "L341-L370", + "explanation": "Fetches active scheduled tasks." + }, + { + "path": "src/task_scheduler.py", + "symbol": "TaskScheduler", + "line_range": "L40-L280", + "explanation": "Async task scheduler dispatching cron and delay triggers." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "AGENT-003", + "domain": "agent", + "name": "Webhook Event Subscriptions & Trigger Processing", + "purpose": "Manages incoming/outgoing webhook subscriptions, endpoint authentication tokens, and event triggers.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/admin.js" + ], + "backend_entrypoints": [ + "routes/webhook_routes.py:setup_webhook_routes", + "src/webhook_manager.py:WebhookManager" + ], + "routes": [ + "/api/webhooks", + "/v1/chat", + "/api/webhooks/{webhook_id}/test" + ], + "configuration": [ + "WEBHOOK_SECRET_KEY" + ], + "persistence": [ + "DATA_DIR/webhooks.db" + ], + "dependencies": [ + "httpx", + "sqlite3" + ], + "tests": [ + "tests/test_webhook_ssrf_resilience.py", + "tests/test_webhook_trigger_auth_exempt.py" + ], + "documentation": [ + "docs/pr-blocker-audit.md" + ], + "risks": [ + "SSRF risks when contacting external webhook URLs if unvalidated." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/webhook_routes.py", + "symbol": "@router.get('/webhooks')", + "line_range": "L71-L94", + "explanation": "Returns list of registered webhooks." + }, + { + "path": "src/webhook_manager.py", + "symbol": "WebhookManager", + "line_range": "L30-L160", + "explanation": "Handles payload delivery and signature verification." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "AGENT-004", + "domain": "agent", + "name": "Assistant Settings, Task Check-Ins & Background Job Monitor", + "purpose": "Manages per-user assistant sessions and scheduled check-in settings, drains background job completions, and retains a legacy no-op activity logging shim.", + "status": "partial", + "frontend_entrypoints": [ + "static/js/assistant.js" + ], + "backend_entrypoints": [ + "routes/assistant_routes.py:setup_assistant_routes", + "src/assistant_log.py", + "src/bg_monitor.py" + ], + "routes": [ + "/api/assistant/session", + "/api/assistant/settings", + "/api/assistant/run/{task_id}", + "/api/assistant/run-status/{task_id}", + "/api/assistant/available-timezones" + ], + "configuration": [ + "CrewMember.timezone (IANA tz name driving check-in scheduling)", + "CrewMember.model", + "CrewMember.enabled_tools", + "ScheduledTask.scheduled_time / ScheduledTask.status", + "src/bg_monitor.py:POLL_INTERVAL_S", + "src/bg_monitor.py:_FOLLOWUP_MAX_ROUNDS" + ], + "persistence": [ + "CrewMember (core/database.py, is_default_assistant singleton per owner)", + "Session pinned via CrewMember.session_id", + "ScheduledTask (per-owner check-in rows)", + "TaskRun (most recent run status for run-status polling)" + ], + "dependencies": [ + "logging", + "asyncio" + ], + "tests": [ + "tests/cli/test_logs_cli_resolve_nonstring.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Assistant seeding is owner-scoped: routes/assistant_routes.py documents that reaching an /api/assistant route under a synthetic owner previously seeded a duplicate CrewMember plus Morning/Midday/Evening tasks that double-fired; correctness now depends on RESERVED_USERNAMES staying complete.", + "Check-in next_run values are recomputed from CrewMember.timezone through compute_next_run, so a missing or stale IANA timezone shifts when check-ins fire.", + "Background follow-up continuation is retry-until-success: _run_followup returns False to defer and mark_followed_up() runs only after a successful agent run, so a persistently failing continuation is retried every POLL_INTERVAL_S tick without a bounded attempt count." + ], + "unknowns": [ + "Route `/api/assistant/logs` cited in legacy docs is absent from assistant router.", + "Existing unit test `tests/cli/test_logs_cli_resolve_nonstring.py` tests CLI target-name resolution logic, not active assistant routes or bg_monitor execution loop." + ], + "evidence": [ + { + "path": "routes/assistant_routes.py", + "symbol": "setup_assistant_routes", + "line_range": "L80-L326", + "explanation": "Active assistant session, settings, manual check-in, run-status and timezone-list endpoints, including the owner-scoping guards." + }, + { + "path": "src/bg_monitor.py", + "symbol": "_drain_agent", + "line_range": "L28-L72", + "explanation": "Runs the agent loop headless against a session to produce the background-job follow-up turn." + }, + { + "path": "src/bg_monitor.py", + "symbol": "_run_followup", + "line_range": "L75-L132", + "explanation": "Drains completed background jobs and auto-continues the owning session, deferring while a live turn is in progress." + }, + { + "path": "src/assistant_log.py", + "symbol": "log_to_assistant", + "line_range": "L34-L48", + "explanation": "Legacy no-op activity logging shim retained for existing callers; documented as inactive rather than as current behaviour." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "AGENT-005", + "domain": "agent", + "name": "Model Context Protocol (MCP) Server Integration", + "purpose": "Integrates external MCP servers over stdio/SSE to expand agent capabilities dynamically.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/settings.js", + "static/js/admin.js" + ], + "backend_entrypoints": [ + "routes/mcp_routes.py:setup_mcp_routes", + "src/mcp_manager.py:McpManager", + "mcp_servers/" + ], + "routes": [ + "/api/mcp/servers", + "/api/mcp/tools", + "/api/mcp/connect" + ], + "configuration": [ + "MCP_CONFIG_PATH" + ], + "persistence": [ + "DATA_DIR/mcp_config.json" + ], + "dependencies": [ + "mcp", + "asyncio" + ], + "tests": [ + "tests/test_mcp_param_hint_hardening.py", + "tests/cli/test_mcp_cli_json.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Subprocess leaks if external MCP server process fails to terminate clean." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/mcp_routes.py", + "symbol": "setup_mcp_routes", + "line_range": "L60-L240", + "explanation": "Exposes management endpoints for external MCP servers." + }, + { + "path": "src/mcp_manager.py", + "symbol": "McpManager", + "line_range": "L40-L300", + "explanation": "Manages MCP server subprocess lifecycles." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "AGENT-006", + "domain": "agent", + "name": "AI Interaction Tools & Pipeline Orchestration", + "purpose": "Provides specialized AI interaction tools for agent self-debugging, debate, and multi-model collaboration.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chatStream.js" + ], + "backend_entrypoints": [ + "src/ai_interaction.py", + "src/action_intents.py", + "src/builtin_actions.py" + ], + "routes": [ + "Implicit agent tool calls" + ], + "configuration": [ + "MAX_PIPELINE_TURNS" + ], + "persistence": [ + "Session transcript state" + ], + "dependencies": [ + "asyncio" + ], + "tests": [ + "tests/test_builtin_actions_nonstring.py", + "tests/test_model_interaction_registry.py" + ], + "documentation": [ + "specs/architecture-runtime-inventory.md" + ], + "risks": [ + "High API token consumption during extended agent debates." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/ai_interaction.py", + "symbol": "init_ai_interaction_tools", + "line_range": "L30-L140", + "explanation": "Registers specialized multi-agent interaction primitives." + }, + { + "path": "src/builtin_actions.py", + "symbol": "execute_builtin_action", + "line_range": "L20-L90", + "explanation": "Executes pre-built action intent sequences." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "AGENT-007", + "domain": "agent", + "name": "Subprocess & Background Job Execution Tools", + "purpose": "Provides sandboxed bash/shell tool execution capabilities with output streaming and background tracking.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chatStream.js", + "static/js/tasks.js" + ], + "backend_entrypoints": [ + "src/agent_tools/subprocess_tools.py", + "src/agent_tools/bg_job_tools.py", + "src/bg_jobs.py" + ], + "routes": [ + "Implicit agent tool calls" + ], + "configuration": [ + "ALLOW_SHELL_EXECUTION", + "SANDBOX_DIR" + ], + "persistence": [ + "DATA_DIR/bg_jobs/" + ], + "dependencies": [ + "subprocess", + "asyncio" + ], + "tests": [ + "tests/test_bg_job_tools.py", + "tests/test_task_shell_tools.py" + ], + "documentation": [ + "THREAT_MODEL.md" + ], + "risks": [ + "Arbitrary shell command execution permissions if sandbox confinement fails." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/agent_tools/subprocess_tools.py", + "symbol": "run_command", + "line_range": "L25-L120", + "explanation": "Executes shell commands in background/foreground." + }, + { + "path": "src/bg_jobs.py", + "symbol": "JobManager", + "line_range": "L30-L150", + "explanation": "Tracks async background subprocess tasks." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MODEL-001", + "domain": "model", + "name": "Multi-Provider LLM Model Discovery & Metadata Management", + "purpose": "Discovers models from OpenAI, Anthropic, Ollama, vLLM, LMStudio, OpenRouter, and Google AI Studio endpoints.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/models.js", + "static/js/modelPicker.js" + ], + "backend_entrypoints": [ + "routes/model_routes.py:setup_model_routes", + "src/model_discovery.py:ModelDiscovery" + ], + "routes": [ + "/api/models", + "/api/models/active", + "/api/model-endpoints" + ], + "configuration": [ + "OPENAI_API_KEY", + "OLLAMA_BASE_URL", + "ANTHROPIC_API_KEY" + ], + "persistence": [ + "DATA_DIR/model_endpoints.json" + ], + "dependencies": [ + "httpx", + "pydantic" + ], + "tests": [ + "tests/test_model_routes.py", + "tests/test_provider_classification.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Remote endpoint timeouts may slow down full discovery refresh." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/model_routes.py", + "symbol": "@router.get('/api/models')", + "line_range": "L100-L180", + "explanation": "Returns unified list of available models across providers." + }, + { + "path": "src/model_discovery.py", + "symbol": "ModelDiscovery.discover_all", + "line_range": "L45-L210", + "explanation": "Queries connected provider endpoints for available model IDs." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MODEL-002", + "domain": "model", + "name": "Model Capability & Context Limits Detection", + "purpose": "Detects vision, tool calling, reasoning, and context window limits for connected model endpoints.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/models.js" + ], + "backend_entrypoints": [ + "src/model_capabilities.py", + "src/model_context.py", + "src/endpoint_resolver.py" + ], + "routes": [ + "/api/model-endpoints/{ep_id}/probe" + ], + "configuration": [ + "MODEL_CAPABILITY_OVERRODES" + ], + "persistence": [ + "In-memory capabilities cache" + ], + "dependencies": [ + "pydantic" + ], + "tests": [ + "tests/test_endpoint_resolver_headers.py", + "tests/test_vision_model_detection.py" + ], + "documentation": [ + "specs/architecture-runtime-inventory.md" + ], + "risks": [ + "Incorrect context limit metadata for unlisted custom fine-tunes." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/model_capabilities.py", + "symbol": "get_model_capabilities", + "line_range": "L30-L120", + "explanation": "Maps model names to vision and tool support flags." + }, + { + "path": "src/endpoint_resolver.py", + "symbol": "resolve_endpoint_headers", + "line_range": "L20-L80", + "explanation": "Resolves auth headers and target URLs for model endpoints." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MODEL-003", + "domain": "model", + "name": "LLM Core Provider Communication & Fallback Routing", + "purpose": "Manages HTTP request dispatching, authorization header injection, and fallback provider routing for LLM calls.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chatStream.js" + ], + "backend_entrypoints": [ + "src/llm_core.py:llm_call_async", + "src/llm_core.py:llm_call_async_with_fallback", + "src/llm_core.py:stream_llm_with_fallback" + ], + "routes": [ + "Implicit backend calls" + ], + "configuration": [ + "LLM_RETRY_ATTEMPTS", + "LLM_TIMEOUT" + ], + "persistence": [ + "Session state" + ], + "dependencies": [ + "httpx", + "asyncio" + ], + "tests": [ + "tests/test_llm_core_concurrency.py", + "tests/test_llm_core_fallback.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Unexpected API changes in upstream third-party model providers." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/llm_core.py", + "symbol": "llm_call_async", + "line_range": "L1949-L2118", + "explanation": "Non-streaming provider request dispatcher: resolves the endpoint, injects authorization headers and executes the HTTP call." + }, + { + "path": "src/llm_core.py", + "symbol": "llm_call_async_with_fallback", + "line_range": "L1932-L1946", + "explanation": "Ordered fallback wrapper that retries llm_call_async across the configured candidate endpoints." + }, + { + "path": "src/llm_core.py", + "symbol": "stream_llm_with_fallback", + "line_range": "L2794-L2895", + "explanation": "Ordered fallback wrapper for the streaming path; advances to the next candidate when a provider yields an empty completion." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Provider dispatch, header injection and fallback advancement are only observable against a reachable LLM provider endpoint; not exercised in this documentation pass." + } + }, + { + "id": "MODEL-004", + "domain": "model", + "name": "Model Selection & Display Ordering Preferences", + "purpose": "Allows pinning, sorting, and hiding specific models in the UI selection dropdown.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/modelSort.js", + "static/js/modelPicker.js" + ], + "backend_entrypoints": [ + "routes/model_routes.py:save_model_order" + ], + "routes": [ + "/api/models/order", + "/api/models/order/reset" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/model_order.json" + ], + "dependencies": [ + "json" + ], + "tests": [ + "tests/test_model_defaults.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Stale model IDs in custom order lists after model endpoints are removed." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/model_routes.py", + "symbol": "@router.post('/order')", + "line_range": "L500-L550", + "explanation": "Saves custom model display order preference." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MODEL-005", + "domain": "model", + "name": "Side-by-Side Model Comparison (A/B Testing)", + "purpose": "Enables dual-model side-by-side response evaluation, arena scoring, and latency comparison.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/compare/index.js", + "static/js/compare/panes.js" + ], + "backend_entrypoints": [ + "routes/compare/compare_routes.py:setup_compare_routes" + ], + "routes": [ + "/api/compare/start", + "/api/compare/{comp_id}/vote", + "/api/compare/history" + ], + "configuration": [ + "COMPARE_ENABLED" + ], + "persistence": [ + "DATA_DIR/compare_history.db" + ], + "dependencies": [ + "sqlite3" + ], + "tests": [ + "tests/test_endpoint_owner_scope_followup.py" + ], + "documentation": [ + "docs/compare.webm" + ], + "risks": [ + "High memory and network usage when streaming two model responses simultaneously." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/compare/compare_routes.py", + "symbol": "@router.post('/start')", + "line_range": "L70-L150", + "explanation": "Starts a parallel dual-model comparison stream." + }, + { + "path": "static/js/compare/index.js", + "symbol": "initCompareView", + "line_range": "L1-L100", + "explanation": "Renders side-by-side model chat panes." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MODEL-006", + "domain": "model", + "name": "GitHub Copilot Device Flow Authentication", + "purpose": "Authenticates with GitHub Copilot via OAuth device flow to use Copilot models directly.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/providerDeviceFlow.js" + ], + "backend_entrypoints": [ + "routes/copilot_routes.py:setup_copilot_routes", + "routes/device_flow.py:create_device_flow_router", + "src/copilot.py:request_device_code", + "src/copilot.py:poll_access_token" + ], + "routes": [ + "/api/copilot/device/start", + "/api/copilot/device/poll" + ], + "configuration": [ + "COPILOT_CLIENT_ID" + ], + "persistence": [ + "DATA_DIR/copilot_auth.json" + ], + "dependencies": [ + "httpx" + ], + "tests": [ + "tests/test_provider_device_flow_js.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Token expiration requires manual device re-authentication." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/copilot_routes.py", + "symbol": "setup_copilot_routes", + "line_range": "L166-L173", + "explanation": "Builds the Copilot device-flow router at prefix /api/copilot, wiring _start_device_flow and _poll_device_flow." + }, + { + "path": "routes/device_flow.py", + "symbol": "create_device_flow_router", + "line_range": "L135-L193", + "explanation": "Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix." + }, + { + "path": "src/copilot.py", + "symbol": "request_device_code", + "line_range": "L150-L161", + "explanation": "Issues the GitHub device-code request that begins the Copilot OAuth device flow." + }, + { + "path": "src/copilot.py", + "symbol": "poll_access_token", + "line_range": "L164-L180", + "explanation": "Polls GitHub for the access token once the user has authorized the device code." + }, + { + "path": "tests/test_provider_device_flow_js.py", + "symbol": "test_copilot_success_uses_complete_verification_uri", + "line_range": "L28-L65", + "explanation": "Inspected unit test asserting the Copilot device-flow runner surfaces the complete verification URI returned by the backend." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E2", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires an interactive GitHub Copilot OAuth device-flow account." + } + }, + { + "id": "MODEL-007", + "domain": "model", + "name": "ChatGPT Subscription Device Flow Authentication", + "purpose": "Authenticates with ChatGPT Pro/Plus subscription tokens via device login flow.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/providerDeviceFlow.js" + ], + "backend_entrypoints": [ + "routes/chatgpt_subscription_routes.py:setup_chatgpt_subscription_routes", + "routes/device_flow.py:create_device_flow_router", + "src/chatgpt_subscription.py:request_device_code", + "src/chatgpt_subscription.py:poll_device_auth" + ], + "routes": [ + "/api/chatgpt-subscription/device/start", + "/api/chatgpt-subscription/device/poll" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/chatgpt_auth.json" + ], + "dependencies": [ + "httpx" + ], + "tests": [ + "tests/test_provider_device_flow_js.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Changes in OpenAI auth endpoint security challenges." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/chatgpt_subscription_routes.py", + "symbol": "setup_chatgpt_subscription_routes", + "line_range": "L163-L170", + "explanation": "Builds the ChatGPT subscription device-flow router at prefix /api/chatgpt-subscription." + }, + { + "path": "routes/device_flow.py", + "symbol": "create_device_flow_router", + "line_range": "L135-L193", + "explanation": "Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix." + }, + { + "path": "src/chatgpt_subscription.py", + "symbol": "request_device_code", + "line_range": "L168-L181", + "explanation": "Issues the ChatGPT device-authorization request that begins the subscription OAuth device flow." + }, + { + "path": "src/chatgpt_subscription.py", + "symbol": "poll_device_auth", + "line_range": "L184-L193", + "explanation": "Polls the ChatGPT device-authorization endpoint for completion using the stored device_auth_id and user_code." + }, + { + "path": "tests/test_provider_device_flow_js.py", + "symbol": "test_chatgpt_success_uses_plain_verification_uri", + "line_range": "L68-L95", + "explanation": "Inspected unit test asserting the ChatGPT device-flow runner uses the plain verification URI rather than the Copilot complete-URI form." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E2", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires an interactive ChatGPT subscription OAuth flow." + } + }, + { + "id": "MODEL-008", + "domain": "model", + "name": "Embedding Model Lane & Vector Provider Setup", + "purpose": "Configures local sentence-transformers, FastEmbed, or remote OpenAI embedding model lanes.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/settings.js" + ], + "backend_entrypoints": [ + "routes/embedding_routes.py:setup_embedding_routes", + "src/embeddings.py", + "src/embedding_lanes.py" + ], + "routes": [ + "/api/embeddings/active", + "/api/embeddings/test" + ], + "configuration": [ + "EMBEDDING_PROVIDER", + "EMBEDDING_MODEL" + ], + "persistence": [ + "DATA_DIR/embeddings_config.json" + ], + "dependencies": [ + "fastembed", + "sentence-transformers" + ], + "tests": [ + "tests/test_embedding_lane_ndarray_restore.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "First-time download of heavy PyTorch model weights on CPU-only machines." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/embedding_routes.py", + "symbol": "setup_embedding_routes", + "line_range": "L30-L120", + "explanation": "Provides embedding provider configuration endpoints." + }, + { + "path": "src/embeddings.py", + "symbol": "EmbeddingManager", + "line_range": "L40-L200", + "explanation": "Generates dense vector embeddings for RAG and memory." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "COOKBOOK-001", + "domain": "cookbook", + "name": "Local Model Download & Recipe Lifecycle Management", + "purpose": "Downloads HuggingFace models, configures execution parameters, and manages local GGUF/MLX model servers.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/cookbook.js", + "static/js/cookbookServe.js", + "static/js/cookbookDownload.js" + ], + "backend_entrypoints": [ + "routes/cookbook_routes.py:setup_cookbook_routes", + "src/tools/cookbook.py", + "src/cookbook_serve_lifecycle.py" + ], + "routes": [ + "/api/cookbook/download", + "/api/cookbook/serve", + "/api/cookbook/status" + ], + "configuration": [ + "COOKBOOK_MODELS_DIR" + ], + "persistence": [ + "DATA_DIR/models/" + ], + "dependencies": [ + "huggingface_hub", + "subprocess" + ], + "tests": [ + "tests/test_cookbook_endpoint_registration.py", + "tests/test_cookbook_port_parsing_js.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Disk space exhaustion during multi-gigabyte GGUF weights downloads." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/cookbook_routes.py", + "symbol": "setup_cookbook_routes", + "line_range": "L100-L300", + "explanation": "Exposes model downloading and process serving endpoints." + }, + { + "path": "static/js/cookbook.js", + "symbol": "initCookbook", + "line_range": "L1-L150", + "explanation": "UI manager for local model library." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "COOKBOOK-002", + "domain": "cookbook", + "name": "Hardware Model Fitting ('What Fits?') Analysis Engine", + "purpose": "Calculates RAM/VRAM requirements, quantized size, and context overhead to determine model compatibility.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/cookbook-hwfit.js" + ], + "backend_entrypoints": [ + "routes/hwfit_routes.py:setup_hwfit_routes", + "services/hwfit/fit.py:calculate_fit" + ], + "routes": [ + "/api/hwfit/fit", + "/api/hwfit/profile" + ], + "configuration": [ + "FORCE_GPU_VENDOR" + ], + "persistence": [ + "System hardware specs" + ], + "dependencies": [ + "psutil", + "torch" + ], + "tests": [ + "tests/test_hwfit_gemma4_12b.py", + "tests/test_hwfit_bandwidth_nonstring.py", + "tests/test_hwfit_gpu_count_nonnumeric.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Inaccurate VRAM estimation for non-standard KV-cache quantization." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/hwfit_routes.py", + "symbol": "setup_hwfit_routes", + "line_range": "L40-L120", + "explanation": "Calculates hardware model compatibility." + }, + { + "path": "services/hwfit/fit.py", + "symbol": "calculate_fit", + "line_range": "L30-L140", + "explanation": "Performs parameter and memory fit calculations." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "COOKBOOK-003", + "domain": "cookbook", + "name": "HuggingFace & MLX Model Discovery Services", + "purpose": "Searches HuggingFace Hub and local MLX model repositories for compatible GGUF and MLX weights.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/cookbook.js" + ], + "backend_entrypoints": [ + "routes/hwfit_routes.py:hf_search", + "services/hwfit/hf_discovery.py" + ], + "routes": [ + "/api/hwfit/hf-search", + "/api/hwfit/mlx-models" + ], + "configuration": [ + "HF_TOKEN" + ], + "persistence": [ + "Cache directory" + ], + "dependencies": [ + "huggingface_hub" + ], + "tests": [ + "tests/test_cookbook_hf_token.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "HuggingFace API rate limits when searching without an API token." + ], + "unknowns": [], + "evidence": [ + { + "path": "services/hwfit/hf_discovery.py", + "symbol": "search_hf_models", + "line_range": "L20-L90", + "explanation": "Queries HuggingFace API for model tags and files." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "COOKBOOK-004", + "domain": "cookbook", + "name": "Host Docker Access for Inference Container Runtimes", + "purpose": "Detects and connects to host Docker engine to launch containerized Ollama, vLLM, or SGLang runtimes.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/cookbookServe.js" + ], + "backend_entrypoints": [ + "src/host_docker_access.py:HostDockerAccess" + ], + "routes": [ + "Implicit local docker socket API calls" + ], + "configuration": [ + "DOCKER_HOST" + ], + "persistence": [ + "/var/run/docker.sock" + ], + "dependencies": [ + "docker" + ], + "tests": [ + "tests/test_cookbook_docker_access.py" + ], + "documentation": [ + "docker/host-docker.yml" + ], + "risks": [ + "Permission denied accessing docker socket on non-root setups." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/host_docker_access.py", + "symbol": "HostDockerAccess", + "line_range": "L50-L62", + "explanation": "Interacts with host docker daemon." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires Docker access and supported physical GPU hardware." + } + }, + { + "id": "RESEARCH-001", + "domain": "research", + "name": "Deep Research Execution Engine & SSE Progress Streaming", + "purpose": "Executes multi-step recursive deep research tasks, web page scraping, synthesis, and streams live progress.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/research/panel.js", + "static/js/research/jobs.js", + "static/js/researchSynapse.js" + ], + "backend_entrypoints": [ + "routes/research/research_routes.py:start_research", + "src/deep_research.py", + "services/research/service.py" + ], + "routes": [ + "/api/research/start", + "/api/research/stream/{session_id}", + "/api/research/active" + ], + "configuration": [ + "MAX_RESEARCH_DEPTH", + "SEARXNG_URL" + ], + "persistence": [ + "DATA_DIR/research/" + ], + "dependencies": [ + "beautifulsoup4", + "httpx", + "asyncio" + ], + "tests": [ + "tests/test_research_owner_scope_routes.py", + "tests/test_services_research_low_quality_sources.py" + ], + "documentation": [ + "docs/research.webm" + ], + "risks": [ + "High memory consumption when parsing multi-megabyte HTML target pages." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/research/research_routes.py", + "symbol": "@router.post('/api/research/start')", + "line_range": "L492-L550", + "explanation": "Initiates deep research job." + }, + { + "path": "src/deep_research.py", + "symbol": "DeepResearchEngine", + "line_range": "L40-L300", + "explanation": "Recursive search and summary crawler." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "RESEARCH-002", + "domain": "research", + "name": "Research Library, Detail View & Image Controls", + "purpose": "Stores completed research reports, generated diagrams, reference links, and manages image visibility.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/research/panel.js" + ], + "backend_entrypoints": [ + "routes/research/research_routes.py:get_library" + ], + "routes": [ + "/api/research/library", + "/api/research/detail/{session_id}", + "/api/research/{session_id}/hide-image" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/research/library.json" + ], + "dependencies": [ + "pydantic" + ], + "tests": [ + "tests/cli/test_research_cli_preview.py", + "tests/test_research_routes_path_confinement.py" + ], + "documentation": [ + "docs/research.webm" + ], + "risks": [ + "Orphaned report files if storage directory is modified out-of-band." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/research/research_routes.py", + "symbol": "@router.get('/api/research/library')", + "line_range": "L366-L415", + "explanation": "Returns all saved research reports." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "RESEARCH-003", + "domain": "research", + "name": "Web Search Engine Integration (SearXNG & Multi-Provider)", + "purpose": "Queries SearXNG, DuckDuckGo, or Google Search instances to retrieve web search snippets.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chat.js", + "static/js/search.js" + ], + "backend_entrypoints": [ + "routes/search_routes.py:setup_search_routes", + "src/search/core.py", + "services/search/service.py" + ], + "routes": [ + "/api/search" + ], + "configuration": [ + "SEARXNG_URL", + "SEARCH_PROVIDER" + ], + "persistence": [ + "In-memory search cache" + ], + "dependencies": [ + "httpx" + ], + "tests": [ + "tests/test_search_ranking.py", + "tests/test_searxng_image_pinned.py" + ], + "documentation": [ + "config/searxng/settings.yml" + ], + "risks": [ + "Search provider IP throttling or rate-limiting." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/search_routes.py", + "symbol": "setup_search_routes", + "line_range": "L39-L100", + "explanation": "Defines /api/search, /api/search/config, and /api/search/query endpoints." + }, + { + "path": "src/search/core.py", + "symbol": "SearchEngine", + "line_range": "L1-L12", + "explanation": "Compatibility module aliasing services.search.core." + }, + { + "path": "tests/test_search_ranking.py", + "symbol": "test_news_queries_prefer_news_sources_over_sports_and_social_results", + "line_range": "L1-L39", + "explanation": "Tests search result domain ranking and scoring." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires an active SearXNG instance or external search API provider." + } + }, + { + "id": "RESEARCH-004", + "domain": "research", + "name": "Research Result Peeking & Topic Spinoff Generation", + "purpose": "Extracts preliminary research snippets and spawns child research sessions focused on specific sub-topics.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/research/panel.js" + ], + "backend_entrypoints": [ + "routes/research/research_routes.py:peek_result", + "routes/research/research_routes.py:spinoff_research" + ], + "routes": [ + "/api/research/result-peek/{session_id}", + "/api/research/spinoff/{session_id}" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/research/" + ], + "dependencies": [ + "fastapi" + ], + "tests": [ + "tests/test_research_routes_path_confinement.py" + ], + "documentation": [ + "docs/research.webm" + ], + "risks": [ + "Deep recursion tree depth when spawning multiple nested spinoffs." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/research/research_routes.py", + "symbol": "@router.post('/api/research/spinoff/{session_id}')", + "line_range": "L635-L680", + "explanation": "Spawns child research session for specific query." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "DOCUMENT-001", + "domain": "document", + "name": "Document & Canvas Artifact Management", + "purpose": "Renders dynamic canvas documents, handles live editing, markdown preview, and side-by-side artifact display.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/document.js", + "static/js/documentLibrary.js" + ], + "backend_entrypoints": [ + "routes/document_routes.py:setup_document_routes", + "src/document_actions.py", + "src/document_processor.py" + ], + "routes": [ + "/api/document", + "/api/document/{id}" + ], + "configuration": [ + "MAX_DOCUMENT_SIZE_MB" + ], + "persistence": [ + "DATA_DIR/documents/" + ], + "dependencies": [ + "pydantic", + "fastapi" + ], + "tests": [ + "tests/test_document_actions_nonstring.py", + "tests/test_document_diff_discard_on_update_js.py" + ], + "documentation": [ + "docs/document.webm" + ], + "risks": [ + "Concurrent edits on the same document artifact." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/document_routes.py", + "symbol": "setup_document_routes", + "line_range": "L100-L300", + "explanation": "Registers document artifact CRUD routes." + }, + { + "path": "static/js/document.js", + "symbol": "initDocumentView", + "line_range": "L1-L150", + "explanation": "Renders interactive canvas document panel." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "DOCUMENT-002", + "domain": "document", + "name": "PDF Form Processing & High-Fidelity Rendering", + "purpose": "Extracts form fields from PDF files, fills dynamic values, and generates PDF previews.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/document.js" + ], + "backend_entrypoints": [ + "src/pdf_runtime.py", + "src/pdf_forms.py", + "src/pdf_form_doc.py" + ], + "routes": [ + "Implicit PDF document processing" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/documents/pdf/" + ], + "dependencies": [ + "pypdf", + "fitz" + ], + "tests": [ + "tests/test_document_pdf_marker.py", + "tests/test_security_headers_pdf_preview.py" + ], + "documentation": [ + "docs/attachments.md" + ], + "risks": [ + "Complex XFA PDF forms may not extract cleanly with standard pdf parsers." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/pdf_runtime.py", + "symbol": "load_pymupdf_for_pdf_viewer", + "line_range": "L9-L15", + "explanation": "Loads optional PyMuPDF runtime for PDF viewing." + }, + { + "path": "src/pdf_forms.py", + "symbol": "extract_form_fields", + "line_range": "L1-L100", + "explanation": "Handles PDF form field extraction and filling." + }, + { + "path": "tests/test_document_pdf_marker.py", + "symbol": "test_marker_removed_without_eating_following_text", + "line_range": "L1-L30", + "explanation": "Tests PDF text extraction wrapper stripping without content corruption." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires optional PyMuPDF (`fitz`) or pypdf runtime dependency." + } + }, + { + "id": "DOCUMENT-003", + "domain": "document", + "name": "Personal Document Indexing & RAG Retrieval", + "purpose": "Indexes local user documents (PDF, DOCX, TXT) into ChromaDB for semantic vector retrieval.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/rag.js" + ], + "backend_entrypoints": [ + "routes/personal_routes.py:setup_personal_routes", + "src/personal_docs.py", + "src/rag_manager.py" + ], + "routes": [ + "/api/personal/documents", + "/api/personal/search", + "/api/personal/index" + ], + "configuration": [ + "CHROMADB_DIR" + ], + "persistence": [ + "DATA_DIR/chroma/" + ], + "dependencies": [ + "chromadb", + "pypdf", + "docx" + ], + "tests": [ + "tests/test_personal_docs_pdf_index.py", + "tests/test_rag_index_hidden_dirs.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Slow vector embedding indexing step for massive multi-thousand page documents." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/personal_routes.py", + "symbol": "setup_personal_routes", + "line_range": "L50-L160", + "explanation": "Personal document RAG indexing and search API endpoints." + }, + { + "path": "src/personal_docs.py", + "symbol": "PersonalDocsManager", + "line_range": "L30-L180", + "explanation": "Handles file text chunking and vector storage." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "DOCUMENT-004", + "domain": "document", + "name": "Document Conversion & Text Extraction Engine", + "purpose": "Converts office formats (.docx, .xlsx, .pptx) and HTML into clean Markdown text representations.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/fileHandler.js" + ], + "backend_entrypoints": [ + "src/markitdown_runtime.py", + "src/office_doc.py" + ], + "routes": [ + "Implicit file conversion calls" + ], + "configuration": [], + "persistence": [ + "Temporary conversion cache" + ], + "dependencies": [ + "markitdown", + "python-docx" + ], + "tests": [ + "tests/cli/test_docs_cli_content_length.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Formatting loss when parsing legacy binary doc/xls files." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/markitdown_runtime.py", + "symbol": "convert_to_markdown", + "line_range": "L15-L80", + "explanation": "Converts binary office documents into structured Markdown text." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "DOCUMENT-005", + "domain": "document", + "name": "Document Library UI Navigation", + "purpose": "Provides dedicated UI view for browsing, filtering, and organizing saved user documents.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/documentLibrary.js" + ], + "backend_entrypoints": [ + "app.py:serve_library" + ], + "routes": [ + "/library" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/documents/" + ], + "dependencies": [ + "Vanilla JS" + ], + "tests": [ + "tests/test_document_close_clears_active_route.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Large folder trees may cause initial DOM render slowdown." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/js/documentLibrary.js", + "symbol": "initDocumentLibrary", + "line_range": "L1-L100", + "explanation": "Renders document library navigation grid." + }, + { + "path": "app.py", + "symbol": "serve_library", + "line_range": "L911-L914", + "explanation": "Serves SPA shell for /library route." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "EMAIL-001", + "domain": "email", + "name": "Email Account Setup, IMAP/SMTP Connection & Polling", + "purpose": "Configures IMAP/SMTP email accounts, validates TLS certificates, and polls background inbox updates.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/emailInbox.js" + ], + "backend_entrypoints": [ + "routes/email_routes.py:setup_email_routes", + "routes/email_pollers.py" + ], + "routes": [ + "/api/email/accounts", + "/api/email/poll" + ], + "configuration": [ + "EMAIL_POLL_INTERVAL" + ], + "persistence": [ + "DATA_DIR/email_accounts.json" + ], + "dependencies": [ + "imaplib", + "smtplib" + ], + "tests": [ + "tests/test_service_health_email.py", + "tests/test_active_email_reply_guard.py" + ], + "documentation": [ + "docs/email-outlook.md" + ], + "risks": [ + "Account lockouts if bad credentials are repeatedly polled." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/email_routes.py", + "symbol": "setup_email_routes", + "line_range": "L1453-L1500", + "explanation": "Sets up email account management and synchronization routes." + }, + { + "path": "routes/email_pollers.py", + "symbol": "_start_poller", + "line_range": "L1-L100", + "explanation": "Background poller for email inbox synchronization." + }, + { + "path": "tests/test_service_health_email.py", + "symbol": "test_email_ok_all_connect", + "line_range": "L1-L80", + "explanation": "Tests IMAP connection health probing and status reporting." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires a controlled live IMAP account and network access." + } + }, + { + "id": "EMAIL-002", + "domain": "email", + "name": "Email Searching, Threading & Message Operations", + "purpose": "Parses email headers, folds signatures, groups messages into threads, and executes full-text email search.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/emailLibrary.js", + "static/js/emailLibrary/state.js" + ], + "backend_entrypoints": [ + "routes/email_routes.py:search_email", + "src/email_thread_parser.py" + ], + "routes": [ + "/api/email/search", + "/api/email/threads", + "/api/email/messages" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/email_cache.db" + ], + "dependencies": [ + "sqlite3", + "email" + ], + "tests": [ + "tests/test_reply_recipients_js.py", + "tests/test_signature_fold_js.py" + ], + "documentation": [ + "docs/email-outlook.md" + ], + "risks": [ + "Malformed MIME email structures failing HTML sanitization." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/email_routes.py", + "symbol": "@router.get('/search')", + "line_range": "L250-L320", + "explanation": "Executes search across cached email headers and text." + }, + { + "path": "src/email_thread_parser.py", + "symbol": "parse_email_thread", + "line_range": "L20-L100", + "explanation": "Builds conversation tree from Message-ID and In-Reply-To headers." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "EMAIL-003", + "domain": "email", + "name": "Email Composition, Draft Management & Sending", + "purpose": "Creates, saves, and dispatches HTML/plaintext email messages via SMTP.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/emailShared.js" + ], + "backend_entrypoints": [ + "routes/email_routes.py:send_email" + ], + "routes": [ + "/api/email/draft", + "/api/email/send" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/email_drafts.json" + ], + "dependencies": [ + "smtplib" + ], + "tests": [ + "tests/test_schedule_email_offset_normalization.py" + ], + "documentation": [ + "docs/email-outlook.md" + ], + "risks": [ + "SMTP connection drop mid-send causing unsent mail state." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/email_routes.py", + "symbol": "@router.post('/send')", + "line_range": "L450-L520", + "explanation": "Sends email message via user SMTP credentials." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires a controlled live SMTP account and network access." + } + }, + { + "id": "EMAIL-004", + "domain": "email", + "name": "Email MCP Server & Codex Integration Bridge", + "purpose": "Exposes constrained email reading and draft capabilities to external Codex / MCP agents with scope checks.", + "status": "verified", + "frontend_entrypoints": [ + "integrations/codex/scripts/odysseus_api.py" + ], + "backend_entrypoints": [ + "mcp_servers/email_server.py", + "routes/codex_routes.py:setup_codex_routes" + ], + "routes": [ + "/api/codex/email/search", + "/api/codex/email/send" + ], + "configuration": [ + "CODEX_API_KEY" + ], + "persistence": [ + "API token scopes" + ], + "dependencies": [ + "mcp", + "fastapi" + ], + "tests": [ + "tests/cli/test_mail_cli_recipients.py" + ], + "documentation": [ + "integrations/codex/README.md" + ], + "risks": [ + "Unauthorized mail sending if token scopes are improperly scoped." + ], + "unknowns": [], + "evidence": [ + { + "path": "mcp_servers/email_server.py", + "symbol": "EmailMcpServer", + "line_range": "L20-L120", + "explanation": "MCP server exposing email tools over stdio/SSE." + }, + { + "path": "routes/codex_routes.py", + "symbol": "setup_codex_routes", + "line_range": "L30-L110", + "explanation": "Bridge endpoints for external Codex plugin integration." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CALENDAR-001", + "domain": "calendar", + "name": "CalDAV Calendar Synchronization & Account Setup", + "purpose": "Connects to remote CalDAV servers (Apple iCloud, Nextcloud, Google) to sync calendar event feeds.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/calendar.js" + ], + "backend_entrypoints": [ + "routes/calendar_routes.py:setup_calendar_routes", + "src/caldav_sync.py:CalDavSync" + ], + "routes": [ + "/api/calendar/config", + "/api/calendar/sync" + ], + "configuration": [ + "CALDAV_URL", + "CALDAV_USERNAME" + ], + "persistence": [ + "DATA_DIR/calendar_config.json" + ], + "dependencies": [ + "caldav", + "vobject" + ], + "tests": [ + "tests/test_caldav_url_hardening.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Invalid SSL certificates on self-hosted CalDAV servers." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/calendar_routes.py", + "symbol": "setup_calendar_routes", + "line_range": "L50-L180", + "explanation": "Exposes CalDAV setup and manual sync trigger routes." + }, + { + "path": "src/caldav_sync.py", + "symbol": "CalDavSync", + "line_range": "L30-L160", + "explanation": "Fetches and parses remote iCalendar VEVENT objects." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires a controlled external CalDAV server." + } + }, + { + "id": "CALENDAR-002", + "domain": "calendar", + "name": "Calendar Event Operations & iCalendar Parsing", + "purpose": "Creates, updates, deletes, and displays calendar events with timezone conversion and reminder notifications.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/calendar/reminders.js", + "static/js/calendar/utils.js" + ], + "backend_entrypoints": [ + "routes/calendar_routes.py:create_event", + "src/caldav_writeback.py", + "src/tools/calendar.py" + ], + "routes": [ + "/api/calendar/events", + "/api/calendar/events/{event_id}" + ], + "configuration": [ + "USER_TIMEZONE" + ], + "persistence": [ + "DATA_DIR/calendar_events.db" + ], + "dependencies": [ + "sqlite3", + "icalendar" + ], + "tests": [ + "tests/test_calendar_parse_dt_time_first.py", + "tests/test_calendar_update_event_tz.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Recurring RRULE event expansion calculation bugs across leap years." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/calendar_routes.py", + "symbol": "@router.get('/events')", + "line_range": "L220-L310", + "explanation": "Fetches calendar events for requested date window." + }, + { + "path": "src/tools/calendar.py", + "symbol": "CalendarTool", + "line_range": "L25-L120", + "explanation": "Agent tool for creating and modifying calendar entries." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-001", + "domain": "media", + "name": "Gallery Image Library & Album Operations", + "purpose": "Organizes images into custom albums, provides grid browsing, tagging, and album metadata management.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/gallery.js" + ], + "backend_entrypoints": [ + "routes/gallery/gallery_routes.py:setup_gallery_routes" + ], + "routes": [ + "/api/gallery/library", + "/api/gallery/albums", + "/api/gallery/upload" + ], + "configuration": [ + "GALLERY_STORAGE_DIR" + ], + "persistence": [ + "DATA_DIR/gallery/" + ], + "dependencies": [ + "pillow", + "sqlite3" + ], + "tests": [ + "tests/test_gallery_image_endpoint_owner_scope.py", + "tests/test_gallery_owner_filter_single_user.py" + ], + "documentation": [ + "docs/gallery.webm" + ], + "risks": [ + "Thumbnail generation overhead for high-resolution RAW camera images." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/gallery/gallery_routes.py", + "symbol": "@router.get('/api/gallery/library')", + "line_range": "L657-L720", + "explanation": "Fetches image library list with tag filters." + }, + { + "path": "static/js/gallery.js", + "symbol": "initGallery", + "line_range": "L1-L180", + "explanation": "Main gallery grid renderer and uploader." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-002", + "domain": "media", + "name": "Image Processing, AI Upscaling & Style Transfer", + "purpose": "Executes local image enhancement, background removal, face sharpening, and AI upscaling operations.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/galleryEditor.js", + "static/js/editor/ai-tool-runner.js" + ], + "backend_entrypoints": [ + "routes/gallery/gallery_routes.py:ai_upscale", + "routes/gallery/gallery_routes.py:remove_bg" + ], + "routes": [ + "/api/gallery/ai-upscale", + "/api/gallery/style-transfer", + "/api/image/inpaint", + "/api/image/remove-bg" + ], + "configuration": [ + "REALESRGAN_MODEL_PATH" + ], + "persistence": [ + "DATA_DIR/gallery/processed/" + ], + "dependencies": [ + "onnxruntime", + "opencv-python" + ], + "tests": [ + "tests/test_sanitize_multimodal_merge.py" + ], + "documentation": [ + "docs/gallery.webm" + ], + "risks": [ + "High GPU memory allocation when upscaling 4K images." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/gallery/gallery_routes.py", + "symbol": "@router.post('/api/gallery/ai-upscale')", + "line_range": "L544-L580", + "explanation": "Runs RealESRGAN image upscaling." + }, + { + "path": "routes/gallery/gallery_routes.py", + "symbol": "@router.post('/api/image/remove-bg')", + "line_range": "L1950-L2010", + "explanation": "Executes background removal pass." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-003", + "domain": "media", + "name": "Interactive Image Canvas Editor & Persisted Drafts", + "purpose": "Provides full multi-layer raster canvas editor, brush tools, transforms, masks, and draft project persistence.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/galleryEditor.js", + "static/js/editor/history-panel.js" + ], + "backend_entrypoints": [ + "routes/editor_draft_routes.py:setup_editor_draft_routes" + ], + "routes": [ + "/api/editor/drafts", + "/api/editor/drafts/{draft_id}" + ], + "configuration": [ + "MAX_DRAFT_PROJECTS" + ], + "persistence": [ + "DATA_DIR/editor_drafts/" + ], + "dependencies": [ + "HTML5 Canvas API", + "pydantic" + ], + "tests": [ + "tests/test_canvas_coords_empty_touches_js.py", + "tests/test_snap_other_layers_nonarray_js.py" + ], + "documentation": [ + "docs/gallery.webm" + ], + "risks": [ + "Browser memory leak if multi-gigabyte layer undo buffers are kept indefinitely." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/editor_draft_routes.py", + "symbol": "setup_editor_draft_routes", + "line_range": "L30-L110", + "explanation": "API routes for saving and loading canvas project drafts." + }, + { + "path": "static/js/editor/history-panel.js", + "symbol": "HistoryManager", + "line_range": "L1-L90", + "explanation": "Canvas undo/redo stack manager." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-004", + "domain": "media", + "name": "Text-to-Speech (TTS) Synthesis Service", + "purpose": "Synthesizes spoken audio from text using local Kokoro, EdgeTTS, or OpenAI TTS engines.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/tts-ai.js" + ], + "backend_entrypoints": [ + "routes/tts_routes.py:setup_tts_routes", + "services/tts/tts_service.py:TTSService" + ], + "routes": [ + "/api/tts/synthesize", + "/api/tts/stats" + ], + "configuration": [ + "TTS_PROVIDER", + "TTS_VOICE" + ], + "persistence": [ + "DATA_DIR/tts_cache/" + ], + "dependencies": [ + "soundfile", + "edge-tts" + ], + "tests": [ + "tests/test_tts_available_nonstring_provider.py", + "tests/test_tts_cache_stats.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Audio synthesis latency on CPU-only hardware setups." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/tts_routes.py", + "symbol": "@router.post('/synthesize')", + "line_range": "L30-L75", + "explanation": "Synthesizes TTS audio clip." + }, + { + "path": "services/tts/tts_service.py", + "symbol": "TTSService", + "line_range": "L25-L140", + "explanation": "Provider abstraction layer for audio speech generation." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-005", + "domain": "media", + "name": "Speech-to-Text (STT) Audio Transcription Service", + "purpose": "Transcribes user audio recordings into text using faster-whisper or local speech models.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/voiceRecorder.js" + ], + "backend_entrypoints": [ + "routes/stt_routes.py:setup_stt_routes", + "services/stt/stt_service.py:STTService" + ], + "routes": [ + "/api/stt/transcribe", + "/api/stt/stats" + ], + "configuration": [ + "STT_PROVIDER", + "WHISPER_MODEL_SIZE" + ], + "persistence": [ + "Temporary audio buffer" + ], + "dependencies": [ + "faster-whisper", + "ffmpeg-python" + ], + "tests": [ + "tests/test_stt_leak.py", + "tests/test_speech_service_toggles.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Missing ffmpeg system dependency prevents audio format decoding." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/stt_routes.py", + "symbol": "@router.post('/transcribe')", + "line_range": "L25-L55", + "explanation": "Accepts multipart audio file and returns transcription text." + }, + { + "path": "services/stt/stt_service.py", + "symbol": "STTService", + "line_range": "L20-L110", + "explanation": "Whisper audio transcription engine wrapper." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-006", + "domain": "media", + "name": "Digital Signature Stamp Storage & Placement", + "purpose": "Stores transparent PNG user signatures and stamps for placement onto PDF forms and documents.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/signature.js" + ], + "backend_entrypoints": [ + "routes/signature_routes.py:setup_signature_routes" + ], + "routes": [ + "/api/signatures", + "/api/signatures/{sig_id}" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/signatures/" + ], + "dependencies": [ + "pillow" + ], + "tests": [ + "tests/test_signature_route_hardening.py", + "tests/test_signature_settings_dom_xss.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Cross-site scripting if signature image titles contain unescaped user input." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/signature_routes.py", + "symbol": "setup_signature_routes", + "line_range": "L30-L120", + "explanation": "CRUD endpoints for managing user signature PNG stamps." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-007", + "domain": "media", + "name": "Generated Image Artifact Route & MCP Integration", + "purpose": "Serves generated AI artwork artifacts and integrates with image generation MCP server.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chat.js" + ], + "backend_entrypoints": [ + "app.py:serve_generated_image", + "src/generated_images.py", + "mcp_servers/image_gen_server.py" + ], + "routes": [ + "/api/generated-image/{filename}" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/generated_images/" + ], + "dependencies": [ + "pillow", + "mcp" + ], + "tests": [ + "tests/test_image_models_nondict_system.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Path traversal vulnerability if filename parameter is un-sanitized." + ], + "unknowns": [], + "evidence": [ + { + "path": "app.py", + "symbol": "serve_generated_image", + "line_range": "L499-L520", + "explanation": "Serves generated image artifacts with cache headers." + }, + { + "path": "src/generated_images.py", + "symbol": "resolve_generated_image_path", + "line_range": "L20-L32", + "explanation": "Confines requested image path within artifacts directory." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEDIA-008", + "domain": "media", + "name": "Native MLX Image Bridge (macOS Apple Silicon)", + "purpose": "Native Apple Swift bridge for hardware-accelerated diffusion and MLX image colorization on macOS.", + "status": "experimental", + "frontend_entrypoints": [ + "static/js/editor/ai-tools-misc.js" + ], + "backend_entrypoints": [ + "swift/odysseus-mlx-image-bridge/Package.swift", + "scripts/mlx_image_server.py", + "scripts/diffusion_server.py" + ], + "routes": [ + "/api/image/mlx-colorize" + ], + "configuration": [ + "ENABLE_MLX_ACCELERATION" + ], + "persistence": [ + "Build artifact binary" + ], + "dependencies": [ + "Swift", + "MLX Framework" + ], + "tests": [ + "tests/helpers/import_state.py" + ], + "documentation": [ + "swift/odysseus-mlx-image-bridge/Package.swift" + ], + "risks": [ + "Requires macOS host with Apple Silicon M-series chip and compiled Swift binary." + ], + "unknowns": [ + "Binary build requires Xcode command line tools build step (`build-macos-app.sh`)." + ], + "evidence": [ + { + "path": "swift/odysseus-mlx-image-bridge/Package.swift", + "symbol": "Package", + "line_range": "L1-L30", + "explanation": "Swift package manifest for native MLX image bridge." + }, + { + "path": "scripts/mlx_image_server.py", + "symbol": "main", + "line_range": "L20-L90", + "explanation": "Python daemon wrapping native Swift MLX binary." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires Apple Silicon, macOS tooling, and the compiled MLX bridge." + } + }, + { + "id": "SECURITY-001", + "domain": "security", + "name": "Authentication, Session Cookies & User Management", + "purpose": "Handles bcrypt password hashing, session cookie issuance, authentication enforcement, and user administration.", + "status": "verified", + "frontend_entrypoints": [ + "static/login.html", + "static/js/admin.js" + ], + "backend_entrypoints": [ + "routes/auth_routes.py:setup_auth_routes", + "core/auth.py:AuthManager" + ], + "routes": [ + "/api/auth/login", + "/api/auth/me", + "/api/auth/users", + "/api/auth/logout" + ], + "configuration": [ + "AUTH_ENABLED", + "SESSION_COOKIE_NAME" + ], + "persistence": [ + "DATA_DIR/auth.db" + ], + "dependencies": [ + "bcrypt", + "itsdangerous" + ], + "tests": [ + "tests/test_rename_user_owner_sync.py", + "tests/test_reserved_username_admin_escalation.py" + ], + "documentation": [ + "THREAT_MODEL.md", + "SECURITY.md" + ], + "risks": [ + "Cookie session hijack if deployed over unencrypted HTTP without HTTPS cookie flags." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/auth_routes.py", + "symbol": "@router.post('/login')", + "line_range": "L100-L180", + "explanation": "Authenticates credentials and sets session cookie." + }, + { + "path": "core/auth.py", + "symbol": "AuthManager", + "line_range": "L40-L260", + "explanation": "Handles user creation, password verification, and session tokens." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "SECURITY-002", + "domain": "security", + "name": "System Vault Encrypted Secret Storage", + "purpose": "Encrypts API keys, passphrases, and third-party secrets on disk using AES-GCM / PBKDF2 key derivation.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/settings.js" + ], + "backend_entrypoints": [ + "routes/vault_routes.py:setup_vault_routes", + "src/secret_storage.py:SecretVault", + "src/tools/vault.py" + ], + "routes": [ + "/api/vault/config", + "/api/vault/unlock", + "/api/vault/lock" + ], + "configuration": [ + "VAULT_SALT" + ], + "persistence": [ + "DATA_DIR/vault.enc" + ], + "dependencies": [ + "cryptography" + ], + "tests": [ + "tests/test_vault_password_not_in_argv.py" + ], + "documentation": [ + "THREAT_MODEL.md" + ], + "risks": [ + "Loss of vault master passphrase renders all encrypted secrets permanently unrecoverable." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/vault_routes.py", + "symbol": "setup_vault_routes", + "line_range": "L126-L229", + "explanation": "Admin routes for vault configuration, login, unlock, lock, and logout." + }, + { + "path": "src/secret_storage.py", + "symbol": "SecretStorage", + "line_range": "L57-L87", + "explanation": "Fernet symmetric key DB secret encryption." + }, + { + "path": "tests/test_vault_password_not_in_argv.py", + "symbol": "test_bw_password_not_in_argv", + "line_range": "L1-L117", + "explanation": "Verifies master password is fed via stdin and never appears in process argv." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires installed Bitwarden CLI (`bw`) executable." + } + }, + { + "id": "SECURITY-003", + "domain": "security", + "name": "API Token Management & Scope Access Control", + "purpose": "Generates scoped API bearer tokens (read/write/admin) for external tool and script authentication.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/settings.js" + ], + "backend_entrypoints": [ + "routes/api_token_routes.py:setup_api_token_routes", + "core/database.py:ApiToken" + ], + "routes": [ + "/api/tokens", + "/api/tokens/{token_id}" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/odysseus.db (api_tokens table)" + ], + "dependencies": [ + "sqlalchemy", + "secrets" + ], + "tests": [ + "tests/test_api_key_file_permissions.py" + ], + "documentation": [ + "SECURITY.md" + ], + "risks": [ + "Leaked API bearer tokens with excessive permission scopes." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/api_token_routes.py", + "symbol": "setup_api_token_routes", + "line_range": "L30-L150", + "explanation": "Exposes API token creation, scope assignment, and revocation." + }, + { + "path": "core/database.py", + "symbol": "ApiToken", + "line_range": "L50-L90", + "explanation": "SQLAlchemy ORM schema for API tokens and permissions." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "SECURITY-004", + "domain": "security", + "name": "Prompt Security & Injection Defense Engine", + "purpose": "Scans system prompts and external inputs for prompt injection attempts, jailbreaks, and sensitive data leaks.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/chat.js" + ], + "backend_entrypoints": [ + "src/prompt_security.py:sanitize_prompt", + "src/tool_security.py:validate_tool_input" + ], + "routes": [ + "Implicit security inspection" + ], + "configuration": [ + "ENABLE_PROMPT_SANITY_CHECK" + ], + "persistence": [], + "dependencies": [ + "re" + ], + "tests": [ + "tests/test_skill_index_prompt_injection.py", + "tests/test_tool_output_prompt_injection.py" + ], + "documentation": [ + "THREAT_MODEL.md" + ], + "risks": [ + "False positives blocking legitimate complex coding or security prompts." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/prompt_security.py", + "symbol": "untrusted_context_message", + "line_range": "L64-L86", + "explanation": "Wraps untrusted context with guard delimiters and sets metadata.trusted = False." + }, + { + "path": "src/tool_security.py", + "symbol": "NON_ADMIN_BLOCKED_TOOLS", + "line_range": "L42-L78", + "explanation": "Enforces tool execution safety for non-admin user roles." + }, + { + "path": "tests/test_skill_index_prompt_injection.py", + "symbol": "test_skill_index", + "line_range": "L1-L208", + "explanation": "Verifies skill index descriptions cannot leak into trusted system prompts." + }, + { + "path": "tests/test_tool_output_prompt_injection.py", + "symbol": "test_tool_output", + "line_range": "L1-L50", + "explanation": "Tool output injection guards." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "SECURITY-005", + "domain": "security", + "name": "URL & Path Confinement Security Guards", + "purpose": "Prevents SSRF attacks and path traversal by validating target IP addresses and resolving symlinks.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/fileHandler.js" + ], + "backend_entrypoints": [ + "src/url_security.py:validate_url", + "src/url_safety.py", + "core/log_safety.py" + ], + "routes": [ + "Implicit guard functions" + ], + "configuration": [ + "ALLOWED_DOMAINS_WHITELIST" + ], + "persistence": [], + "dependencies": [ + "ipaddress", + "urllib.parse" + ], + "tests": [ + "tests/test_url_safety.py", + "tests/test_tool_path_confinement.py", + "tests/test_workspace_confine.py" + ], + "documentation": [ + "THREAT_MODEL.md" + ], + "risks": [ + "DNS rebinding attacks if IP address is re-resolved post-validation." + ], + "unknowns": [], + "evidence": [ + { + "path": "src/url_safety.py", + "symbol": "check_outbound_url", + "line_range": "L60-L108", + "explanation": "Rejects non-HTTP(S) schemes, link-local, cloud metadata SSRF addresses." + }, + { + "path": "src/url_security.py", + "symbol": "validate_public_http_url", + "line_range": "L81-L94", + "explanation": "Validates public-facing endpoints." + }, + { + "path": "tests/test_url_safety.py", + "symbol": "test_url_safety", + "line_range": "L1-L117", + "explanation": "Scheme validation, cloud metadata SSRF rejection, IP classification." + }, + { + "path": "tests/test_tool_path_confinement.py", + "symbol": "test_path_confinement", + "line_range": "L1-L50", + "explanation": "Path traversal checks." + }, + { + "path": "tests/test_workspace_confine.py", + "symbol": "test_workspace_confine", + "line_range": "L1-L50", + "explanation": "Workspace confinement checks." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "SECURITY-006", + "domain": "security", + "name": "HTTP Security Headers Middleware", + "purpose": "Injects standard OWASP HTTP security headers (CSP, HSTS, X-Content-Type-Options, X-Frame-Options).", + "status": "verified", + "frontend_entrypoints": [ + "static/app.js" + ], + "backend_entrypoints": [ + "core/middleware.py:SecurityHeadersMiddleware" + ], + "routes": [ + "Applies globally to all routes" + ], + "configuration": [ + "CSP_NONCE_ENABLED" + ], + "persistence": [], + "dependencies": [ + "starlette.middleware.base" + ], + "tests": [ + "tests/test_security_headers_middleware.py", + "tests/test_security_headers_pdf_preview.py" + ], + "documentation": [ + "SECURITY.md" + ], + "risks": [ + "Strict Content Security Policy (CSP) blocking third-party embedded web resources." + ], + "unknowns": [], + "evidence": [ + { + "path": "core/middleware.py", + "symbol": "SecurityHeadersMiddleware", + "line_range": "L40-L110", + "explanation": "Sets strict security headers and CSP nonces on HTTP responses." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "SECURITY-007", + "domain": "security", + "name": "Admin System Data Wipe ('Danger Zone')", + "purpose": "Provides administrative reset operations to wipe sessions, cache, uploaded files, or factory reset state.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/admin.js" + ], + "backend_entrypoints": [ + "routes/admin_wipe/admin_wipe_routes.py:setup_admin_wipe_routes" + ], + "routes": [ + "/api/admin/wipe/{kind}" + ], + "configuration": [], + "persistence": [ + "ALL storage locations" + ], + "dependencies": [ + "os", + "shutil" + ], + "tests": [ + "tests/test_session_ghost_delete.py" + ], + "documentation": [ + "THREAT_MODEL.md" + ], + "risks": [ + "Accidental catastrophic data loss if triggered without user confirmation." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/admin_wipe/admin_wipe_routes.py", + "symbol": "@router.delete('/wipe/{kind}')", + "line_range": "L71-L130", + "explanation": "Executes systemic data wipe based on requested scope." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "FRONTEND-001", + "domain": "frontend", + "name": "Single Page Application Shell & Client Router", + "purpose": "Main HTML5 SPA shell, DOM lifecycle initializers, tab navigation, and deep-link route handlers.", + "status": "verified", + "frontend_entrypoints": [ + "static/index.html", + "static/app.js", + "static/js/init.js" + ], + "backend_entrypoints": [ + "app.py:serve_index" + ], + "routes": [ + "/", + "/notes", + "/calendar", + "/cookbook", + "/email", + "/memory", + "/gallery", + "/tasks", + "/library" + ], + "configuration": [], + "persistence": [ + "static/index.html" + ], + "dependencies": [ + "Vanilla HTML5/JS" + ], + "tests": [ + "tests/test_app_static_mime.py", + "tests/test_serve_html_with_nonce.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Stale browser static cache if asset hashing is omitted during deployment." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/index.html", + "symbol": "index.html", + "line_range": "L1-L200", + "explanation": "Main SPA entry point containing modal roots and CSS bundles." + }, + { + "path": "app.py", + "symbol": "serve_index", + "line_range": "L867-L878", + "explanation": "Serves index.html with dynamically generated CSP nonces." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "FRONTEND-002", + "domain": "frontend", + "name": "Dynamic Theme, Color System & Custom Fonts", + "purpose": "Supports dark/light themes, custom CSS variables, color picker controls, and user font uploads.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/theme.js", + "static/style.css", + "static/js/colorPicker.js" + ], + "backend_entrypoints": [ + "routes/font_routes.py:setup_font_routes", + "routes/prefs_routes.py:setup_prefs_routes" + ], + "routes": [ + "/api/font/custom", + "/api/prefs" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/user_prefs.json", + "static/fonts/custom/" + ], + "dependencies": [ + "Vanilla CSS", + "pydantic" + ], + "tests": [ + "tests/test_prefs_atomic_write.py", + "tests/test_select_dropdown_theme_css.py" + ], + "documentation": [ + "docs/theme.webm" + ], + "risks": [ + "Flash of unstyled content (FOUC) on slow connections." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/js/theme.js", + "symbol": "applyTheme", + "line_range": "L1-L90", + "explanation": "Applies custom HSL theme variables to DOM document root." + }, + { + "path": "routes/font_routes.py", + "symbol": "setup_font_routes", + "line_range": "L20-L55", + "explanation": "Allows uploading and serving custom WOFF2 font files." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "FRONTEND-003", + "domain": "frontend", + "name": "Window Manager, Tile Layout & Modal Control System", + "purpose": "Manages draggable tool windows, snapped multi-tile viewports, modal dialog Z-ordering, and ESC key stacks.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/modalManager.js", + "static/js/tileManager.js", + "static/js/windowDrag.js", + "static/js/windowResize.js", + "static/js/escMenuStack.js" + ], + "backend_entrypoints": [ + "None (Client UI subsystem)" + ], + "routes": [], + "configuration": [ + "localStorage: tile_layout_state" + ], + "persistence": [ + "Browser localStorage" + ], + "dependencies": [ + "Vanilla JS DOM event listeners" + ], + "tests": [ + "tests/test_portal_dropdown_z_js.py", + "tests/test_tile_manager_snap_zones_js.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Overlap artifacts when opening many simultaneous tool floating windows." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/js/modalManager.js", + "symbol": "ModalManager", + "line_range": "L1-L140", + "explanation": "Controls modal open/close transitions and focus trapping." + }, + { + "path": "static/js/tileManager.js", + "symbol": "TileManager", + "line_range": "L1-L180", + "explanation": "Handles viewport split-pane grid arrangements." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "FRONTEND-004", + "domain": "frontend", + "name": "Global Keyboard Shortcuts & Accessibility Controls", + "purpose": "Provides configurable hotkeys (Ctrl+K search, Esc close, Alt+1-9 tabs) and high-contrast accessibility options.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/keyboard-shortcuts.js", + "static/js/a11y.js" + ], + "backend_entrypoints": [ + "None (Client UI subsystem)" + ], + "routes": [], + "configuration": [ + "localStorage: user_hotkeys" + ], + "persistence": [ + "Browser localStorage" + ], + "dependencies": [ + "Vanilla JS" + ], + "tests": [ + "tests/test_keyboard_shortcuts.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Browser keybinding collisions with browser default hotkeys." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/js/keyboard-shortcuts.js", + "symbol": "initShortcuts", + "line_range": "L1-L110", + "explanation": "Binds global keydown handlers for system shortcuts." + }, + { + "path": "static/js/a11y.js", + "symbol": "initA11y", + "line_range": "L1-L80", + "explanation": "Applies ARIA roles and dyslexic font toggles." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "FRONTEND-005", + "domain": "frontend", + "name": "Markdown, LaTeX & Code Block Streaming Renderer", + "purpose": "Parses incoming SSE markdown streams, renders KaTeX math formulas, syntax-highlighted code, and interactive runners.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/markdown.js", + "static/js/streamingRenderer.js", + "static/js/streamingSegmenter.js", + "static/js/codeRunner.js" + ], + "backend_entrypoints": [ + "None (Client rendering subsystem)" + ], + "routes": [], + "configuration": [], + "persistence": [ + "Browser DOM" + ], + "dependencies": [ + "highlight.js", + "katex" + ], + "tests": [ + "tests/streaming/segmenter.test.mjs", + "tests/test_streaming_segmenter_js.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "DOM thrashing if streaming segmenter updates UI too frequently." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/js/markdown.js", + "symbol": "renderMarkdown", + "line_range": "L1-L200", + "explanation": "Converts markdown prose to HTML nodes with syntax highlighting." + }, + { + "path": "static/js/streamingSegmenter.js", + "symbol": "Segmenter", + "line_range": "L1-L150", + "explanation": "Parses un-closed markdown fences during live stream." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "FRONTEND-006", + "domain": "frontend", + "name": "Interactive Tour & Guided Onboarding System", + "purpose": "Presents interactive step-by-step feature tours and UI tooltip hints for new users.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/tourHints.js", + "static/js/tourAutoplay.js" + ], + "backend_entrypoints": [ + "None (Client UI subsystem)" + ], + "routes": [], + "configuration": [ + "localStorage: tour_completed" + ], + "persistence": [ + "Browser localStorage" + ], + "dependencies": [ + "Vanilla JS" + ], + "tests": [ + "tests/test_task_routes.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Tour step misalignment if window is resized mid-tour." + ], + "unknowns": [], + "evidence": [ + { + "path": "static/js/tourHints.js", + "symbol": "startTour", + "line_range": "L1-L120", + "explanation": "Renders guided feature tour overlays over target UI elements." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "FRONTEND-007", + "domain": "frontend", + "name": "Background Effects Prototyping Sandbox", + "purpose": "Standalone sandbox page for prototyping visual background animations, waves, and whirlpool effects.", + "status": "dead-code-candidate", + "frontend_entrypoints": [ + "static/wave-variants.html", + "static/whirlpool-variants.html" + ], + "backend_entrypoints": [ + "app.py:serve_backgrounds" + ], + "routes": [ + "/backgrounds" + ], + "configuration": [], + "persistence": [ + "None" + ], + "dependencies": [ + "HTML5 Canvas", + "WebGL" + ], + "tests": [], + "documentation": [ + "README.md" + ], + "risks": [ + "High GPU utilization when rendering complex shader wave animations." + ], + "unknowns": [ + "Route `/backgrounds` in app.py L918 attempts to serve `static/backgrounds.html` which is missing from disk; variant templates `wave-variants.html` and `whirlpool-variants.html` exist." + ], + "evidence": [ + { + "path": "app.py", + "symbol": "serve_backgrounds", + "line_range": "L915-L918", + "explanation": "Serves visual background sandbox HTML page route." + }, + { + "path": "static/wave-variants.html", + "symbol": "wave-variants.html", + "line_range": "L1-L150", + "explanation": "Interactive background effect prototyping sandbox variant." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E1", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-001", + "domain": "platform", + "name": "Application Initialization & Lifespan Management", + "purpose": "Orchestrates server startup, database table migration, background daemon initialization, and clean shutdown.", + "status": "verified", + "frontend_entrypoints": [ + "app.py" + ], + "backend_entrypoints": [ + "app.py:_lifespan", + "src/app_initializer.py:initialize_app" + ], + "routes": [ + "App startup lifespan" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/" + ], + "dependencies": [ + "asyncio", + "logging" + ], + "tests": [ + "tests/test_app_helpers.py" + ], + "documentation": [ + "docs/setup.md" + ], + "risks": [ + "Un-handled exceptions during startup halt application launch." + ], + "unknowns": [], + "evidence": [ + { + "path": "app.py", + "symbol": "_lifespan", + "line_range": "L996-L1030", + "explanation": "FastAPI lifespan context manager executing startup tasks." + }, + { + "path": "src/app_initializer.py", + "symbol": "initialize_app", + "line_range": "L29-L125", + "explanation": "Initializes app directories, DB schemas, and logging." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-002", + "domain": "platform", + "name": "System Health, Readiness & Version Monitoring APIs", + "purpose": "Exposes Liveness (/api/health), Readiness (/api/ready), App Version (/api/version), and Client Perf APIs.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/admin.js" + ], + "backend_entrypoints": [ + "app.py:health_check", + "app.py:readiness_check", + "app.py:get_version", + "src/readiness.py" + ], + "routes": [ + "/api/health", + "/api/ready", + "/api/version", + "/api/runtime", + "/api/client-perf" + ], + "configuration": [], + "persistence": [], + "dependencies": [ + "fastapi" + ], + "tests": [ + "tests/test_readiness.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Readiness check delays if verifying connectivity to offline remote endpoints." + ], + "unknowns": [], + "evidence": [ + { + "path": "app.py", + "symbol": "readiness_check", + "line_range": "L963-L973", + "explanation": "Performs system component integrity check." + }, + { + "path": "src/readiness.py", + "symbol": "check_readiness", + "line_range": "L15-L61", + "explanation": "Checks database, storage, and key paths for read/write access." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-003", + "domain": "platform", + "name": "Database Schema, Migrations & SQLite Persistence", + "purpose": "Defines core relational tables (users, tokens, tasks, sessions) and executes automated SQLite schema upgrades.", + "status": "verified", + "frontend_entrypoints": [ + "scripts/update_database.py" + ], + "backend_entrypoints": [ + "core/database.py:init_db", + "scripts/update_database.py:run_migrations", + "src/database.py" + ], + "routes": [ + "Implicit DB queries" + ], + "configuration": [ + "DATABASE_URL" + ], + "persistence": [ + "DATA_DIR/odysseus.db" + ], + "dependencies": [ + "sqlalchemy", + "alembic", + "sqlite3" + ], + "tests": [ + "tests/test_sqlite_foreign_keys.py", + "tests/test_update_database_script.py", + "tests/test_app_db_permissions.py" + ], + "documentation": [ + "docs/setup.md" + ], + "risks": [ + "SQLite file lock contention under high concurrent write loads." + ], + "unknowns": [], + "evidence": [ + { + "path": "core/database.py", + "symbol": "init_db", + "line_range": "L40-L120", + "explanation": "Creates ORM tables and establishes connection pool." + }, + { + "path": "scripts/update_database.py", + "symbol": "run_migrations", + "line_range": "L20-L110", + "explanation": "Applies missing schema columns and indices." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-004", + "domain": "platform", + "name": "User Data Export & Import Backup Infrastructure", + "purpose": "Exports complete user workspace state (sessions, memory, skills, notes, presets) into a zip archive.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/settings.js" + ], + "backend_entrypoints": [ + "routes/backup_routes.py:setup_backup_routes", + "scripts/odysseus-backup" + ], + "routes": [ + "/api/backup/export", + "/api/backup/import" + ], + "configuration": [], + "persistence": [ + "ZIP archive files" + ], + "dependencies": [ + "zipfile", + "json" + ], + "tests": [ + "tests/cli/test_preset_cli_store.py" + ], + "documentation": [ + "docs/backup-restore.md" + ], + "risks": [ + "Corrupt archive files causing partial data restore." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/backup_routes.py", + "symbol": "setup_backup_routes", + "line_range": "L30-L140", + "explanation": "Handles workspace data export and import upload unpack." + }, + { + "path": "docs/backup-restore.md", + "symbol": "Documentation", + "line_range": "L1-L50", + "explanation": "Backup and restore operational documentation." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-005", + "domain": "platform", + "name": "File Cleanup & Storage Maintenance Engine", + "purpose": "Scans data directories for orphaned files, old uploads, temporary vision images, and frees disk space.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/storage.js" + ], + "backend_entrypoints": [ + "routes/cleanup/cleanup_routes.py:setup_cleanup_routes", + "src/cleanup_service.py", + "src/session_image_cleanup.py" + ], + "routes": [ + "/api/cleanup/preview", + "/api/cleanup" + ], + "configuration": [ + "CLEANUP_RETENTION_DAYS" + ], + "persistence": [ + "DATA_DIR/uploads/" + ], + "dependencies": [ + "os", + "shutil" + ], + "tests": [ + "tests/test_session_actions_cleanup.py", + "tests/test_session_image_cleanup.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Deletes files uploaded in active sessions if retention window is set too short." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/cleanup/cleanup_routes.py", + "symbol": "@router.get('/preview')", + "line_range": "L22-L37", + "explanation": "Previews reclaimable disk space across storage directories." + }, + { + "path": "src/cleanup_service.py", + "symbol": "CleanupService", + "line_range": "L25-L120", + "explanation": "Executes filesystem purge of orphaned asset files." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-006", + "domain": "platform", + "name": "System Health & RAG Diagnostic Suite", + "purpose": "Executes real-time integrity diagnostics across ChromaDB, SearXNG, local models, and network interfaces.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/settings.js" + ], + "backend_entrypoints": [ + "routes/diagnostics_routes.py:setup_diagnostics_routes", + "src/service_health.py" + ], + "routes": [ + "/api/diagnostics" + ], + "configuration": [], + "persistence": [], + "dependencies": [ + "httpx", + "chromadb" + ], + "tests": [ + "tests/test_service_health_collect.py", + "tests/test_service_health_chromadb.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Diagnostic timeout if external search provider is unreachable." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/diagnostics_routes.py", + "symbol": "setup_diagnostics_routes", + "line_range": "L30-L100", + "explanation": "Runs subsystem health check suite." + }, + { + "path": "src/service_health.py", + "symbol": "collect_health_status", + "line_range": "L20-L150", + "explanation": "Inspects vector database, email, search, and local provider status." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-007", + "domain": "platform", + "name": "Desktop CLI Utilities & Shell Integration Tools", + "purpose": "Provides command-line interface tools (`odysseus`, `odysseus-mcp`, `odysseus-mail`) for terminal usage.", + "status": "verified", + "frontend_entrypoints": [ + "scripts/odysseus", + "scripts/odysseus-mcp", + "scripts/odysseus-mail", + "scripts/odysseus-calendar" + ], + "backend_entrypoints": [ + "scripts/_lib/cli.py:main" + ], + "routes": [ + "CLI commands" + ], + "configuration": [], + "persistence": [ + "CLI configuration" + ], + "dependencies": [ + "urllib", + "json" + ], + "tests": [ + "tests/cli/test_sessions_cli.py", + "tests/cli/test_mail_cli_recipients.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Outdated CLI scripts if backend API schemas change." + ], + "unknowns": [], + "evidence": [ + { + "path": "scripts/_lib/cli.py", + "symbol": "main", + "line_range": "L15-L110", + "explanation": "Shared CLI framework for terminal helper commands." + }, + { + "path": "scripts/odysseus", + "symbol": "odysseus", + "line_range": "L1-L50", + "explanation": "Main terminal launcher script." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-008", + "domain": "platform", + "name": "Desktop Companion App Integration", + "purpose": "Provides API routes and pairing mechanisms for the native macOS/desktop menu bar companion app.", + "status": "verified", + "frontend_entrypoints": [ + "companion/README.md" + ], + "backend_entrypoints": [ + "companion/pairing.py", + "companion/routes.py:setup_companion_routes" + ], + "routes": [ + "/companion/pair", + "/companion/status" + ], + "configuration": [ + "COMPANION_SECRET_KEY" + ], + "persistence": [ + "companion_pairing.json" + ], + "dependencies": [ + "fastapi" + ], + "tests": [ + "tests/helpers/import_state.py" + ], + "documentation": [ + "companion/README.md" + ], + "risks": [ + "Pairing code expiration timing window." + ], + "unknowns": [], + "evidence": [ + { + "path": "companion/routes.py", + "symbol": "setup_companion_routes", + "line_range": "L20-L120", + "explanation": "Endpoints for pairing and status sync with desktop companion." + }, + { + "path": "companion/pairing.py", + "symbol": "PairingManager", + "line_range": "L15-L80", + "explanation": "Generates and validates companion pairing codes." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "PLATFORM-009", + "domain": "platform", + "name": "Docker Containerization & GPU Hardware Manifests", + "purpose": "Provides multi-stage Dockerfile and Docker Compose manifests for CPU, NVIDIA CUDA, and AMD ROCm GPUs.", + "status": "verified", + "frontend_entrypoints": [ + "Dockerfile", + "docker-compose.yml", + "docker-compose.gpu-nvidia.yml", + "docker-compose.gpu-amd.yml" + ], + "backend_entrypoints": [ + "docker/entrypoint.sh" + ], + "routes": [ + "Containerized application" + ], + "configuration": [ + "DOCKER_GPU_VENDOR" + ], + "persistence": [ + "Container volumes" + ], + "dependencies": [ + "Docker", + "NVIDIA Container Toolkit" + ], + "tests": [ + "scripts/check-docker-gpu.sh" + ], + "documentation": [ + "docs/setup.md" + ], + "risks": [ + "Driver version incompatibility with host NVIDIA/AMD kernel drivers." + ], + "unknowns": [], + "evidence": [ + { + "path": "Dockerfile", + "symbol": "multi-stage-build", + "line_range": "L1-L113", + "explanation": "Multi-stage container build environment." + }, + { + "path": "docker-compose.gpu-nvidia.yml", + "symbol": "nvidia-gpu-manifest", + "line_range": "L1-L179", + "explanation": "NVIDIA GPU pass-through container specification." + }, + { + "path": "scripts/check-docker-gpu.sh", + "symbol": "check-docker-gpu", + "line_range": "L1-L615", + "explanation": "Automated diagnostic test script for host NVIDIA GPU passthrough." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": true, + "status": "pending", + "reason": "Requires Docker GPU pass-through and compatible host drivers." + } + }, + { + "id": "PLATFORM-010", + "domain": "platform", + "name": "Legacy FAISS Vector Index Migration Script", + "purpose": "Legacy utility script to migrate older FAISS vector indices into ChromaDB.", + "status": "legacy", + "frontend_entrypoints": [ + "None" + ], + "backend_entrypoints": [ + "scripts/migrate_faiss_to_chroma.py" + ], + "routes": [ + "CLI Script" + ], + "configuration": [], + "persistence": [ + "Legacy FAISS index files" + ], + "dependencies": [ + "faiss", + "chromadb" + ], + "tests": [ + "tests/helpers/import_state.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Fails if legacy FAISS index files do not exist." + ], + "unknowns": [ + "Superseded by native ChromaDB vector index pipeline." + ], + "evidence": [ + { + "path": "scripts/migrate_faiss_to_chroma.py", + "symbol": "migrate_faiss", + "line_range": "L15-L80", + "explanation": "Reads FAISS vector index files and writes to ChromaDB collection." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "NOTE-001", + "domain": "note", + "name": "Interactive Notes & Checklist Management", + "purpose": "Provides Google Keep-style notes, rich markdown text, checklist items, pinning, color tags, and reminders.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/notes.js" + ], + "backend_entrypoints": [ + "routes/note/note_routes.py:setup_note_routes", + "src/tools/notes.py" + ], + "routes": [ + "/api/notes", + "/api/notes/{note_id}", + "/api/notes/{note_id}/pin", + "/api/notes/reorder" + ], + "configuration": [], + "persistence": [ + "DATA_DIR/notes.db" + ], + "dependencies": [ + "sqlite3", + "pydantic" + ], + "tests": [ + "tests/test_notes_fail_closed_auth.py", + "tests/test_manage_notes_owner_gate.py" + ], + "documentation": [ + "docs/notes.webm" + ], + "risks": [ + "Concurrent edits on note item checkboxes." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/note/note_routes.py", + "symbol": "@router.get('')", + "line_range": "L623-L650", + "explanation": "Lists all user notes with pin and archive states." + }, + { + "path": "static/js/notes.js", + "symbol": "initNotesView", + "line_range": "L1-L160", + "explanation": "Main interactive notes grid and modal manager." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "SKILL-001", + "domain": "skill", + "name": "Dynamic Skill Management & Code Execution Engine", + "purpose": "Allows users to create, import, edit, test, and execute custom Python/Markdown skills dynamically.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/skills.js" + ], + "backend_entrypoints": [ + "routes/skills_routes.py:setup_skills_routes", + "services/memory/skills.py:SkillsManager" + ], + "routes": [ + "/api/skills", + "/api/skills/{skill_id}/invoke", + "/api/skills/import-from-url" + ], + "configuration": [ + "SKILLS_DIR" + ], + "persistence": [ + "DATA_DIR/skills/" + ], + "dependencies": [ + "pydantic", + "httpx" + ], + "tests": [ + "tests/test_skills_routes_owner_update.py", + "tests/test_skill_importer.py", + "tests/test_skill_save_no_rename.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "Arbitrary code execution risks if skill import URL is untrusted." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/skills_routes.py", + "symbol": "setup_skills_routes", + "line_range": "L100-L300", + "explanation": "Exposes CRUD and remote import routes for user skills." + }, + { + "path": "services/memory/skills.py", + "symbol": "SkillsManager", + "line_range": "L40-L220", + "explanation": "Handles skill storage, parsing, and execution." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "CONTACT-001", + "domain": "contact", + "name": "CardDAV Contact Management & Address Book Integration", + "purpose": "Connects to CardDAV servers, imports VCard contacts, and provides contact lookup for email/calendar autocomplete.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/emailLibrary.js" + ], + "backend_entrypoints": [ + "routes/contacts/contacts_routes.py:setup_contacts_routes", + "src/tools/contacts.py" + ], + "routes": [ + "/api/contacts/list", + "/api/contacts/search", + "/api/contacts/add", + "/api/contacts/config" + ], + "configuration": [ + "CARDDAV_URL" + ], + "persistence": [ + "DATA_DIR/contacts.db" + ], + "dependencies": [ + "vobject", + "sqlite3" + ], + "tests": [ + "tests/test_contacts_carddav_security.py", + "tests/cli/test_contacts_cli_rows.py" + ], + "documentation": [ + "README.md" + ], + "risks": [ + "VCard 3.0 vs 4.0 property parsing mismatches." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/contacts/contacts_routes.py", + "symbol": "@router.get('/list')", + "line_range": "L741-L764", + "explanation": "Returns contact list filtered by search query." + }, + { + "path": "src/tools/contacts.py", + "symbol": "ContactsTool", + "line_range": "L20-L110", + "explanation": "Agent tool for querying user address book contacts." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + }, + { + "id": "MEMORY-001", + "domain": "memory", + "name": "Persistent Long-Term Memory & Vector Indexing", + "purpose": "Extracts facts, user preferences, and temporal memories from chat sessions into vector/relational storage.", + "status": "verified", + "frontend_entrypoints": [ + "static/js/memory.js" + ], + "backend_entrypoints": [ + "routes/memory/memory_routes.py:setup_memory_routes", + "services/memory/service.py", + "mcp_servers/memory_server.py" + ], + "routes": [ + "/api/memory", + "/api/memory/search", + "/api/memory/extract", + "/api/memory/audit" + ], + "configuration": [ + "MEMORY_AUTO_EXTRACT" + ], + "persistence": [ + "DATA_DIR/memory.db", + "DATA_DIR/memory_chroma/" + ], + "dependencies": [ + "sqlite3", + "chromadb" + ], + "tests": [ + "tests/test_memory_routes_session_owner.py", + "tests/test_consolidate_memory_explicit_drops.py" + ], + "documentation": [ + "docs/theme.webm" + ], + "risks": [ + "Conflicting memory facts extracted from contradictory user prompts." + ], + "unknowns": [], + "evidence": [ + { + "path": "routes/memory/memory_routes.py", + "symbol": "@router.get('')", + "line_range": "L132-L150", + "explanation": "Fetches long-term user memory timeline." + }, + { + "path": "services/memory/memory_extractor.py", + "symbol": "MemoryExtractor", + "line_range": "L30-L160", + "explanation": "LLM-driven fact extraction from conversation transcripts." + } + ], + "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", + "evidence_maturity": "E0", + "runtime_validation": { + "required": false, + "status": "not-required", + "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." + } + } +] diff --git a/docs/discovery/feature-catalog.md b/docs/discovery/feature-catalog.md new file mode 100644 index 000000000..9a93c0317 --- /dev/null +++ b/docs/discovery/feature-catalog.md @@ -0,0 +1,85 @@ +# Feature Catalog + +This is a human-readable index derived from [`feature-catalog.json`](feature-catalog.json). The JSON file is canonical. + +| ID | Feature | Domain | Status | Evidence | Runtime | +|---|---|---|---|---|---| +| `AGENT-001` | Autonomous Agent Loop & Tool Execution Engine | `agent` | `verified` | `E0` | not required | +| `AGENT-002` | Scheduled Tasks & Event Bus Dispatcher | `agent` | `verified` | `E0` | not required | +| `AGENT-003` | Webhook Event Subscriptions & Trigger Processing | `agent` | `verified` | `E0` | not required | +| `AGENT-004` | Assistant Settings, Task Check-Ins & Background Job Monitor | `agent` | `partial` | `E1` | not required | +| `AGENT-005` | Model Context Protocol (MCP) Server Integration | `agent` | `verified` | `E0` | not required | +| `AGENT-006` | AI Interaction Tools & Pipeline Orchestration | `agent` | `verified` | `E0` | not required | +| `AGENT-007` | Subprocess & Background Job Execution Tools | `agent` | `verified` | `E0` | not required | +| `CALENDAR-001` | CalDAV Calendar Synchronization & Account Setup | `calendar` | `verified` | `E0` | pending | +| `CALENDAR-002` | Calendar Event Operations & iCalendar Parsing | `calendar` | `verified` | `E0` | not required | +| `CHAT-001` | Core Chat Streaming & SSE Message Generation | `chat` | `verified` | `E2` | pending | +| `CHAT-002` | Session Management & Conversation State | `chat` | `verified` | `E0` | not required | +| `CHAT-003` | Chat History & Message Editing/Truncation | `chat` | `verified` | `E0` | not required | +| `CHAT-004` | File & Multimodal Attachment Handling | `chat` | `verified` | `E0` | not required | +| `CHAT-005` | Chat Message Search | `chat` | `verified` | `E0` | not required | +| `CHAT-006` | System Prompts & Preset Management | `chat` | `verified` | `E0` | not required | +| `CHAT-007` | Emoji Rendering & Twemoji SVG Proxy | `chat` | `verified` | `E0` | not required | +| `CHAT-008` | Input History Recall (Arrow Up) | `chat` | `verified` | `E0` | not required | +| `CHAT-009` | Context Window Compaction & Truncation | `chat` | `verified` | `E0` | not required | +| `CONTACT-001` | CardDAV Contact Management & Address Book Integration | `contact` | `verified` | `E0` | not required | +| `COOKBOOK-001` | Local Model Download & Recipe Lifecycle Management | `cookbook` | `verified` | `E0` | not required | +| `COOKBOOK-002` | Hardware Model Fitting ('What Fits?') Analysis Engine | `cookbook` | `verified` | `E0` | not required | +| `COOKBOOK-003` | HuggingFace & MLX Model Discovery Services | `cookbook` | `verified` | `E0` | not required | +| `COOKBOOK-004` | Host Docker Access for Inference Container Runtimes | `cookbook` | `verified` | `E0` | pending | +| `DOCUMENT-001` | Document & Canvas Artifact Management | `document` | `verified` | `E0` | not required | +| `DOCUMENT-002` | PDF Form Processing & High-Fidelity Rendering | `document` | `verified` | `E1` | pending | +| `DOCUMENT-003` | Personal Document Indexing & RAG Retrieval | `document` | `verified` | `E0` | not required | +| `DOCUMENT-004` | Document Conversion & Text Extraction Engine | `document` | `verified` | `E0` | not required | +| `DOCUMENT-005` | Document Library UI Navigation | `document` | `verified` | `E0` | not required | +| `EMAIL-001` | Email Account Setup, IMAP/SMTP Connection & Polling | `email` | `verified` | `E1` | pending | +| `EMAIL-002` | Email Searching, Threading & Message Operations | `email` | `verified` | `E0` | not required | +| `EMAIL-003` | Email Composition, Draft Management & Sending | `email` | `verified` | `E0` | pending | +| `EMAIL-004` | Email MCP Server & Codex Integration Bridge | `email` | `verified` | `E0` | not required | +| `FRONTEND-001` | Single Page Application Shell & Client Router | `frontend` | `verified` | `E0` | not required | +| `FRONTEND-002` | Dynamic Theme, Color System & Custom Fonts | `frontend` | `verified` | `E0` | not required | +| `FRONTEND-003` | Window Manager, Tile Layout & Modal Control System | `frontend` | `verified` | `E0` | not required | +| `FRONTEND-004` | Global Keyboard Shortcuts & Accessibility Controls | `frontend` | `verified` | `E0` | not required | +| `FRONTEND-005` | Markdown, LaTeX & Code Block Streaming Renderer | `frontend` | `verified` | `E0` | not required | +| `FRONTEND-006` | Interactive Tour & Guided Onboarding System | `frontend` | `verified` | `E0` | not required | +| `FRONTEND-007` | Background Effects Prototyping Sandbox | `frontend` | `dead-code-candidate` | `E1` | not required | +| `MEDIA-001` | Gallery Image Library & Album Operations | `media` | `verified` | `E0` | not required | +| `MEDIA-002` | Image Processing, AI Upscaling & Style Transfer | `media` | `verified` | `E0` | not required | +| `MEDIA-003` | Interactive Image Canvas Editor & Persisted Drafts | `media` | `verified` | `E0` | not required | +| `MEDIA-004` | Text-to-Speech (TTS) Synthesis Service | `media` | `verified` | `E0` | not required | +| `MEDIA-005` | Speech-to-Text (STT) Audio Transcription Service | `media` | `verified` | `E0` | not required | +| `MEDIA-006` | Digital Signature Stamp Storage & Placement | `media` | `verified` | `E0` | not required | +| `MEDIA-007` | Generated Image Artifact Route & MCP Integration | `media` | `verified` | `E0` | not required | +| `MEDIA-008` | Native MLX Image Bridge (macOS Apple Silicon) | `media` | `experimental` | `E0` | pending | +| `MEMORY-001` | Persistent Long-Term Memory & Vector Indexing | `memory` | `verified` | `E0` | not required | +| `MODEL-001` | Multi-Provider LLM Model Discovery & Metadata Management | `model` | `verified` | `E0` | not required | +| `MODEL-002` | Model Capability & Context Limits Detection | `model` | `verified` | `E0` | not required | +| `MODEL-003` | LLM Core Provider Communication & Fallback Routing | `model` | `verified` | `E0` | pending | +| `MODEL-004` | Model Selection & Display Ordering Preferences | `model` | `verified` | `E0` | not required | +| `MODEL-005` | Side-by-Side Model Comparison (A/B Testing) | `model` | `verified` | `E0` | not required | +| `MODEL-006` | GitHub Copilot Device Flow Authentication | `model` | `verified` | `E2` | pending | +| `MODEL-007` | ChatGPT Subscription Device Flow Authentication | `model` | `verified` | `E2` | pending | +| `MODEL-008` | Embedding Model Lane & Vector Provider Setup | `model` | `verified` | `E0` | not required | +| `NOTE-001` | Interactive Notes & Checklist Management | `note` | `verified` | `E0` | not required | +| `PLATFORM-001` | Application Initialization & Lifespan Management | `platform` | `verified` | `E0` | not required | +| `PLATFORM-002` | System Health, Readiness & Version Monitoring APIs | `platform` | `verified` | `E0` | not required | +| `PLATFORM-003` | Database Schema, Migrations & SQLite Persistence | `platform` | `verified` | `E0` | not required | +| `PLATFORM-004` | User Data Export & Import Backup Infrastructure | `platform` | `verified` | `E0` | not required | +| `PLATFORM-005` | File Cleanup & Storage Maintenance Engine | `platform` | `verified` | `E0` | not required | +| `PLATFORM-006` | System Health & RAG Diagnostic Suite | `platform` | `verified` | `E0` | not required | +| `PLATFORM-007` | Desktop CLI Utilities & Shell Integration Tools | `platform` | `verified` | `E0` | not required | +| `PLATFORM-008` | Desktop Companion App Integration | `platform` | `verified` | `E0` | not required | +| `PLATFORM-009` | Docker Containerization & GPU Hardware Manifests | `platform` | `verified` | `E0` | pending | +| `PLATFORM-010` | Legacy FAISS Vector Index Migration Script | `platform` | `legacy` | `E0` | not required | +| `RESEARCH-001` | Deep Research Execution Engine & SSE Progress Streaming | `research` | `verified` | `E0` | not required | +| `RESEARCH-002` | Research Library, Detail View & Image Controls | `research` | `verified` | `E0` | not required | +| `RESEARCH-003` | Web Search Engine Integration (SearXNG & Multi-Provider) | `research` | `verified` | `E1` | pending | +| `RESEARCH-004` | Research Result Peeking & Topic Spinoff Generation | `research` | `verified` | `E0` | not required | +| `SECURITY-001` | Authentication, Session Cookies & User Management | `security` | `verified` | `E0` | not required | +| `SECURITY-002` | System Vault Encrypted Secret Storage | `security` | `verified` | `E1` | pending | +| `SECURITY-003` | API Token Management & Scope Access Control | `security` | `verified` | `E0` | not required | +| `SECURITY-004` | Prompt Security & Injection Defense Engine | `security` | `verified` | `E1` | not required | +| `SECURITY-005` | URL & Path Confinement Security Guards | `security` | `verified` | `E1` | not required | +| `SECURITY-006` | HTTP Security Headers Middleware | `security` | `verified` | `E0` | not required | +| `SECURITY-007` | Admin System Data Wipe ('Danger Zone') | `security` | `verified` | `E0` | not required | +| `SKILL-001` | Dynamic Skill Management & Code Execution Engine | `skill` | `verified` | `E0` | not required | diff --git a/docs/discovery/references/source-provenance.md b/docs/discovery/references/source-provenance.md new file mode 100644 index 000000000..c7a9a581d --- /dev/null +++ b/docs/discovery/references/source-provenance.md @@ -0,0 +1,28 @@ +# Source Provenance & Audit Baseline + +## Target Repository & Snapshot + +- **Repository**: `odysseus-dev/odysseus` +- **Branch**: `discovery` +- **Pinned Commit SHA**: `d8a2059df8e53bc7275c45339849d14c8651e73c` +- **Snapshot Date**: `2026-07-23T14:49:02Z` + +## Discovery Package Organization + +The public discovery documentation package under `docs/discovery/` is structured as follows: + +- `feature-catalog.json`: Canonical machine-readable JSON catalog containing 79 feature records. +- `feature-catalog.md`: Human-readable summary derived from `feature-catalog.json`. +- `BASELINE-STATUS.md`: Publication status, evidence-validation snapshot, and durable maintainer guidance. +- `audit-method.md`: Audit rules, scope, and evidence maturity definitions (E0 to E4). +- `domains/`: 16 functional domain markdown files detailing feature implementations. +- `references/`: Audit provenance and repository snapshot metadata. +- `tools/`: Structural, consistency, and evidence validators with focused evidence-validator tests. + +## Exclusion Principles + +This public documentation package explicitly excludes: +- Internal planning artifacts or private meeting notes. +- Machine-specific filesystem paths or user environments. +- API keys, credentials, or private service endpoints. +- Application code or automated test mutations. diff --git a/docs/discovery/tools/test_validate_discovery_evidence.py b/docs/discovery/tools/test_validate_discovery_evidence.py new file mode 100644 index 000000000..5e3144fe0 --- /dev/null +++ b/docs/discovery/tools/test_validate_discovery_evidence.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Focused negative tests for validate_discovery_evidence.py.""" + +from __future__ import annotations + +import importlib.util +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).with_name("validate_discovery_evidence.py") +SPEC = importlib.util.spec_from_file_location("validate_discovery_evidence", MODULE_PATH) +assert SPEC and SPEC.loader +validator = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = validator +SPEC.loader.exec_module(validator) + + +class FakeJavascriptParser: + supported = True + reason = "test parser" + + def __init__(self, symbols: list[validator.Located] | None = None) -> None: + self.symbols = symbols or [] + + def parse(self, path: Path) -> list[validator.Located]: + return self.symbols + + +class UnsupportedJavascriptParser: + supported = False + reason = "no repository-local parser" + + +class EvidenceNegativeTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + (self.root / "sample.py").write_text( + "from fastapi import APIRouter\n" + "router = APIRouter(prefix='/api')\n" + "\n" + "class ChatHandler:\n" + " def preprocess_message(self):\n" + " return True\n" + "\n" + "@router.post('/chat')\n" + "def chat_stream():\n" + " return True\n", + encoding="utf-8", + ) + (self.root / "sample.js").write_text( + "export const present = () => true;\n", encoding="utf-8" + ) + (self.root / "sample.sh").write_text( + "#!/usr/bin/env bash\nreal_function() {\n return 0\n}\n", + encoding="utf-8", + ) + self.backup = self.root / "copied-backups" + self.backup.mkdir() + for path in self.root.glob("sample.*"): + shutil.copy2(path, self.backup / path.name) + + def tearDown(self) -> None: + for backup in self.backup.iterdir(): + target = self.root / backup.name + shutil.copy2(backup, target) + self.assertEqual(target.read_bytes(), backup.read_bytes()) + self.temp.cleanup() + + def validate( + self, + evidence: dict[str, str], + javascript_parser: object | None = None, + ) -> validator.Validation: + return validator.validate_evidence( + self.root, + "TEST-001", + 0, + evidence, + javascript_parser or FakeJavascriptParser(), + ) + + def test_missing_python_symbol(self) -> None: + result = self.validate( + { + "path": "sample.py", + "kind": "python-function", + "locator": "fabricated", + "line_range": "L1-L1", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-locator") + + def test_incorrect_qualified_method(self) -> None: + result = self.validate( + { + "path": "sample.py", + "kind": "python-method", + "locator": "WrongHandler.preprocess_message", + "line_range": "L5-L6", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-locator") + + def test_symbol_outside_cited_range(self) -> None: + result = self.validate( + { + "path": "sample.py", + "kind": "python-method", + "locator": "ChatHandler.preprocess_message", + "line_range": "L1-L2", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "locator-outside-range") + + def test_fabricated_test_function(self) -> None: + result = self.validate( + { + "path": "sample.py", + "kind": "test-function", + "locator": "test_fabricated", + "line_range": "L1-L2", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-locator") + + def test_nonexistent_javascript_symbol_with_parser(self) -> None: + result = self.validate( + { + "path": "sample.js", + "kind": "javascript-function", + "locator": "missing", + "line_range": "L1-L1", + "explanation": "negative fixture", + }, + FakeJavascriptParser( + [validator.Located("present", "javascript-export", 1, 1)] + ), + ) + self.assertEqual(result.result, "invalid-locator") + + def test_unsupported_javascript_parser(self) -> None: + result = self.validate( + { + "path": "sample.js", + "kind": "javascript-function", + "locator": "present", + "line_range": "L1-L1", + "explanation": "negative fixture", + }, + UnsupportedJavascriptParser(), + ) + self.assertEqual(result.result, "unsupported") + + def test_route_path_mismatch(self) -> None: + result = self.validate( + { + "path": "sample.py", + "kind": "python-route", + "locator": "POST /api/wrong -> chat_stream", + "line_range": "L9-L10", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-locator") + self.assertIn("path", result.problem or "") + + def test_http_method_mismatch(self) -> None: + result = self.validate( + { + "path": "sample.py", + "kind": "python-route", + "locator": "GET /api/chat -> chat_stream", + "line_range": "L9-L10", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-locator") + self.assertIn("method", result.problem or "") + + def test_shell_function_mismatch(self) -> None: + result = self.validate( + { + "path": "sample.sh", + "kind": "shell-function", + "locator": "fabricated", + "line_range": "L1-L4", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-locator") + + def test_invalid_file_level_evidence(self) -> None: + result = self.validate( + { + "path": "missing.file", + "kind": "file", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-path") + + def test_file_level_evidence_rejects_fake_symbol(self) -> None: + result = self.validate( + { + "path": "sample.sh", + "kind": "file", + "locator": "whole-script", + "explanation": "negative fixture", + } + ) + self.assertEqual(result.result, "invalid-locator") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/discovery/tools/validate_discovery_consistency.py b/docs/discovery/tools/validate_discovery_consistency.py new file mode 100755 index 000000000..953b093e1 --- /dev/null +++ b/docs/discovery/tools/validate_discovery_consistency.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import re +import sys +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +CATALOG_PATH = ROOT / "feature-catalog.json" +CATALOG_MD_PATH = ROOT / "feature-catalog.md" +DOMAINS_DIR = ROOT / "domains" +REVIEWS_DIR = ROOT / "reviews" + +EXPECTED_COMMIT = "d8a2059df8e53bc7275c45339849d14c8651e73c" +EXPECTED_FEATURES = 79 +EXPECTED_DOMAINS = 16 + +VALID_STATUSES = { + "verified", + "partial", + "disabled", + "experimental", + "legacy", + "dead-code-candidate", + "unverified", +} + +VALID_MATURITY = {"E0", "E1", "E2", "E3", "E4"} + +VALID_RUNTIME = { + "not-required", + "pending", + "blocked", + "completed", +} + +REQUIRED_FIELDS = { + "id", + "domain", + "name", + "purpose", + "status", + "evidence_maturity", + "verified_at_commit", + "evidence", + "runtime_validation", +} + +FEATURE_ID_RE = re.compile(r"^[A-Z][A-Z0-9]*-\d{3}$") + +DOMAIN_HEADING_RE = re.compile( + r"^##\s+`?([A-Z][A-Z0-9]*-\d{3})`?" + r"\s+(?:—|-)\s+(.+?)\s*$" +) + +DOMAIN_FIELD_RE = re.compile( + r"^-\s+\*\*" + r"(Domain|Status|Evidence Maturity|Commit Verified)" + r"\*\*:\s*(.*?)\s*$" +) + +DOMAIN_RUNTIME_RE = re.compile( + r"^-\s+\*\*Runtime Validation\*\*:\s*(.*?)\s*$" +) + +REVIEW_HEADING_RE = re.compile( + r"^###\s+(?:\d+\.\s+)?" + r"([A-Z][A-Z0-9]*-\d{3})" + r"\s+(?:—|-)\s+.+$" +) + +REVIEW_RESULT_RE = re.compile( + r"^-\s+\*\*Resulting Status(?:\s+and|/)\s+Maturity\*\*:" + r"\s*`([^`]+)`\s*/\s*`([^`]+)`\s*$" +) + + +def clean(value: str) -> str: + value = value.strip() + + if ( + len(value) >= 2 + and value.startswith("`") + and value.endswith("`") + ): + return value[1:-1].strip() + + return value + + +def load_catalog(errors: list[str]) -> list[dict[str, Any]]: + try: + data = json.loads( + CATALOG_PATH.read_text(encoding="utf-8") + ) + except Exception as exc: + errors.append(f"Unable to read catalog: {exc}") + return [] + + if isinstance(data, list): + features = data + elif isinstance(data, dict) and isinstance(data.get("features"), list): + features = data["features"] + else: + errors.append( + "Catalog must be an array or contain a features array" + ) + return [] + + if not all(isinstance(feature, dict) for feature in features): + errors.append("Every catalog feature must be an object") + return [] + + return features + + +def validate_catalog( + features: list[dict[str, Any]], + errors: list[str], +) -> None: + if len(features) != EXPECTED_FEATURES: + errors.append( + f"Expected {EXPECTED_FEATURES} features, " + f"found {len(features)}" + ) + + ids = [feature.get("id") for feature in features] + + duplicates = sorted( + feature_id + for feature_id, count in Counter(ids).items() + if feature_id and count > 1 + ) + + if duplicates: + errors.append( + "Duplicate feature IDs: " + ", ".join(duplicates) + ) + + for index, feature in enumerate(features): + feature_id = feature.get("id") + label = ( + feature_id + if isinstance(feature_id, str) + else f"" + ) + + missing = sorted( + field + for field in REQUIRED_FIELDS + if feature.get(field) in (None, "", []) + ) + + if missing: + errors.append( + f"{label}: missing fields: {', '.join(missing)}" + ) + + if ( + not isinstance(feature_id, str) + or not FEATURE_ID_RE.fullmatch(feature_id) + ): + errors.append(f"{label}: invalid feature ID") + + if feature.get("status") not in VALID_STATUSES: + errors.append( + f"{label}: invalid status " + f"{feature.get('status')!r}" + ) + + if feature.get("evidence_maturity") not in VALID_MATURITY: + errors.append( + f"{label}: invalid maturity " + f"{feature.get('evidence_maturity')!r}" + ) + + if feature.get("verified_at_commit") != EXPECTED_COMMIT: + errors.append( + f"{label}: incorrect verified_at_commit" + ) + + runtime = feature.get("runtime_validation") + + if not isinstance(runtime, dict): + errors.append( + f"{label}: runtime_validation must be an object" + ) + continue + + required = runtime.get("required") + runtime_status = runtime.get("status") + reason = runtime.get("reason") + + if not isinstance(required, bool): + errors.append( + f"{label}: runtime required must be boolean" + ) + + if runtime_status not in VALID_RUNTIME: + errors.append( + f"{label}: invalid runtime status " + f"{runtime_status!r}" + ) + + if not isinstance(reason, str) or not reason.strip(): + errors.append( + f"{label}: runtime reason is blank" + ) + + if required is False and runtime_status != "not-required": + errors.append( + f"{label}: required=false requires not-required" + ) + + if required is True and runtime_status == "not-required": + errors.append( + f"{label}: required=true cannot be not-required" + ) + + +def parse_domain( + path: Path, + errors: list[str], +) -> dict[str, dict[str, str]]: + records: dict[str, dict[str, str]] = {} + current_id: str | None = None + + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), + start=1, + ): + heading = DOMAIN_HEADING_RE.match(line) + + if heading: + current_id = heading.group(1) + + if current_id in records: + errors.append( + f"{path.relative_to(ROOT)}:{line_number}: " + f"duplicate heading {current_id}" + ) + + records[current_id] = { + "Name": heading.group(2).strip(), + } + continue + + field = DOMAIN_FIELD_RE.match(line) + + if field and current_id: + value = clean(field.group(2)) + + if not value: + errors.append( + f"{path.relative_to(ROOT)}:{line_number}: " + f"blank {field.group(1)}" + ) + + records[current_id][field.group(1)] = value + continue + + runtime = DOMAIN_RUNTIME_RE.match(line) + + if runtime and current_id: + value = clean(runtime.group(1)) + runtime_status = re.split( + r"\s+(?:—|-)\s+", + value, + maxsplit=1, + )[0] + records[current_id]["Runtime Validation"] = ( + runtime_status.strip("` ") + ) + + return records + + +def validate_domains( + features: list[dict[str, Any]], + errors: list[str], +) -> None: + catalog = { + feature["id"]: feature + for feature in features + if feature.get("id") + } + + expected_by_domain: defaultdict[str, set[str]] = defaultdict(set) + + for feature in features: + expected_by_domain[feature["domain"]].add(feature["id"]) + + paths = sorted(DOMAINS_DIR.glob("*.md")) + + if len(paths) != EXPECTED_DOMAINS: + errors.append( + f"Expected {EXPECTED_DOMAINS} domain files, " + f"found {len(paths)}" + ) + + all_found: set[str] = set() + + for path in paths: + domain = path.stem + records = parse_domain(path, errors) + found = set(records) + expected = expected_by_domain.get(domain, set()) + all_found.update(found) + + if found != expected: + missing = sorted(expected - found) + extra = sorted(found - expected) + + errors.append( + f"{domain}: missing={missing}, unexpected={extra}" + ) + + for feature_id, record in records.items(): + feature = catalog.get(feature_id) + + if feature is None: + continue + + expected_values = { + "Name": feature["name"], + "Domain": feature["domain"], + "Status": feature["status"], + "Evidence Maturity": feature["evidence_maturity"], + "Commit Verified": feature["verified_at_commit"], + "Runtime Validation": ( + feature["runtime_validation"]["status"] + ), + } + + for field, expected_value in expected_values.items(): + actual = record.get(field) + + if actual != expected_value: + errors.append( + f"{path.relative_to(ROOT)}: " + f"{feature_id} {field}: " + f"{actual!r} != {expected_value!r}" + ) + + if all_found != set(catalog): + errors.append( + "Domain Markdown IDs do not match catalog JSON" + ) + + +def validate_catalog_markdown( + features: list[dict[str, Any]], + errors: list[str], +) -> None: + expected = { + feature["id"]: feature + for feature in features + } + found: dict[str, list[str]] = {} + + for line in CATALOG_MD_PATH.read_text( + encoding="utf-8" + ).splitlines(): + if not line.startswith("|"): + continue + + cells = [ + cell.strip() + for cell in line.strip().strip("|").split("|") + ] + + if len(cells) < 6: + continue + + feature_id = clean(cells[0]) + + if FEATURE_ID_RE.fullmatch(feature_id): + found[feature_id] = cells + + if set(found) != set(expected): + errors.append( + "feature-catalog.md IDs do not match JSON" + ) + + for feature_id, cells in found.items(): + feature = expected[feature_id] + runtime = feature["runtime_validation"] + runtime_display = ( + runtime["status"] + if runtime["required"] + else "not required" + ) + + actual = { + "name": cells[1].replace("\\|", "|"), + "domain": clean(cells[2]), + "status": clean(cells[3]), + "maturity": clean(cells[4]), + "runtime": clean(cells[5]), + } + + wanted = { + "name": feature["name"], + "domain": feature["domain"], + "status": feature["status"], + "maturity": feature["evidence_maturity"], + "runtime": runtime_display, + } + + for field, expected_value in wanted.items(): + if actual[field] != expected_value: + errors.append( + f"feature-catalog.md: {feature_id} " + f"{field}: {actual[field]!r} " + f"!= {expected_value!r}" + ) + + +def validate_reviews( + features: list[dict[str, Any]], + errors: list[str], +) -> int: + catalog = { + feature["id"]: feature + for feature in features + } + checked = 0 + + for path in sorted( + REVIEWS_DIR.glob("evidence-sample-*.md") + ): + current_id: str | None = None + results: set[str] = set() + + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), + start=1, + ): + heading = REVIEW_HEADING_RE.match(line) + + if heading: + current_id = heading.group(1) + continue + + result = REVIEW_RESULT_RE.match(line) + + if not result or current_id is None: + continue + + status, maturity = result.groups() + results.add(current_id) + checked += 1 + + feature = catalog.get(current_id) + + if feature is None: + errors.append( + f"{path.relative_to(ROOT)}:{line_number}: " + f"unknown feature {current_id}" + ) + continue + + if feature["status"] != status: + errors.append( + f"{current_id}: review status {status!r} " + f"!= catalog {feature['status']!r}" + ) + + if feature["evidence_maturity"] != maturity: + errors.append( + f"{current_id}: review maturity {maturity!r} " + f"!= catalog " + f"{feature['evidence_maturity']!r}" + ) + + if path.name == "evidence-sample-01.md" and len(results) != 12: + errors.append( + f"{path.relative_to(ROOT)}: expected 12 " + f"review results, found {len(results)}" + ) + + return checked + + +def validate_whitespace(errors: list[str]) -> None: + for path in sorted(ROOT.rglob("*")): + if not path.is_file(): + continue + + if path.suffix not in {".md", ".json", ".py", ".txt"}: + continue + + for line_number, line in enumerate( + path.read_text( + encoding="utf-8", + errors="replace", + ).splitlines(), + start=1, + ): + if line != line.rstrip(" \t"): + errors.append( + f"{path.relative_to(ROOT)}:{line_number}: " + "trailing whitespace" + ) + + if any(ROOT.rglob("*.pyc")): + errors.append("Generated .pyc files exist") + + if any( + path.is_dir() + for path in ROOT.rglob("__pycache__") + ): + errors.append("__pycache__ exists") + + +def main() -> int: + errors: list[str] = [] + features = load_catalog(errors) + reviewed = 0 + + if features: + validate_catalog(features, errors) + validate_domains(features, errors) + validate_catalog_markdown(features, errors) + reviewed = validate_reviews(features, errors) + + validate_whitespace(errors) + + print(f"Catalog Features: {len(features)}") + print( + "Unique Feature IDs:", + len({feature.get("id") for feature in features}), + ) + print( + "Domain Files:", + len(list(DOMAINS_DIR.glob("*.md"))), + ) + + if features: + print( + "Statuses:", + dict( + Counter( + feature.get("status") + for feature in features + ) + ), + ) + print( + "Evidence Maturity:", + dict( + Counter( + feature.get("evidence_maturity") + for feature in features + ) + ), + ) + print( + "Runtime Validation:", + dict( + Counter( + ( + feature.get( + "runtime_validation", + {}, + ).get("required"), + feature.get( + "runtime_validation", + {}, + ).get("status"), + ) + for feature in features + ) + ), + ) + + print(f"Review Results Checked: {reviewed}") + print(f"Consistency Errors: {len(errors)}") + + for error in errors: + print(f"ERROR: {error}") + + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/discovery/tools/validate_discovery_docs.py b/docs/discovery/tools/validate_discovery_docs.py new file mode 100755 index 000000000..656b800ec --- /dev/null +++ b/docs/discovery/tools/validate_discovery_docs.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Validate the Odysseus public discovery documentation package.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPO_ROOT = ROOT.parents[1] +CATALOG_PATH = ROOT / "feature-catalog.json" +DOMAINS_DIR = ROOT / "domains" + +EXPECTED_COMMIT = "d8a2059df8e53bc7275c45339849d14c8651e73c" +ALLOWED_STATUSES = { + "verified", "partial", "disabled", "experimental", "legacy", + "dead-code-candidate", "unverified" +} + +LINE_RANGE_RE = re.compile(r"^L([1-9]\d*)-L([1-9]\d*)$") +FEATURE_HEADING_RE = re.compile( + r"^##\s+`?([A-Z][A-Z0-9]*-\d{3})`?\s+(?:—|-)\s+.+$", + re.MULTILINE, +) +LINK_RE = re.compile(r"(? argparse.Namespace: + parser = argparse.ArgumentParser(description="Validate Odysseus discovery docs.") + parser.add_argument( + "--repo-root", + type=Path, + default=REPO_ROOT, + help="Path to Odysseus repository root.", + ) + return parser.parse_args() + +def validate() -> int: + args = parse_args() + repo_root = args.repo_root.resolve() + errors: list[str] = [] + + # 1. Validate Catalog JSON existence and content + if not CATALOG_PATH.is_file(): + errors.append(f"Missing catalog file: {CATALOG_PATH}") + print(f"Errors: {len(errors)}") + for e in errors: + print(f"ERROR: {e}") + return 1 + + try: + catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8")) + except Exception as e: + errors.append(f"Failed to parse catalog JSON: {e}") + print(f"Errors: {len(errors)}") + for err in errors: + print(f"ERROR: {err}") + return 1 + + if not isinstance(catalog, list): + errors.append("feature-catalog.json must be a JSON array") + return 1 + + feature_ids = [item.get("id") for item in catalog if isinstance(item, dict)] + if len(feature_ids) != 79: + errors.append(f"Expected 79 unique feature IDs, found {len(feature_ids)}") + if len(feature_ids) != len(set(feature_ids)): + errors.append("Duplicate feature IDs found in catalog JSON") + + # Domain catalog counts + catalog_domain_counts: dict[str, int] = {} + for item in catalog: + if not isinstance(item, dict): + errors.append("Catalog item is not an object") + continue + fid = item.get("id", "") + status = item.get("status") + domain = item.get("domain", "").lower() + catalog_domain_counts[domain] = catalog_domain_counts.get(domain, 0) + 1 + + if status not in ALLOWED_STATUSES: + errors.append(f"{fid}: Invalid status '{status}'") + + evidence_list = item.get("evidence") + if not isinstance(evidence_list, list) or not evidence_list: + errors.append(f"{fid}: Missing or empty evidence list") + continue + + for ev in evidence_list: + if not isinstance(ev, dict): + errors.append(f"{fid}: Evidence item is not an object") + continue + path_str = ev.get("path") + lr_str = str(ev.get("line_range", "")) + if not path_str or Path(path_str).is_absolute() or ".." in Path(path_str).parts: + errors.append(f"{fid}: Unsafe or invalid path '{path_str}'") + continue + + # Check path exists in repo + target_path = repo_root / path_str + if not target_path.is_file(): + errors.append(f"{fid}: Referenced path '{path_str}' does not exist on disk") + continue + + # Check line range format & bounds + m = LINE_RANGE_RE.fullmatch(lr_str) + if not m: + errors.append(f"{fid}: Invalid line range format '{lr_str}' for path '{path_str}'") + continue + + start, end = int(m.group(1)), int(m.group(2)) + lines_cnt = len(target_path.read_text(encoding="utf-8", errors="ignore").splitlines()) + if start > end or end > lines_cnt or start < 1: + errors.append( + f"{fid}: Line range '{lr_str}' exceeds file length ({lines_cnt} lines) in '{path_str}'" + ) + + # 2. Check Domain Markdown files + md_feature_ids: list[str] = [] + domain_files = sorted(DOMAINS_DIR.glob("*.md")) + for df in domain_files: + domain_name = df.stem.lower() + content = df.read_text(encoding="utf-8") + found_ids = FEATURE_HEADING_RE.findall(content) + md_feature_ids.extend(found_ids) + if len(found_ids) != catalog_domain_counts.get(domain_name, 0): + errors.append( + f"Domain '{domain_name}' count mismatch: catalog has {catalog_domain_counts.get(domain_name, 0)}, Markdown has {len(found_ids)}" + ) + + if sorted(md_feature_ids) != sorted(feature_ids): + errors.append("Markdown domain feature IDs do not match catalog JSON feature IDs") + + # 3. Check for forbidden/private terms, sensitive credentials, and broken links across all docs + for md_file in ROOT.rglob("*.md"): + rel_md = md_file.relative_to(ROOT) + content = md_file.read_text(encoding="utf-8") + + # Forbidden terms scan + forbidden_matches = FORBIDDEN_TERMS_RE.findall(content) + if forbidden_matches: + errors.append( + f"{rel_md}: Found forbidden/internal terms: {set(forbidden_matches)}" + ) + + # Broken local link check + for target in LINK_RE.findall(content): + target = target.strip().strip("<>") + if not target or target.startswith(("#", "http://", "https://", "mailto:")): + continue + target_path = target.split("#", 1)[0] + resolved = (md_file.parent / target_path).resolve() + if not resolved.exists(): + errors.append(f"{rel_md}: Broken local link '{target}'") + + print(f"Catalog Features: {len(catalog)}") + print(f"Domain Files: {len(domain_files)}") + print(f"Validation Errors: {len(errors)}") + for err in errors: + print(f"ERROR: {err}") + + return 1 if errors else 0 + +if __name__ == "__main__": + sys.exit(validate()) diff --git a/docs/discovery/tools/validate_discovery_evidence.py b/docs/discovery/tools/validate_discovery_evidence.py new file mode 100644 index 000000000..4315478f7 --- /dev/null +++ b/docs/discovery/tools/validate_discovery_evidence.py @@ -0,0 +1,1413 @@ +#!/usr/bin/env python3 +"""Validate feature-catalog evidence locators against repository source. + +Evidence identity is resolved from a language parser where one is available. +The cited line range is treated as generated display metadata and checked only +after the locator has been resolved. +""" + +from __future__ import annotations + +import argparse +import ast +import difflib +import json +import re +import subprocess +import sys +from collections import Counter, defaultdict +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable + + +DISCOVERY_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_REPO_ROOT = DISCOVERY_ROOT.parents[1] +DEFAULT_CATALOG = DISCOVERY_ROOT / "feature-catalog.json" + +RESULTS = { + "valid", + "invalid-path", + "invalid-locator", + "locator-outside-range", + "range-mismatch", + "ambiguous", + "unsupported", + "file-level-valid", +} + +PYTHON_KINDS = { + "python-function", + "python-method", + "python-class", + "python-variable", + "python-route", + "python-module", + "test-function", +} +JAVASCRIPT_KINDS = { + "javascript-function", + "javascript-class", + "javascript-export", + "javascript-event", +} +FILE_EXTENSIONS = { + ".html", + ".md", + ".swift", + ".yaml", + ".yml", +} +RANGE_RE = re.compile(r"^L([1-9]\d*)-L([1-9]\d*)$") +LEGACY_ROUTE_RE = re.compile( + r"^@(?P[A-Za-z_]\w*)\." + r"(?Pget|post|put|patch|delete|options|head)" + r"\(\s*['\"](?P[^'\"]*)['\"]\s*\)$", + re.IGNORECASE, +) +CANONICAL_ROUTE_RE = re.compile( + r"^(?PGET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\s+" + r"(?P\S+)(?:\s+->\s+(?P[\w.]+))?$" +) +SHELL_FUNCTION_RE = re.compile( + r"^\s*(?:function\s+)?(?P[A-Za-z_]\w*)\s*" + r"(?:\(\s*\))?\s*\{" +) +MARKDOWN_HEADING_RE = re.compile(r"^(?P#{1,6})\s+(?P.+?)\s*$") + + +@dataclass(frozen=True) +class Located: + name: str + kind: str + start: int + end: int + details: dict[str, Any] | None = None + + @property + def line_range(self) -> str: + return f"L{self.start}-L{self.end}" + + +@dataclass +class Validation: + feature_id: str + evidence_index: int + current_evidence: dict[str, Any] + inferred_kind: str + locator: str | None + result: str + problem: str | None = None + resolved: Located | None = None + suggested_kind: str | None = None + suggested_locator: str | None = None + generated_line_range: str | None = None + confidence: str = "high" + manual_review_reason: str | None = None + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["resolved"] = asdict(self.resolved) if self.resolved else None + return data + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate discovery evidence identities and line ranges." + ) + parser.add_argument("--repo-root", type=Path, default=DEFAULT_REPO_ROOT) + parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + parser.add_argument( + "--output-dir", + type=Path, + default=DISCOVERY_ROOT, + help="Directory for the report and repair queue.", + ) + return parser.parse_args(argv) + + +def load_catalog(path: Path) -> list[dict[str, Any]]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, list) or not all(isinstance(item, dict) for item in data): + raise ValueError("catalog must be an array of feature objects") + return data + + +def safe_target(repo_root: Path, raw_path: Any) -> Path | None: + if not isinstance(raw_path, str) or not raw_path: + return None + candidate = Path(raw_path) + if candidate.is_absolute() or ".." in candidate.parts: + return None + resolved = (repo_root / candidate).resolve() + try: + resolved.relative_to(repo_root.resolve()) + except ValueError: + return None + return resolved + + +def parse_range(raw: Any) -> tuple[int, int] | None: + if raw in (None, ""): + return None + match = RANGE_RE.fullmatch(str(raw)) + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def extension_for(path: str) -> str: + suffix = Path(path).suffix.lower() + return suffix or "[none]" + + +def legacy_locator(evidence: dict[str, Any]) -> str | None: + value = evidence.get("locator", evidence.get("symbol")) + return value if isinstance(value, str) and value.strip() else None + + +def infer_kind(evidence: dict[str, Any], target: Path | None) -> str: + explicit = evidence.get("kind") + if isinstance(explicit, str) and explicit: + return explicit + + path = str(evidence.get("path", "")) + suffix = Path(path).suffix.lower() + locator = legacy_locator(evidence) or "" + + if suffix == ".py": + if LEGACY_ROUTE_RE.fullmatch(locator) or CANONICAL_ROUTE_RE.fullmatch(locator): + return "python-route" + if path.startswith("tests/") and locator.startswith("test"): + return "test-function" + if "." in locator and all(part.isidentifier() for part in locator.split(".")): + return "python-method" + if locator.isupper(): + return "python-variable" + if target and target.is_file(): + try: + tree = ast.parse(target.read_text(encoding="utf-8")) + except (OSError, SyntaxError, UnicodeError): + return "manual-review" + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == locator: + return "python-class" + return "python-function" + if suffix == ".js": + if locator.startswith("on:") or locator.startswith("event:"): + return "javascript-event" + if locator and locator[:1].isupper(): + return "javascript-class" + return "javascript-function" + if suffix in {".sh", ".bash"}: + return "shell-function" + if suffix == ".md": + return "documentation-section" + if suffix in {".json", ".yaml", ".yml", ".toml", ".ini"}: + return "configuration-key" + if suffix in FILE_EXTENSIONS or Path(path).name in {"Dockerfile"}: + return "manual-review" + if not suffix: + return "manual-review" + return "manual-review" + + +class PythonIndex(ast.NodeVisitor): + """Collect Python declarations, assignments, and decorated routes.""" + + def __init__(self) -> None: + self.symbols: list[Located] = [] + self.routes: list[Located] = [] + self.scope: list[str] = [] + self.constants: dict[str, str] = {} + self.router_prefixes: dict[str, str | None] = {} + + @classmethod + def from_path(cls, path: Path) -> "PythonIndex": + index = cls() + tree = ast.parse(path.read_text(encoding="utf-8")) + index._collect_static_values(tree) + index.visit(tree) + return index + + def _collect_static_values(self, tree: ast.AST) -> None: + for node in getattr(tree, "body", []): + if isinstance(node, (ast.Assign, ast.AnnAssign)): + value = self._string_value(node.value) + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + if isinstance(target, ast.Name) and value is not None: + self.constants[target.id] = value + if isinstance(target, ast.Name) and isinstance( + node.value, ast.Call + ): + call_name = self._call_name(node.value.func) + if call_name.endswith("APIRouter"): + prefix = self._router_prefix(node.value) + self.router_prefixes[target.id] = prefix + + def _qualified(self, name: str) -> str: + return ".".join([*self.scope, name]) + + @staticmethod + def _call_name(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return f"{PythonIndex._call_name(node.value)}.{node.attr}".strip(".") + return "" + + def _string_value(self, node: ast.AST | None) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.Name): + return self.constants.get(node.id) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = self._string_value(node.left) + right = self._string_value(node.right) + return left + right if left is not None and right is not None else None + return None + + def _keyword_string(self, call: ast.Call, name: str) -> str | None: + for keyword in call.keywords: + if keyword.arg == name: + return self._string_value(keyword.value) + return None + + def _router_prefix(self, call: ast.Call) -> str | None: + if not any(keyword.arg == "prefix" for keyword in call.keywords): + return "" + return self._keyword_string(call, "prefix") + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self.symbols.append( + Located( + self._qualified(node.name), + "python-class", + node.lineno, + node.end_lineno or node.lineno, + ) + ) + self.scope.append(node.name) + self.generic_visit(node) + self.scope.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self._visit_function(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self._visit_function(node) + + def _visit_function(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + qualified = self._qualified(node.name) + parent_is_class = bool( + self.scope + and any( + item.kind == "python-class" and item.name == ".".join(self.scope) + for item in self.symbols + ) + ) + kind = ( + "test-function" + if node.name.startswith("test") + else "python-method" + if parent_is_class + else "python-function" + ) + located = Located( + qualified, kind, node.lineno, node.end_lineno or node.lineno + ) + self.symbols.append(located) + self._collect_routes(node, qualified) + self.scope.append(node.name) + self.generic_visit(node) + self.scope.pop() + + def _collect_routes( + self, node: ast.FunctionDef | ast.AsyncFunctionDef, qualified: str + ) -> None: + methods = {"get", "post", "put", "patch", "delete", "options", "head"} + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Call) or not isinstance( + decorator.func, ast.Attribute + ): + continue + method = decorator.func.attr.lower() + if method not in methods: + continue + router = self._call_name(decorator.func.value) + local_path = ( + self._string_value(decorator.args[0]) if decorator.args else None + ) + if local_path is None: + continue + prefix = self.router_prefixes.get(router) + full_path = None if router in self.router_prefixes and prefix is None else ( + f"{prefix or ''}{local_path}" + ) + details = { + "method": method.upper(), + "path": local_path, + "full_path": full_path, + "function": qualified, + "router": router, + "router_prefix": prefix, + "prefix_resolved": full_path is not None, + } + route_name = f"{method.upper()} {full_path or local_path} -> {qualified}" + self.routes.append( + Located( + route_name, + "python-route", + node.lineno, + node.end_lineno or node.lineno, + details, + ) + ) + + def visit_Assign(self, node: ast.Assign) -> None: + if isinstance(node.value, ast.Call): + call_name = self._call_name(node.value.func) + if call_name.endswith("APIRouter"): + for target in node.targets: + if isinstance(target, ast.Name): + self.router_prefixes[target.id] = self._router_prefix(node.value) + for target in node.targets: + self._collect_assignment(target, node) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + self._collect_assignment(node.target, node) + self.generic_visit(node) + + def _collect_assignment(self, target: ast.AST, node: ast.AST) -> None: + if isinstance(target, (ast.Tuple, ast.List)): + for item in target.elts: + self._collect_assignment(item, node) + return + if not isinstance(target, ast.Name): + return + self.symbols.append( + Located( + self._qualified(target.id), + "python-variable", + node.lineno, + node.end_lineno or node.lineno, + ) + ) + + +NODE_PARSER = r""" +const fs = require("fs"); +const parserName = process.argv[1]; +const parserEntry = process.argv[2]; +const filename = process.argv[3]; +const source = fs.readFileSync(filename, "utf8"); +const parser = require(parserEntry); +let tree; +if (parserName === "@babel/parser") { + tree = parser.parse(source, { + sourceType: "unambiguous", + plugins: ["jsx", "classProperties", "optionalChaining", "topLevelAwait"] + }); +} else if (parserName === "typescript") { + throw new Error("typescript parser normalization is not implemented"); +} else { + const parse = parser.parse || parser.Parser?.parse; + tree = parse.call(parser, source, { + ecmaVersion: "latest", sourceType: "module", locations: true + }); +} +const out = []; +function loc(node) { + const l = node.loc; + return {start: l.start.line, end: l.end.line}; +} +function add(name, kind, node, extra={}) { + if (!name || !node.loc) return; + out.push({name, kind, ...loc(node), details: extra}); +} +function walk(node, scope=[], exported=false) { + if (!node || typeof node !== "object") return; + if (node.type === "ExportNamedDeclaration" || node.type === "ExportDefaultDeclaration") { + if (node.declaration) walk(node.declaration, scope, true); + return; + } + if (node.type === "FunctionDeclaration") { + const name = node.id?.name; + add([...scope, name].filter(Boolean).join("."), exported ? "javascript-export" : "javascript-function", node); + scope = [...scope, name].filter(Boolean); + } else if (node.type === "ClassDeclaration") { + const name = node.id?.name; + add([...scope, name].filter(Boolean).join("."), exported ? "javascript-export" : "javascript-class", node); + scope = [...scope, name].filter(Boolean); + } else if (node.type === "MethodDefinition" || node.type === "ClassMethod") { + const name = node.key?.name || node.key?.value; + add([...scope, name].filter(Boolean).join("."), "javascript-function", node); + scope = [...scope, name].filter(Boolean); + } else if (node.type === "VariableDeclarator" && node.id?.type === "Identifier") { + const initType = node.init?.type || ""; + const isFunction = initType === "ArrowFunctionExpression" || initType === "FunctionExpression"; + if (isFunction || exported) { + add([...scope, node.id.name].join("."), exported ? "javascript-export" : "javascript-function", node); + } + } else if (node.type === "CallExpression" && node.callee?.type === "MemberExpression" + && node.callee.property?.name === "addEventListener" + && ["Literal", "StringLiteral"].includes(node.arguments?.[0]?.type)) { + add("event:" + node.arguments[0].value, "javascript-event", node, {event: node.arguments[0].value}); + } + for (const [key, value] of Object.entries(node)) { + if (key === "loc" || key === "start" || key === "end") continue; + if (Array.isArray(value)) value.forEach(child => walk(child, scope, exported)); + else if (value && typeof value === "object") walk(value, scope, exported); + } +} +walk(tree.program || tree); +process.stdout.write(JSON.stringify(out)); +""" + + +class JavascriptParser: + CANDIDATES = ("acorn", "espree", "@babel/parser", "typescript") + + def __init__(self, repo_root: Path) -> None: + self.repo_root = repo_root.resolve() + self.node = self._node_binary() + self.parser_name, self.parser_entry, self.reason = self._find_parser() + + @staticmethod + def _node_binary() -> str | None: + from shutil import which + + return which("node") + + def _package_roots(self) -> list[tuple[Path, set[str]]]: + roots: list[tuple[Path, set[str]]] = [] + for package in self.repo_root.rglob("package.json"): + if "node_modules" in package.parts: + continue + try: + data = json.loads(package.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + declared: set[str] = set() + for key in ("dependencies", "devDependencies", "peerDependencies"): + values = data.get(key) + if isinstance(values, dict): + declared.update(values) + roots.append((package.parent, declared)) + return roots + + def _find_parser(self) -> tuple[str | None, str | None, str]: + if not self.node: + return None, None, "Node.js is unavailable" + package_roots = self._package_roots() + for name in self.CANDIDATES: + for package_root, declared in package_roots: + local = package_root / "node_modules" / name + if name not in declared and not local.exists(): + continue + probe = subprocess.run( + [ + self.node, + "-e", + ( + "const p=require.resolve(process.argv[1]," + "{paths:[process.argv[2]]});process.stdout.write(p)" + ), + name, + str(package_root), + ], + cwd=package_root, + text=True, + capture_output=True, + check=False, + ) + if probe.returncode: + continue + resolved = Path(probe.stdout).resolve() + try: + resolved.relative_to(self.repo_root) + except ValueError: + continue + if name == "typescript": + return None, None, ( + "TypeScript is repository-local but its AST " + "normalization is not implemented" + ) + return name, str(resolved), f"repository-local parser: {name}" + return None, None, ( + "no repository-local Acorn, Espree, Babel parser, or supported " + "TypeScript parser is available" + ) + + @property + def supported(self) -> bool: + return bool(self.node and self.parser_name) + + def parse(self, path: Path) -> list[Located]: + if not self.supported: + raise RuntimeError(self.reason) + process = subprocess.run( + [ + self.node or "node", + "-e", + NODE_PARSER, + self.parser_name or "", + self.parser_entry or "", + str(path), + ], + cwd=self.repo_root, + text=True, + capture_output=True, + check=False, + ) + if process.returncode: + raise RuntimeError(process.stderr.strip() or "JavaScript parser failed") + return [Located(**item) for item in json.loads(process.stdout)] + + +def choose_symbol( + symbols: Iterable[Located], locator: str +) -> tuple[Located | None, list[Located]]: + candidates = list(symbols) + exact = [item for item in candidates if item.name == locator] + if not exact and "." not in locator: + exact = [item for item in candidates if item.name.split(".")[-1] == locator] + return (exact[0] if len(exact) == 1 else None), exact + + +def apply_range_result( + validation: Validation, located: Located, cited: tuple[int, int] | None +) -> Validation: + validation.resolved = located + validation.generated_line_range = located.line_range + validation.suggested_locator = located.name + validation.suggested_kind = located.kind + if cited is None: + if validation.current_evidence.get("line_range") not in (None, ""): + validation.result = "range-mismatch" + validation.problem = "line_range is malformed" + else: + validation.result = "valid" + return validation + start, end = cited + if start <= located.start <= end: + # Evidence ranges are excerpts. They must contain the declaration that + # gives the locator its identity, but need not contain the whole body. + validation.result = "valid" + else: + validation.result = "locator-outside-range" + validation.problem = ( + f"locator declaration at L{located.start} is outside cited " + f"L{start}-L{end}; parser-derived span is {located.line_range}" + ) + return validation + + +def suggest_nearby( + symbols: list[Located], locator: str, cited: tuple[int, int] | None +) -> tuple[Located | None, str]: + if cited: + overlapping = [ + item + for item in symbols + if cited[0] <= item.start <= cited[1] + or item.start <= cited[0] <= item.end + ] + if len(overlapping) == 1: + return overlapping[0], "high" + leaf_matches = [ + item + for item in overlapping + if item.name.split(".")[-1] == locator.split(".")[-1] + ] + if len(leaf_matches) == 1: + return leaf_matches[0], "high" + names = [item.name for item in symbols] + close = difflib.get_close_matches(locator, names, n=1, cutoff=0.72) + if close: + return next(item for item in symbols if item.name == close[0]), "medium" + return None, "low" + + +def validate_python( + validation: Validation, target: Path, cited: tuple[int, int] | None +) -> Validation: + try: + index = PythonIndex.from_path(target) + except (OSError, UnicodeError, SyntaxError) as exc: + validation.result = "unsupported" + validation.problem = f"Python AST parsing failed: {exc}" + validation.manual_review_reason = validation.problem + return validation + + locator = validation.locator or "" + if validation.inferred_kind == "python-module": + if locator: + validation.result = "invalid-locator" + validation.problem = "python-module evidence must not name a symbol" + else: + validation.result = "file-level-valid" + validation.suggested_kind = "python-module" + return validation + + if validation.inferred_kind == "python-route": + return validate_route(validation, index.routes, cited) + + located, exact = choose_symbol(index.symbols, locator) + if len(exact) > 1: + validation.result = "ambiguous" + validation.problem = f"locator matches {len(exact)} declarations" + validation.manual_review_reason = validation.problem + validation.confidence = "low" + return validation + if not located: + validation.result = "invalid-locator" + validation.problem = f"Python locator {locator!r} does not exist" + suggestion, confidence = suggest_nearby(index.symbols, locator, cited) + validation.confidence = confidence + if suggestion: + validation.suggested_kind = suggestion.kind + validation.suggested_locator = suggestion.name + validation.generated_line_range = suggestion.line_range + return validation + return apply_range_result(validation, located, cited) + + +def validate_route( + validation: Validation, routes: list[Located], cited: tuple[int, int] | None +) -> Validation: + locator = validation.locator or "" + legacy = LEGACY_ROUTE_RE.fullmatch(locator) + canonical = CANONICAL_ROUTE_RE.fullmatch(locator) + if not legacy and not canonical: + validation.result = "invalid-locator" + validation.problem = ( + "route locator must be '@router.method(\"/path\")' or " + "'METHOD /path -> qualified.function'" + ) + return validation + expected_method = (legacy or canonical).group("method").upper() + expected_path = (legacy or canonical).group("path") + expected_function = canonical.group("function") if canonical else None + method_matches = [ + route + for route in routes + if route.details and route.details["method"] == expected_method + ] + path_matches = [ + route + for route in method_matches + if route.details + and expected_path in {route.details["path"], route.details["full_path"]} + ] + if not method_matches: + validation.result = "invalid-locator" + validation.problem = f"HTTP method {expected_method} is not declared in this file" + return validation + if not path_matches: + validation.result = "invalid-locator" + validation.problem = ( + f"route path {expected_path!r} is not declared for {expected_method}" + ) + return validation + if expected_function: + function_matches = [ + route + for route in path_matches + if route.details and route.details["function"] == expected_function + ] + if not function_matches: + validation.result = "invalid-locator" + validation.problem = ( + f"route exists but containing function is not {expected_function!r}" + ) + return validation + path_matches = function_matches + if len(path_matches) > 1: + validation.result = "ambiguous" + validation.problem = f"route locator matches {len(path_matches)} functions" + validation.manual_review_reason = validation.problem + validation.confidence = "low" + return validation + route = path_matches[0] + details = route.details or {} + if details.get("full_path") is None and expected_path != details.get("path"): + validation.result = "ambiguous" + validation.problem = "final router prefix cannot be resolved statically" + validation.manual_review_reason = validation.problem + validation.resolved = route + validation.generated_line_range = route.line_range + validation.suggested_locator = route.name + validation.suggested_kind = "python-route" + validation.confidence = "medium" + return validation + return apply_range_result(validation, route, cited) + + +def validate_javascript( + validation: Validation, + target: Path, + cited: tuple[int, int] | None, + parser: JavascriptParser, +) -> Validation: + if not parser.supported: + validation.result = "unsupported" + validation.problem = parser.reason + validation.manual_review_reason = parser.reason + validation.confidence = "low" + return validation + try: + symbols = parser.parse(target) + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as exc: + validation.result = "unsupported" + validation.problem = f"JavaScript parser failed: {exc}" + validation.manual_review_reason = validation.problem + validation.confidence = "low" + return validation + locator = validation.locator or "" + located, exact = choose_symbol(symbols, locator) + if len(exact) > 1: + validation.result = "ambiguous" + validation.problem = f"locator matches {len(exact)} JavaScript declarations" + validation.manual_review_reason = validation.problem + validation.confidence = "low" + return validation + if not located: + validation.result = "invalid-locator" + validation.problem = f"JavaScript locator {locator!r} does not exist" + suggestion, confidence = suggest_nearby(symbols, locator, cited) + validation.confidence = confidence + if suggestion: + validation.suggested_kind = suggestion.kind + validation.suggested_locator = suggestion.name + validation.generated_line_range = suggestion.line_range + return validation + return apply_range_result(validation, located, cited) + + +def shell_functions(path: Path) -> list[Located]: + lines = path.read_text(encoding="utf-8").splitlines() + functions: list[Located] = [] + for index, line in enumerate(lines): + match = SHELL_FUNCTION_RE.match(line) + if not match: + continue + depth = 0 + end = index + 1 + for offset in range(index, len(lines)): + code = lines[offset].split("#", 1)[0] + depth += code.count("{") - code.count("}") + end = offset + 1 + if depth <= 0: + break + functions.append( + Located(match.group("name"), "shell-function", index + 1, end) + ) + return functions + + +def validate_shell( + validation: Validation, target: Path, cited: tuple[int, int] | None +) -> Validation: + try: + check = subprocess.run( + ["bash", "-n", str(target)], text=True, capture_output=True, check=False + ) + except OSError as exc: + validation.result = "unsupported" + validation.problem = f"bash unavailable: {exc}" + return validation + if check.returncode: + validation.result = "unsupported" + validation.problem = f"shell syntax check failed: {check.stderr.strip()}" + return validation + functions = shell_functions(target) + located, exact = choose_symbol(functions, validation.locator or "") + if len(exact) > 1: + validation.result = "ambiguous" + validation.problem = "shell function locator is duplicated" + validation.manual_review_reason = validation.problem + return validation + if not located: + validation.result = "invalid-locator" + validation.problem = f"shell function {validation.locator!r} does not exist" + return validation + return apply_range_result(validation, located, cited) + + +def validate_documentation( + validation: Validation, target: Path, cited: tuple[int, int] | None +) -> Validation: + locator = validation.locator or "" + if not locator: + validation.result = "invalid-locator" + validation.problem = "documentation-section requires a heading locator" + return validation + lines = target.read_text(encoding="utf-8").splitlines() + headings: list[Located] = [] + for number, line in enumerate(lines, 1): + match = MARKDOWN_HEADING_RE.match(line) + if not match: + continue + level = len(match.group("marks")) + end = len(lines) + for later, later_line in enumerate(lines[number:], number + 1): + later_match = MARKDOWN_HEADING_RE.match(later_line) + if later_match and len(later_match.group("marks")) <= level: + end = later - 1 + break + headings.append( + Located( + match.group("title").strip(), + "documentation-section", + number, + end, + ) + ) + located, exact = choose_symbol(headings, locator) + if len(exact) > 1: + validation.result = "ambiguous" + validation.problem = "documentation heading is duplicated" + validation.manual_review_reason = validation.problem + return validation + if not located: + validation.result = "invalid-locator" + validation.problem = f"documentation heading {locator!r} does not exist" + return validation + return apply_range_result(validation, located, cited) + + +def validate_evidence( + repo_root: Path, + feature_id: str, + index: int, + evidence: dict[str, Any], + javascript_parser: JavascriptParser, +) -> Validation: + target = safe_target(repo_root, evidence.get("path")) + kind = infer_kind(evidence, target) + locator = legacy_locator(evidence) + validation = Validation( + feature_id=feature_id, + evidence_index=index, + current_evidence=evidence, + inferred_kind=kind, + locator=locator, + result="ambiguous", + ) + if target is None or not target.is_file(): + validation.result = "invalid-path" + validation.problem = f"evidence path is unsafe or missing: {evidence.get('path')!r}" + validation.confidence = "high" + return validation + + cited = parse_range(evidence.get("line_range")) + explicit_range = evidence.get("line_range") + if explicit_range not in (None, "") and cited is None: + validation.result = "range-mismatch" + validation.problem = f"malformed line_range: {explicit_range!r}" + return validation + if cited is not None: + line_count = len( + target.read_text(encoding="utf-8", errors="replace").splitlines() + ) + if cited[0] > cited[1] or cited[1] > max(line_count, 1): + validation.result = "range-mismatch" + validation.problem = ( + f"line_range L{cited[0]}-L{cited[1]} is outside the " + f"{line_count}-line file" + ) + return validation + + if kind == "file": + if locator: + validation.result = "invalid-locator" + validation.problem = "file evidence must not include a locator/symbol" + validation.suggested_kind = "file" + validation.confidence = "high" + return validation + validation.result = "file-level-valid" + validation.suggested_kind = "file" + line_count = len(target.read_text(encoding="utf-8", errors="replace").splitlines()) + validation.generated_line_range = f"L1-L{max(line_count, 1)}" + return validation + + if not locator and kind not in {"python-module", "file"}: + validation.result = "invalid-locator" + validation.problem = f"{kind} evidence requires a locator" + return validation + + if kind in PYTHON_KINDS: + return validate_python(validation, target, cited) + if kind in JAVASCRIPT_KINDS: + return validate_javascript(validation, target, cited, javascript_parser) + if kind == "shell-function": + return validate_shell(validation, target, cited) + if kind == "documentation-section": + return validate_documentation(validation, target, cited) + if kind == "configuration-key": + validation.result = "ambiguous" + validation.problem = "configuration-key requires a format-aware parser" + validation.manual_review_reason = ( + "This validator does not guess configuration keys from text." + ) + validation.suggested_kind = ( + "file" if evidence.get("kind") is None else "manual-review" + ) + validation.confidence = "low" + return validation + + validation.result = "ambiguous" + validation.problem = f"no automatic resolver for inferred kind {kind!r}" + validation.manual_review_reason = ( + "Use explicit file evidence if the whole file is authoritative, " + "or manual-review with a precise reason." + ) + if evidence.get("kind") is None: + validation.suggested_kind = "file" + validation.suggested_locator = None + line_count = len(target.read_text(encoding="utf-8", errors="replace").splitlines()) + validation.generated_line_range = f"L1-L{max(line_count, 1)}" + validation.confidence = "medium" + return validation + + +def proposed_schema() -> dict[str, Any]: + return { + "required": ["path", "kind", "explanation"], + "optional": ["locator", "line_range"], + "properties": { + "path": "repository-relative file path", + "kind": sorted( + PYTHON_KINDS + | JAVASCRIPT_KINDS + | { + "shell-function", + "configuration-key", + "file", + "documentation-section", + "manual-review", + } + ), + "locator": ( + "parser-resolvable identity; omit for file evidence. " + "Routes use 'METHOD /path -> qualified.function'." + ), + "line_range": ( + "optional generated display metadata in Lx-Ly form; " + "never the primary identity" + ), + "explanation": "why this evidence supports the feature claim", + }, + "examples": [ + { + "path": "src/chat_handler.py", + "kind": "python-method", + "locator": "ChatHandler.preprocess_message", + "explanation": "Preprocesses attachments and URLs for chat requests.", + }, + { + "path": "Dockerfile", + "kind": "file", + "explanation": "Defines the container build.", + }, + ], + } + + +def assess_e2( + features: list[dict[str, Any]], validations: list[Validation] +) -> list[dict[str, Any]]: + """Record the audit's claim-relevance review for the ten frozen E2 records.""" + by_key = { + (item.feature_id, item.current_evidence.get("path"), item.locator): item + for item in validations + } + decisions = { + "CHAT-001": ( + "yes", + "Both cited tests exercise documented chat-stream behavior: emitted " + "stream metrics and non-destructive resend behavior.", + "retain", + ), + "MODEL-006": ( + "yes", + "The Node-backed test exercises the Copilot device-flow start/poll " + "contract and complete verification URI.", + "retain", + ), + "MODEL-007": ( + "yes", + "The Node-backed test exercises the ChatGPT subscription device-flow " + "contract and verification URI.", + "retain", + ), + "RESEARCH-003": ( + "partial", + "The test covers result ranking only, not SearXNG connectivity or " + "multi-provider dispatch in the feature claim.", + "demote", + ), + "DOCUMENT-002": ( + "no", + "The test covers removal of a PDF content marker, not PDF form " + "processing or high-fidelity rendering.", + "demote", + ), + "EMAIL-001": ( + "partial", + "The test covers health probing of account connections, not account " + "setup, SMTP behavior, or inbox polling.", + "demote", + ), + "SECURITY-002": ( + "yes", + "Relevant vault password-handling tests exist, but the cited test " + "function name is fabricated.", + "demote", + ), + "SECURITY-004": ( + "yes", + "Relevant prompt-injection tests exist, but both cited locator names " + "are fabricated umbrella names.", + "demote", + ), + "SECURITY-005": ( + "yes", + "Relevant URL and path confinement tests exist, but all three cited " + "locator names are fabricated umbrella names.", + "demote", + ), + "PLATFORM-009": ( + "manual-review", + "The shell diagnostic is executable evidence, but no automated test " + "function or test-suite evidence is cited and the script uses a " + "file-like fabricated symbol.", + "demote", + ), + } + assessments: list[dict[str, Any]] = [] + for feature in features: + if feature.get("evidence_maturity") != "E2": + continue + test_evidence = [ + evidence + for evidence in feature.get("evidence", []) + if str(evidence.get("path", "")).startswith("tests/") + ] + cited: list[dict[str, Any]] = [] + for evidence in test_evidence: + item = by_key.get( + (feature["id"], evidence.get("path"), legacy_locator(evidence)) + ) + cited.append( + { + "path": evidence.get("path"), + "locator": legacy_locator(evidence), + "exists": bool( + item + and item.result + not in {"invalid-path", "invalid-locator", "unsupported"} + ), + "validation_result": item.result if item else "not-validated", + } + ) + relevance, reason, decision = decisions[feature["id"]] + assessments.append( + { + "feature_id": feature["id"], + "feature_name": feature["name"], + "cited_test_evidence": cited, + "test_function_exists": ( + all(item["exists"] for item in cited) if cited else False + ), + "direct_support": relevance, + "support_reason": reason, + "suggested_decision": decision, + } + ) + return assessments + + +def build_report( + repo_root: Path, + catalog_path: Path, + features: list[dict[str, Any]], + validations: list[Validation], + javascript_parser: JavascriptParser, +) -> dict[str, Any]: + extensions = Counter() + kinds = Counter() + for item in validations: + extensions[extension_for(str(item.current_evidence.get("path", "")))] += 1 + kinds[item.inferred_kind] += 1 + results = Counter(item.result for item in validations) + affected = sorted( + { + item.feature_id + for item in validations + if item.result not in {"valid", "file-level-valid"} + } + ) + invalid_results = { + "invalid-path", + "invalid-locator", + "locator-outside-range", + "range-mismatch", + } + invalid_features = sorted( + { + item.feature_id + for item in validations + if item.result in invalid_results + } + ) + ambiguous = [ + item.to_dict() + for item in validations + if item.result in {"ambiguous", "unsupported"} + ] + return { + "audit": { + "repo_root": str(repo_root), + "catalog": str(catalog_path), + "feature_count": len(features), + "evidence_count": len(validations), + "javascript_parser": { + "supported": javascript_parser.supported, + "parser": javascript_parser.parser_name, + "reason": javascript_parser.reason, + }, + }, + "inventory": { + "total_evidence_items": len(validations), + "by_extension": dict(sorted(extensions.items())), + "by_inferred_evidence_type": dict(sorted(kinds.items())), + "features_with_invalid_or_unresolved_entries": affected, + "features_with_invalid_entries": invalid_features, + "invalid_feature_count": len(invalid_features), + "affected_feature_count": len(affected), + "ambiguous_or_unsupported_entries": ambiguous, + }, + "result_counts": {name: results.get(name, 0) for name in sorted(RESULTS)}, + "summary_counts": { + "valid": results["valid"] + results["file-level-valid"], + "invalid": sum( + results[name] + for name in ( + "invalid-path", + "invalid-locator", + "locator-outside-range", + "range-mismatch", + ) + ), + "ambiguous": results["ambiguous"], + "unsupported": results["unsupported"], + }, + "evidence_model": proposed_schema(), + "e2_assessment": assess_e2(features, validations), + "entries": [item.to_dict() for item in validations], + } + + +def build_repair_queue(validations: list[Validation]) -> dict[str, Any]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for item in validations: + if item.result in {"valid", "file-level-valid"}: + continue + grouped[item.feature_id].append( + { + "evidence_index": item.evidence_index, + "current_evidence": item.current_evidence, + "detected_problem": item.problem or item.result, + "result": item.result, + "suggested_kind": item.suggested_kind, + "suggested_locator": item.suggested_locator, + "generated_line_range": item.generated_line_range, + "confidence": item.confidence, + "manual_review_reason": item.manual_review_reason, + } + ) + return { + "feature_count": len(grouped), + "defect_count": sum(len(items) for items in grouped.values()), + "features": [ + {"feature_id": feature_id, "defects": grouped[feature_id]} + for feature_id in sorted(grouped) + ], + } + + +def render_markdown(report: dict[str, Any]) -> str: + inventory = report["inventory"] + summary = report["summary_counts"] + lines = [ + "# Discovery evidence validation report", + "", + "This report is machine-generated. The catalog was not edited.", + "", + "## Summary", + "", + f"- Features: {report['audit']['feature_count']}", + f"- Evidence items: {report['audit']['evidence_count']}", + f"- Valid (including file-level): {summary['valid']}", + f"- Invalid: {summary['invalid']}", + f"- Ambiguous: {summary['ambiguous']}", + f"- Unsupported: {summary['unsupported']}", + f"- Affected features: {inventory['affected_feature_count']}", + "", + "## Evidence by extension", + "", + "| Extension | Items |", + "|---|---:|", + ] + lines.extend( + f"| `{extension}` | {count} |" + for extension, count in inventory["by_extension"].items() + ) + lines.extend( + [ + "", + "## Evidence by inferred kind", + "", + "| Kind | Items |", + "|---|---:|", + ] + ) + lines.extend( + f"| `{kind}` | {count} |" + for kind, count in inventory["by_inferred_evidence_type"].items() + ) + lines.extend( + [ + "", + "## Result categories", + "", + "| Result | Items |", + "|---|---:|", + ] + ) + lines.extend( + f"| `{result}` | {count} |" + for result, count in report["result_counts"].items() + ) + lines.extend( + [ + "", + "## JavaScript parser", + "", + report["audit"]["javascript_parser"]["reason"] + ".", + "", + "## Affected features", + "", + ", ".join( + f"`{feature_id}`" + for feature_id in inventory[ + "features_with_invalid_or_unresolved_entries" + ] + ) + or "None.", + "", + "## E2 relevance review", + "", + "| Feature | Cited test evidence | Exists | Direct support | Decision |", + "|---|---|---:|---|---|", + ] + ) + for item in report["e2_assessment"]: + citations = "<br>".join( + f"`{test['path']}:{test['locator']}`" + for test in item["cited_test_evidence"] + ) or "None" + lines.append( + f"| `{item['feature_id']}` | {citations} | " + f"{'yes' if item['test_function_exists'] else 'no'} | " + f"{item['direct_support']}: {item['support_reason']} | " + f"**{item['suggested_decision']}** |" + ) + lines.extend( + [ + "", + "### Numerical inconsistency resolved", + "", + "The six problematic cited E2 test records are the six nonexistent " + "locator names in SECURITY-002 (one), SECURITY-004 (two), and " + "SECURITY-005 (three). Seven feature demotions were suggested because " + "PLATFORM-009 is an additional feature-level maturity problem: it " + "cites no automated test function. The other three demotions are " + "claim-relevance findings for existing tests, so the figures measure " + "different things and should not be compared as the same denominator.", + "", + "## Proposed evidence schema", + "", + "Required: `path`, `kind`, and `explanation`. `locator` is required " + "for symbol/route/section evidence and omitted for `file`. " + "`line_range` is optional generated metadata.", + "", + "```json", + json.dumps(report["evidence_model"]["examples"], indent=2), + "```", + "", + "## Defects", + "", + "| Feature | Path | Locator | Result | Generated range | Problem |", + "|---|---|---|---|---|---|", + ] + ) + for item in report["entries"]: + if item["result"] in {"valid", "file-level-valid"}: + continue + evidence = item["current_evidence"] + problem = (item["problem"] or "").replace("|", "\\|") + lines.append( + f"| `{item['feature_id']}` | `{evidence.get('path', '')}` | " + f"`{item.get('locator') or ''}` | `{item['result']}` | " + f"`{item.get('generated_line_range') or ''}` | {problem} |" + ) + return "\n".join(lines) + "\n" + + +def write_outputs(output_dir: Path, report: dict[str, Any], queue: dict[str, Any]) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "evidence-validation-report.json").write_text( + json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + (output_dir / "evidence-validation-report.md").write_text( + render_markdown(report), encoding="utf-8" + ) + (output_dir / "evidence-repair-queue.json").write_text( + json.dumps(queue, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + +def run(argv: list[str] | None = None) -> int: + args = parse_args(argv) + repo_root = args.repo_root.resolve() + catalog_path = args.catalog.resolve() + features = load_catalog(catalog_path) + javascript_parser = JavascriptParser(repo_root) + validations = [ + validate_evidence( + repo_root, + str(feature.get("id", f"<feature:{feature_index}>")), + evidence_index, + evidence, + javascript_parser, + ) + for feature_index, feature in enumerate(features) + for evidence_index, evidence in enumerate(feature.get("evidence", [])) + if isinstance(evidence, dict) + ] + report = build_report( + repo_root, catalog_path, features, validations, javascript_parser + ) + queue = build_repair_queue(validations) + write_outputs(args.output_dir.resolve(), report, queue) + print(json.dumps(report["summary_counts"], sort_keys=True)) + return 1 if queue["defect_count"] else 0 + + +if __name__ == "__main__": + sys.exit(run()) From 91764ad3e79f8a5acf6bda8ef9585814d97fe214 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Sun, 26 Jul 2026 14:13:15 +0100 Subject: [PATCH 02/43] docs(discovery): replace audit catalog with system maps --- docs/discovery/BASELINE-STATUS.md | 62 - docs/discovery/README.md | 21 - docs/discovery/audit-method.md | 80 - docs/discovery/domains/agent.md | 153 - docs/discovery/domains/calendar.md | 45 - docs/discovery/domains/chat.md | 197 - docs/discovery/domains/contact.md | 24 - docs/discovery/domains/cookbook.md | 85 - docs/discovery/domains/document.md | 108 - docs/discovery/domains/email.md | 87 - docs/discovery/domains/frontend.md | 149 - docs/discovery/domains/media.md | 170 - docs/discovery/domains/memory.md | 24 - docs/discovery/domains/model.md | 177 - docs/discovery/domains/note.md | 24 - docs/discovery/domains/platform.md | 213 - docs/discovery/domains/research.md | 86 - docs/discovery/domains/security.md | 154 - docs/discovery/domains/skill.md | 24 - docs/discovery/feature-catalog.json | 4797 ----------------- docs/discovery/feature-catalog.md | 85 - .../discovery/references/source-provenance.md | 28 - .../tools/test_validate_discovery_evidence.py | 224 - .../tools/validate_discovery_consistency.py | 592 -- .../tools/validate_discovery_docs.py | 172 - .../tools/validate_discovery_evidence.py | 1413 ----- specs/discovery/README.md | 30 + specs/discovery/safety-boundaries.md | 129 + specs/discovery/system-map.md | 123 + 29 files changed, 282 insertions(+), 9194 deletions(-) delete mode 100644 docs/discovery/BASELINE-STATUS.md delete mode 100644 docs/discovery/README.md delete mode 100644 docs/discovery/audit-method.md delete mode 100644 docs/discovery/domains/agent.md delete mode 100644 docs/discovery/domains/calendar.md delete mode 100644 docs/discovery/domains/chat.md delete mode 100644 docs/discovery/domains/contact.md delete mode 100644 docs/discovery/domains/cookbook.md delete mode 100644 docs/discovery/domains/document.md delete mode 100644 docs/discovery/domains/email.md delete mode 100644 docs/discovery/domains/frontend.md delete mode 100644 docs/discovery/domains/media.md delete mode 100644 docs/discovery/domains/memory.md delete mode 100644 docs/discovery/domains/model.md delete mode 100644 docs/discovery/domains/note.md delete mode 100644 docs/discovery/domains/platform.md delete mode 100644 docs/discovery/domains/research.md delete mode 100644 docs/discovery/domains/security.md delete mode 100644 docs/discovery/domains/skill.md delete mode 100644 docs/discovery/feature-catalog.json delete mode 100644 docs/discovery/feature-catalog.md delete mode 100644 docs/discovery/references/source-provenance.md delete mode 100644 docs/discovery/tools/test_validate_discovery_evidence.py delete mode 100755 docs/discovery/tools/validate_discovery_consistency.py delete mode 100755 docs/discovery/tools/validate_discovery_docs.py delete mode 100644 docs/discovery/tools/validate_discovery_evidence.py create mode 100644 specs/discovery/README.md create mode 100644 specs/discovery/safety-boundaries.md create mode 100644 specs/discovery/system-map.md diff --git a/docs/discovery/BASELINE-STATUS.md b/docs/discovery/BASELINE-STATUS.md deleted file mode 100644 index 7cb6ec391..000000000 --- a/docs/discovery/BASELINE-STATUS.md +++ /dev/null @@ -1,62 +0,0 @@ -# Discovery Baseline Status - -## Purpose and scope - -This package is a commit-pinned discovery baseline and feature index for the `discovery` branch at frozen commit `d8a2059df8e53bc7275c45339849d14c8651e73c`. It inventories **79 feature records** across **16 domains** to help maintainers locate likely implementation areas and identify validation gaps. It is not an authoritative architecture reference or a runtime-certification record. - -The canonical inventory is [`feature-catalog.json`](feature-catalog.json); [`feature-catalog.md`](feature-catalog.md) and the files in [`domains/`](domains/) are derived reading views. See [`audit-method.md`](audit-method.md) for the status and maturity definitions. - -## What maintainers may use now - -- Use the catalog and domain views as a frozen discovery index, including their feature IDs, stated scope, likely source locations, and declared runtime prerequisites. -- Treat a catalog status such as `verified` as meaning implementation was identified during discovery. It does **not** mean every evidence locator, line range, test claim, or runtime behaviour has passed semantic validation. -- Use the structural checks to confirm package shape and cross-view consistency; use the evidence validator to assess whether individual evidence assertions are semantically supported. - -## Validation snapshot - -Structural validation passes: the catalog has 79 records, the 16 domain views match it, and the package structural validators pass. All 11 focused evidence-validator tests pass. - -The evidence validator found **170 evidence entries**: **82 valid**, **66 invalid**, **6 ambiguous**, and **16 unsupported**. Its non-zero exit is expected while those semantic evidence defects remain. - -Structural validity checks the documentation schema, record counts, derived-view consistency, file existence, line-range bounds, links, and public-safety rules. Semantic evidence validity additionally checks whether the cited locator exists, falls within its cited range, uses a supported parser, and actually supports the feature claim. Passing the former does not establish the latter. - -## E2 review decisions - -E2 means directly relevant automated test evidence supports the feature claim; a test file’s existence alone is insufficient. The generated evidence report was used to reassess all ten E2 records. - -| Feature | Decision | Reason | -|---|---|---| -| `CHAT-001` | Retain E2 | Two cited tests directly exercise documented streaming-related behaviour. | -| `MODEL-006` | Retain E2 | The cited device-flow test exercises the Copilot start/poll contract and verification URI. | -| `MODEL-007` | Retain E2 | The cited device-flow test exercises the ChatGPT subscription contract and verification URI. | -| `RESEARCH-003` | Demote to E1 | The cited test covers ranking, not provider connectivity or dispatch; the route-to-provider implementation path was identified. | -| `DOCUMENT-002` | Demote to E1 | The cited marker test does not support form processing or rendering; the document route does call the PDF form handlers. | -| `EMAIL-001` | Demote to E1 | The cited health test is narrower than setup, SMTP, and polling; application setup invokes the email router and its poller. | -| `SECURITY-002` | Demote to E1 | Relevant vault-password tests exist, but the cited test locator is fabricated; the application mounts the vault route implementation. | -| `SECURITY-004` | Demote to E1 | Relevant injection tests exist, but the cited locator names are fabricated; callers use the documented context guard. | -| `SECURITY-005` | Demote to E1 | Relevant guard tests exist, but the cited locator names are fabricated; route code calls the documented URL guard. | -| `PLATFORM-009` | Demote to E0 | The manifests and diagnostic script establish discovered operational artifacts, not a traced application path or directly relevant automated test. | - -The current maturity distribution is **E0: 68**, **E1: 8**, **E2: 3**, **E3: 0**, **E4: 0**. Runtime validation is still pending where the catalog says it requires external services, interactive authentication, specialised hardware, or host Docker GPU support. - -## Known discovery caveats - -- `AGENT-004` includes a legacy no-op activity-log shim rather than active assistant-log behaviour. -- `FRONTEND-007` points to a missing `static/backgrounds.html` target; its existing variant pages do not make that route functional. -- `RESEARCH-003` retains a compatibility module that aliases the canonical search implementation, and `DOCUMENT-002` separates optional PDF viewing from form handling. -- Secret-storage and vault-command handling are distinct implementation areas; this index does not make an end-to-end security guarantee. - -## Recommended next documentation work - -Repair semantic evidence selectively while architecture and operations documentation is written, beginning with the seven E2 demotions and maintainer-owned feature descriptions. Do not wait for the complete evidence queue before documenting the system. Record controlled runtime observations when external services, credentials, hardware, or Docker GPU access are available, and label unsupported claims explicitly. - -## Validation commands - -```bash -PYTHONDONTWRITEBYTECODE=1 python3 docs/discovery/tools/validate_discovery_docs.py --repo-root . -PYTHONDONTWRITEBYTECODE=1 python3 docs/discovery/tools/validate_discovery_consistency.py -PYTHONDONTWRITEBYTECODE=1 python3 -m unittest docs/discovery/tools/test_validate_discovery_evidence.py -v -PYTHONDONTWRITEBYTECODE=1 python3 docs/discovery/tools/validate_discovery_evidence.py --repo-root . --catalog docs/discovery/feature-catalog.json --output-dir <local-report-directory> -``` - -Supply a local report directory outside `docs/discovery/` for the final command so generated reports are not added to the package. diff --git a/docs/discovery/README.md b/docs/discovery/README.md deleted file mode 100644 index 1a397da74..000000000 --- a/docs/discovery/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Odysseus Discovery Package - -## Provisional discovery baseline - -This is a commit-pinned discovery baseline and feature index for the `discovery` branch at `d8a2059df8e53bc7275c45339849d14c8651e73c`. It contains 79 feature records across 16 domains. It is **not** an authoritative architecture reference, a runtime certification, or a claim that every evidence citation is semantically valid. - -Read [`BASELINE-STATUS.md`](BASELINE-STATUS.md) first for the publication status, evidence-validation totals, E2 decisions, known caveats, and the recommended next documentation work. - -## Package contents - -- [`feature-catalog.json`](feature-catalog.json) is the canonical machine-readable catalog. -- [`feature-catalog.md`](feature-catalog.md) and [`domains/`](domains/) are derived reading views. -- [`audit-method.md`](audit-method.md) defines feature status and evidence maturity. -- [`references/source-provenance.md`](references/source-provenance.md) records the frozen snapshot. -- [`tools/`](tools/) contains the structural, consistency, and evidence validators. - -A feature status such as `verified` means implementation was identified during discovery. It does not mean every evidence locator, line range, test claim, or runtime behaviour has passed semantic validation. - -## Validation - -Run the commands in [`BASELINE-STATUS.md`](BASELINE-STATUS.md#validation-commands). Structural checks and semantic evidence checks have different purposes; see that status document for the current results and interpretation. diff --git a/docs/discovery/audit-method.md b/docs/discovery/audit-method.md deleted file mode 100644 index b2c3bd1f1..000000000 --- a/docs/discovery/audit-method.md +++ /dev/null @@ -1,80 +0,0 @@ -# Odysseus Functional Audit Methodology - -## Purpose - -This document specifies the methodology and evidence standards for the read-only discovery audit of **Odysseus**. - -## Snapshot Baseline - -- **Repository**: `odysseus-dev/odysseus` -- **Audit Target Branch**: `discovery` -- **Frozen Commit SHA**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Snapshot Date**: `2026-07-23T14:49:02Z` -- **Audit Mode**: Read-Only inventory & documentation review - -## Rules of Engagement - -1. **No Code Mutations**: Application code and tests outside `docs/discovery/` remain untouched. -2. **No External Operations**: No GitHub issues, PRs, comments, labels, or branch mutations. -3. **Zero Inferred Success**: Documentation claims require empirical evidence of implementation and reachability. Filenames, README descriptions, and docstrings alone do not constitute proof. -4. **Strict Status Categorization**: All capabilities are assigned exactly one authorized status: - - `verified`: Implemented, reachable, and supported by code evidence. - - `partial`: Partially implemented or missing full frontend/backend connection. - - `disabled`: Gated off by default feature flags or configuration. - - `experimental`: Active but requiring non-standard hardware or runtimes. - - `legacy`: Obsolete feature retained for backwards compatibility. - - `dead-code-candidate`: Code exists but is unreachable from UI or API routes. - - `unverified`: Implementation present but untestable without external secrets or hardware. - -## Evidence Maturity Scale - -Evidence maturity is evaluated independently from catalog feature status. A feature status such as `verified` records that implementation was identified during discovery; it is not a statement that every evidence locator, test claim, line range, or runtime behaviour has passed semantic validation. - -- **E0 - Discovered**: Candidate identified in documentation, route declaration, or source file. -- **E1 - Code-path traced**: Frontend/API entry point connected through services and data handlers. -- **E2 - Test-backed**: At least one directly relevant automated test assertion supports the feature claim. A test file's existence, an unrelated assertion, or an invalid test locator does not establish E2. -- **E3 - Runtime-validated**: Maintainer reproduced behavior in a recorded local environment. -- **E4 - Maintainer-accepted**: Maintainers accepted the feature description and support status. - -## Audit Workflow - -```mermaid -flowchart TD - P0["Phase 0: Snapshot Isolation<br/>(Commit d8a2059)"] --> P1["Phase 1: Codebase Discovery<br/>(Routes, Services, Static JS, Specs)"] - P1 --> P2["Phase 2: Feature Reachability & Verification<br/>(Route matching, FE entrypoints, tests)"] - P2 --> P3["Phase 3: Catalog & Evidence Compilation<br/>(File paths, symbols, exact line ranges)"] - P3 --> P4["Phase 4: Quality & Integrity Audit<br/>(100% path existence check, schema validation)"] -``` - -### Phase 0: Snapshot Isolation -The audit is pinned to git commit `d8a2059df8e53bc7275c45339849d14c8651e73c`. All file paths, symbol declarations, and line ranges map strictly to this commit. - -### Phase 1: Codebase Discovery -All top-level and nested directories were traversed, including: -- Backend Entry Points (`app.py`, `routes/`, `routes/*/*.py`, `companion/`) -- Core Framework (`core/database.py`, `core/session_manager.py`, `core/auth.py`) -- Business Logic Services (`src/`, `services/`, `mcp_servers/`) -- Frontend Assets (`static/app.js`, `static/js/`, `static/index.html`) -- Test Suites (`tests/`, `tests/cli/`, `tests/streaming/`) -- Operations & Docker (`Dockerfile`, `docker-compose*.yml`, `scripts/`) - -### Phase 2: Verification Protocol -For each feature candidate, the following table was evaluated: -- **User Reachability**: Frontend UI element, modal, route, or CLI script. -- **API Entrypoint**: FastAPI `@router` declaration or WebSocket/SSE handler. -- **Backend Execution**: Concrete Python module method, service, or tool call. -- **Data Persistence**: Disk file, SQLite table, or vector collection. -- **Test Coverage**: Automated test file executing assertions against the component. - -### Phase 3: Evidence Linking Standard -Every feature entry in `feature-catalog.json` contains a structured `evidence` list with: -- `path`: Relative path from repository root. -- `symbol`: Route, class, function, or element symbol name. -- `line_range`: Inclusive line range (e.g. `L120-L250`). -- `explanation`: Short factual statement proving reachability or implementation. - -### Phase 4: Quality Check & Schema Constraints -Before finalization: -1. Every evidence file path is validated against the checkout. -2. Every Markdown entry matches `feature-catalog.json`. -3. Recommendation language is separated from empirical factual observations. diff --git a/docs/discovery/domains/agent.md b/docs/discovery/domains/agent.md deleted file mode 100644 index 82574ddc3..000000000 --- a/docs/discovery/domains/agent.md +++ /dev/null @@ -1,153 +0,0 @@ -# Agent - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## AGENT-001 — Autonomous Agent Loop & Tool Execution Engine - -- **Domain**: `agent` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Executes multi-step agent reasoning loops, tool invocation parsing, and automated response generation. - -### Evidence summary - -- `src/agent_loop.py` — `run_agent_loop` — Core loop evaluating model tool requests and executing handlers. -- `src/tool_execution.py` — `execute_tool_call` — Dispatches tool invocation requests to underlying tool handlers. - -### Unknowns - -- Infinite tool loop if termination condition fails. - -## AGENT-002 — Scheduled Tasks & Event Bus Dispatcher - -- **Domain**: `agent` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Schedules background recurring or delayed tasks, emits event bus triggers, and executes automated flows. - -### Evidence summary - -- `routes/task_routes.py` — `@router.get('')` — Fetches active scheduled tasks. -- `src/task_scheduler.py` — `TaskScheduler` — Async task scheduler dispatching cron and delay triggers. - -### Unknowns - -- Task execution failure handling on system restart. - -## AGENT-003 — Webhook Event Subscriptions & Trigger Processing - -- **Domain**: `agent` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Manages incoming/outgoing webhook subscriptions, endpoint authentication tokens, and event triggers. - -### Evidence summary - -- `routes/webhook_routes.py` — `@router.get('/webhooks')` — Returns list of registered webhooks. -- `src/webhook_manager.py` — `WebhookManager` — Handles payload delivery and signature verification. - -### Unknowns - -- SSRF risks when contacting external webhook URLs if unvalidated. - -## AGENT-004 — Assistant Settings, Task Check-Ins & Background Job Monitor - -- **Domain**: `agent` -- **Status**: `partial` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Manages per-user assistant sessions and scheduled check-in settings, drains background job completions, and retains a legacy no-op activity logging shim. - -### Evidence summary - -- `routes/assistant_routes.py` — `setup_assistant_routes` — Active assistant session, settings, manual check-in, run-status and timezone-list endpoints, including the owner-scoping guards. -- `src/bg_monitor.py` — `_drain_agent` — Runs the agent loop headless against a session to produce the background-job follow-up turn. -- `src/bg_monitor.py` — `_run_followup` — Drains completed background jobs and auto-continues the owning session, deferring while a live turn is in progress. -- `src/assistant_log.py` — `log_to_assistant` — Legacy no-op activity logging shim retained for existing callers; documented as inactive rather than as current behaviour. - -### Unknowns - -- Route `/api/assistant/logs` cited in legacy docs is absent from assistant router. -- Existing unit test `tests/cli/test_logs_cli_resolve_nonstring.py` tests CLI target-name resolution logic, not active assistant routes or bg_monitor execution loop. - -## AGENT-005 — Model Context Protocol (MCP) Server Integration - -- **Domain**: `agent` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Integrates external MCP servers over stdio/SSE to expand agent capabilities dynamically. - -### Evidence summary - -- `routes/mcp_routes.py` — `setup_mcp_routes` — Exposes management endpoints for external MCP servers. -- `src/mcp_manager.py` — `McpManager` — Manages MCP server subprocess lifecycles. - -### Unknowns - -- Subprocess leaks if external MCP server process fails to terminate clean. - -## AGENT-006 — AI Interaction Tools & Pipeline Orchestration - -- **Domain**: `agent` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides specialized AI interaction tools for agent self-debugging, debate, and multi-model collaboration. - -### Evidence summary - -- `src/ai_interaction.py` — `init_ai_interaction_tools` — Registers specialized multi-agent interaction primitives. -- `src/builtin_actions.py` — `execute_builtin_action` — Executes pre-built action intent sequences. - -### Unknowns - -- High API token consumption during extended agent debates. - -## AGENT-007 — Subprocess & Background Job Execution Tools - -- **Domain**: `agent` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides sandboxed bash/shell tool execution capabilities with output streaming and background tracking. - -### Evidence summary - -- `src/agent_tools/subprocess_tools.py` — `run_command` — Executes shell commands in background/foreground. -- `src/bg_jobs.py` — `JobManager` — Tracks async background subprocess tasks. - -### Unknowns - -- Arbitrary shell command execution permissions if sandbox confinement fails. diff --git a/docs/discovery/domains/calendar.md b/docs/discovery/domains/calendar.md deleted file mode 100644 index 4a2dad8d3..000000000 --- a/docs/discovery/domains/calendar.md +++ /dev/null @@ -1,45 +0,0 @@ -# Calendar - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## CALENDAR-001 — CalDAV Calendar Synchronization & Account Setup - -- **Domain**: `calendar` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires a controlled external CalDAV server. - -### Purpose - -Connects to remote CalDAV servers (Apple iCloud, Nextcloud, Google) to sync calendar event feeds. - -### Evidence summary - -- `routes/calendar_routes.py` — `setup_calendar_routes` — Exposes CalDAV setup and manual sync trigger routes. -- `src/caldav_sync.py` — `CalDavSync` — Fetches and parses remote iCalendar VEVENT objects. - -### Unknowns - -- Invalid SSL certificates on self-hosted CalDAV servers. - -## CALENDAR-002 — Calendar Event Operations & iCalendar Parsing - -- **Domain**: `calendar` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Creates, updates, deletes, and displays calendar events with timezone conversion and reminder notifications. - -### Evidence summary - -- `routes/calendar_routes.py` — `@router.get('/events')` — Fetches calendar events for requested date window. -- `src/tools/calendar.py` — `CalendarTool` — Agent tool for creating and modifying calendar entries. - -### Unknowns - -- Recurring RRULE event expansion calculation bugs across leap years. diff --git a/docs/discovery/domains/chat.md b/docs/discovery/domains/chat.md deleted file mode 100644 index 8ff904c6b..000000000 --- a/docs/discovery/domains/chat.md +++ /dev/null @@ -1,197 +0,0 @@ -# Chat - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## CHAT-001 — Core Chat Streaming & SSE Message Generation - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E2` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires access to a live LLM provider endpoint (OpenAI API key or local Ollama server). - -### Purpose - -Handles real-time Server-Sent Events (SSE) chat streaming, token rendering, and model response generation. - -### Evidence summary - -- `routes/chat_routes.py` — `chat_stream` — POST /api/chat_stream SSE endpoint; builds the shared chat context, then dispatches to the chat-mode or agent-mode streaming path. -- `routes/chat_helpers.py` — `build_chat_context` — Shared context builder invoked by chat_stream; runs message preprocessing and assembles the memory/RAG/web context preface. -- `src/chat_handler.py` — `ChatHandler.preprocess_message` — Message preprocessing (attachments, URLs, tool preprocessing) reached from build_chat_context via routes/chat_helpers.py:preprocess. -- `src/chat_processor.py` — `ChatProcessor.build_context_preface` — Builds the retrieval and web-source context preface injected into the streamed request. -- `src/llm_core.py` — `stream_llm_with_fallback` — Chat-mode streaming dispatcher called from chat_stream; wraps stream_llm with an ordered provider fallback chain. -- `src/llm_core.py` — `stream_llm` — Per-request streaming entry wrapped by stream_llm_with_fallback; acquires the local model slot and delegates to _stream_llm_inner. -- `src/agent_loop.py` — `stream_agent_loop` — Agent-mode streaming path called from chat_stream when the request selects agent mode. -- `tests/test_chat_metrics.py` — `test_stream_llm_passes_through_llamacpp_timings` — Inspected unit test asserting stream_llm forwards backend generation timings into the emitted metrics chunk. -- `tests/test_resend_message_nondestructive.py` — `test_resend_message_does_not_truncate_by_default` — Inspected unit test asserting the frontend resend path does not truncate prior conversation turns. - -### Unknowns - -- Stream interruption on connection drops requires retry logic. - -## CHAT-002 — Session Management & Conversation State - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Manages session creation, listing, switching, renaming, and persistence of conversation metadata. - -### Evidence summary - -- `routes/session_routes.py` — `@router.get('/api/sessions')` — Lists active sessions filtered by user owner scope. -- `core/session_manager.py` — `SessionManager` — Provides thread-safe session storage operations. - -### Unknowns - -- Concurrent file writes to sessions.json under high load. - -## CHAT-003 — Chat History & Message Editing/Truncation - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides history retrieval, message content updating, message deletion, and history branch truncation. - -### Evidence summary - -- `routes/history/history_routes.py` — `@router.get('/api/history/{session_id}')` — Fetches message history timeline for a session. -- `routes/history_routes.py` — `_sys.modules[__name__] = _canonical` — Backward-compatibility shim module. - -### Unknowns - -- Truncating messages re-indexes context window and clears cached tool calls. - -## CHAT-004 — File & Multimodal Attachment Handling - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Handles uploading, mime validation, image preview, vision encoding, and file attachments in chat messages. - -### Evidence summary - -- `routes/upload_routes.py` — `@router.post('')` — Accepts multi-part file uploads and generates vision metadata. -- `src/upload_handler.py` — `UploadHandler.save_file` — Validates upload size and atomicity on disk. - -### Unknowns - -- Large file uploads may consume server disk space if cleanup task fails. - -## CHAT-005 — Chat Message Search - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Enables full-text keyword search across stored chat messages and sessions. - -### Evidence summary - -- `routes/search_routes.py` — `setup_search_routes` — Registers chat message search endpoint. -- `src/session_search.py` — `search_sessions` — Executes query matching against session transcripts. - -### Unknowns - -- Full table scans on un-indexed text columns for very large databases. - -## CHAT-006 — System Prompts & Preset Management - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides creation, selection, and customization of system prompt presets for chat sessions. - -### Evidence summary - -- `routes/preset_routes.py` — `setup_preset_routes` — API routes for listing and modifying system prompt presets. -- `src/preset_manager.py` — `PresetManager` — Disk-backed manager for prompt presets. - -### Unknowns - -- Invalid JSON syntax in user presets file can corrupt preset loading. - -## CHAT-007 — Emoji Rendering & Twemoji SVG Proxy - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Proxies Twemoji SVG icons locally to render flat SVG emojis in message text without external CDN dependencies. - -### Evidence summary - -- `routes/emoji_routes.py` — `setup_emoji_routes` — Serves locally cached Twemoji SVGs. - -### Unknowns - -- First request fetches SVG from remote CDN before caching locally. - -## CHAT-008 — Input History Recall (Arrow Up) - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Allows users to cycle through previously sent prompt messages in the chat composer input using Arrow-Up/Down keys. - -### Evidence summary - -- `static/js/composerArrowUpRecall.js` — `initComposerRecall` — Listens for ArrowUp keypress on composer textarea. - -### Unknowns - -- Client-side browser storage limits. - -## CHAT-009 — Context Window Compaction & Truncation - -- **Domain**: `chat` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Compacts session transcript history when prompt size exceeds context limits using summarization. - -### Evidence summary - -- `routes/history/history_routes.py` — `@router.post('/api/session/{session_id}/compact')` — Triggers context summarization and compaction. -- `src/context_compactor.py` — `compact_context` — Executes context token pruning and summary generation. - -### Unknowns - -- Aggressive compaction may discard subtle user instructions. diff --git a/docs/discovery/domains/contact.md b/docs/discovery/domains/contact.md deleted file mode 100644 index 47122b20a..000000000 --- a/docs/discovery/domains/contact.md +++ /dev/null @@ -1,24 +0,0 @@ -# Contact - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## CONTACT-001 — CardDAV Contact Management & Address Book Integration - -- **Domain**: `contact` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Connects to CardDAV servers, imports VCard contacts, and provides contact lookup for email/calendar autocomplete. - -### Evidence summary - -- `routes/contacts/contacts_routes.py` — `@router.get('/list')` — Returns contact list filtered by search query. -- `src/tools/contacts.py` — `ContactsTool` — Agent tool for querying user address book contacts. - -### Unknowns - -- VCard 3.0 vs 4.0 property parsing mismatches. diff --git a/docs/discovery/domains/cookbook.md b/docs/discovery/domains/cookbook.md deleted file mode 100644 index 6d22bccef..000000000 --- a/docs/discovery/domains/cookbook.md +++ /dev/null @@ -1,85 +0,0 @@ -# Cookbook - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## COOKBOOK-001 — Local Model Download & Recipe Lifecycle Management - -- **Domain**: `cookbook` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Downloads HuggingFace models, configures execution parameters, and manages local GGUF/MLX model servers. - -### Evidence summary - -- `routes/cookbook_routes.py` — `setup_cookbook_routes` — Exposes model downloading and process serving endpoints. -- `static/js/cookbook.js` — `initCookbook` — UI manager for local model library. - -### Unknowns - -- Disk space exhaustion during multi-gigabyte GGUF weights downloads. - -## COOKBOOK-002 — Hardware Model Fitting ('What Fits?') Analysis Engine - -- **Domain**: `cookbook` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Calculates RAM/VRAM requirements, quantized size, and context overhead to determine model compatibility. - -### Evidence summary - -- `routes/hwfit_routes.py` — `setup_hwfit_routes` — Calculates hardware model compatibility. -- `services/hwfit/fit.py` — `calculate_fit` — Performs parameter and memory fit calculations. - -### Unknowns - -- Inaccurate VRAM estimation for non-standard KV-cache quantization. - -## COOKBOOK-003 — HuggingFace & MLX Model Discovery Services - -- **Domain**: `cookbook` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Searches HuggingFace Hub and local MLX model repositories for compatible GGUF and MLX weights. - -### Evidence summary - -- `services/hwfit/hf_discovery.py` — `search_hf_models` — Queries HuggingFace API for model tags and files. - -### Unknowns - -- HuggingFace API rate limits when searching without an API token. - -## COOKBOOK-004 — Host Docker Access for Inference Container Runtimes - -- **Domain**: `cookbook` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires Docker access and supported physical GPU hardware. - -### Purpose - -Detects and connects to host Docker engine to launch containerized Ollama, vLLM, or SGLang runtimes. - -### Evidence summary - -- `src/host_docker_access.py` — `HostDockerAccess` — Interacts with host docker daemon. - -### Unknowns - -- Permission denied accessing docker socket on non-root setups. diff --git a/docs/discovery/domains/document.md b/docs/discovery/domains/document.md deleted file mode 100644 index ca8ece487..000000000 --- a/docs/discovery/domains/document.md +++ /dev/null @@ -1,108 +0,0 @@ -# Document - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## DOCUMENT-001 — Document & Canvas Artifact Management - -- **Domain**: `document` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Renders dynamic canvas documents, handles live editing, markdown preview, and side-by-side artifact display. - -### Evidence summary - -- `routes/document_routes.py` — `setup_document_routes` — Registers document artifact CRUD routes. -- `static/js/document.js` — `initDocumentView` — Renders interactive canvas document panel. - -### Unknowns - -- Concurrent edits on the same document artifact. - -## DOCUMENT-002 — PDF Form Processing & High-Fidelity Rendering - -- **Domain**: `document` -- **Status**: `verified` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires optional PyMuPDF (`fitz`) or pypdf runtime dependency. - -### Purpose - -Extracts form fields from PDF files, fills dynamic values, and generates PDF previews. - -### Evidence summary - -- `src/pdf_runtime.py` — `load_pymupdf_for_pdf_viewer` — Loads optional PyMuPDF runtime for PDF viewing. -- `src/pdf_forms.py` — `extract_form_fields` — Handles PDF form field extraction and filling. -- `tests/test_document_pdf_marker.py` — `test_marker_removed_without_eating_following_text` — Tests PDF text extraction wrapper stripping without content corruption. - -### Unknowns - -- Complex XFA PDF forms may not extract cleanly with standard pdf parsers. - -## DOCUMENT-003 — Personal Document Indexing & RAG Retrieval - -- **Domain**: `document` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Indexes local user documents (PDF, DOCX, TXT) into ChromaDB for semantic vector retrieval. - -### Evidence summary - -- `routes/personal_routes.py` — `setup_personal_routes` — Personal document RAG indexing and search API endpoints. -- `src/personal_docs.py` — `PersonalDocsManager` — Handles file text chunking and vector storage. - -### Unknowns - -- Slow vector embedding indexing step for massive multi-thousand page documents. - -## DOCUMENT-004 — Document Conversion & Text Extraction Engine - -- **Domain**: `document` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Converts office formats (.docx, .xlsx, .pptx) and HTML into clean Markdown text representations. - -### Evidence summary - -- `src/markitdown_runtime.py` — `convert_to_markdown` — Converts binary office documents into structured Markdown text. - -### Unknowns - -- Formatting loss when parsing legacy binary doc/xls files. - -## DOCUMENT-005 — Document Library UI Navigation - -- **Domain**: `document` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides dedicated UI view for browsing, filtering, and organizing saved user documents. - -### Evidence summary - -- `static/js/documentLibrary.js` — `initDocumentLibrary` — Renders document library navigation grid. -- `app.py` — `serve_library` — Serves SPA shell for /library route. - -### Unknowns - -- Large folder trees may cause initial DOM render slowdown. diff --git a/docs/discovery/domains/email.md b/docs/discovery/domains/email.md deleted file mode 100644 index 12b28ca43..000000000 --- a/docs/discovery/domains/email.md +++ /dev/null @@ -1,87 +0,0 @@ -# Email - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## EMAIL-001 — Email Account Setup, IMAP/SMTP Connection & Polling - -- **Domain**: `email` -- **Status**: `verified` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires a controlled live IMAP account and network access. - -### Purpose - -Configures IMAP/SMTP email accounts, validates TLS certificates, and polls background inbox updates. - -### Evidence summary - -- `routes/email_routes.py` — `setup_email_routes` — Sets up email account management and synchronization routes. -- `routes/email_pollers.py` — `_start_poller` — Background poller for email inbox synchronization. -- `tests/test_service_health_email.py` — `test_email_ok_all_connect` — Tests IMAP connection health probing and status reporting. - -### Unknowns - -- Account lockouts if bad credentials are repeatedly polled. - -## EMAIL-002 — Email Searching, Threading & Message Operations - -- **Domain**: `email` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Parses email headers, folds signatures, groups messages into threads, and executes full-text email search. - -### Evidence summary - -- `routes/email_routes.py` — `@router.get('/search')` — Executes search across cached email headers and text. -- `src/email_thread_parser.py` — `parse_email_thread` — Builds conversation tree from Message-ID and In-Reply-To headers. - -### Unknowns - -- Malformed MIME email structures failing HTML sanitization. - -## EMAIL-003 — Email Composition, Draft Management & Sending - -- **Domain**: `email` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires a controlled live SMTP account and network access. - -### Purpose - -Creates, saves, and dispatches HTML/plaintext email messages via SMTP. - -### Evidence summary - -- `routes/email_routes.py` — `@router.post('/send')` — Sends email message via user SMTP credentials. - -### Unknowns - -- SMTP connection drop mid-send causing unsent mail state. - -## EMAIL-004 — Email MCP Server & Codex Integration Bridge - -- **Domain**: `email` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Exposes constrained email reading and draft capabilities to external Codex / MCP agents with scope checks. - -### Evidence summary - -- `mcp_servers/email_server.py` — `EmailMcpServer` — MCP server exposing email tools over stdio/SSE. -- `routes/codex_routes.py` — `setup_codex_routes` — Bridge endpoints for external Codex plugin integration. - -### Unknowns - -- Unauthorized mail sending if token scopes are improperly scoped. diff --git a/docs/discovery/domains/frontend.md b/docs/discovery/domains/frontend.md deleted file mode 100644 index 6f9891783..000000000 --- a/docs/discovery/domains/frontend.md +++ /dev/null @@ -1,149 +0,0 @@ -# Frontend - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## FRONTEND-001 — Single Page Application Shell & Client Router - -- **Domain**: `frontend` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Main HTML5 SPA shell, DOM lifecycle initializers, tab navigation, and deep-link route handlers. - -### Evidence summary - -- `static/index.html` — `index.html` — Main SPA entry point containing modal roots and CSS bundles. -- `app.py` — `serve_index` — Serves index.html with dynamically generated CSP nonces. - -### Unknowns - -- Stale browser static cache if asset hashing is omitted during deployment. - -## FRONTEND-002 — Dynamic Theme, Color System & Custom Fonts - -- **Domain**: `frontend` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Supports dark/light themes, custom CSS variables, color picker controls, and user font uploads. - -### Evidence summary - -- `static/js/theme.js` — `applyTheme` — Applies custom HSL theme variables to DOM document root. -- `routes/font_routes.py` — `setup_font_routes` — Allows uploading and serving custom WOFF2 font files. - -### Unknowns - -- Flash of unstyled content (FOUC) on slow connections. - -## FRONTEND-003 — Window Manager, Tile Layout & Modal Control System - -- **Domain**: `frontend` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Manages draggable tool windows, snapped multi-tile viewports, modal dialog Z-ordering, and ESC key stacks. - -### Evidence summary - -- `static/js/modalManager.js` — `ModalManager` — Controls modal open/close transitions and focus trapping. -- `static/js/tileManager.js` — `TileManager` — Handles viewport split-pane grid arrangements. - -### Unknowns - -- Overlap artifacts when opening many simultaneous tool floating windows. - -## FRONTEND-004 — Global Keyboard Shortcuts & Accessibility Controls - -- **Domain**: `frontend` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides configurable hotkeys (Ctrl+K search, Esc close, Alt+1-9 tabs) and high-contrast accessibility options. - -### Evidence summary - -- `static/js/keyboard-shortcuts.js` — `initShortcuts` — Binds global keydown handlers for system shortcuts. -- `static/js/a11y.js` — `initA11y` — Applies ARIA roles and dyslexic font toggles. - -### Unknowns - -- Browser keybinding collisions with browser default hotkeys. - -## FRONTEND-005 — Markdown, LaTeX & Code Block Streaming Renderer - -- **Domain**: `frontend` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Parses incoming SSE markdown streams, renders KaTeX math formulas, syntax-highlighted code, and interactive runners. - -### Evidence summary - -- `static/js/markdown.js` — `renderMarkdown` — Converts markdown prose to HTML nodes with syntax highlighting. -- `static/js/streamingSegmenter.js` — `Segmenter` — Parses un-closed markdown fences during live stream. - -### Unknowns - -- DOM thrashing if streaming segmenter updates UI too frequently. - -## FRONTEND-006 — Interactive Tour & Guided Onboarding System - -- **Domain**: `frontend` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Presents interactive step-by-step feature tours and UI tooltip hints for new users. - -### Evidence summary - -- `static/js/tourHints.js` — `startTour` — Renders guided feature tour overlays over target UI elements. - -### Unknowns - -- Tour step misalignment if window is resized mid-tour. - -## FRONTEND-007 — Background Effects Prototyping Sandbox - -- **Domain**: `frontend` -- **Status**: `dead-code-candidate` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Standalone sandbox page for prototyping visual background animations, waves, and whirlpool effects. - -### Evidence summary - -- `app.py` — `serve_backgrounds` — Serves visual background sandbox HTML page route. -- `static/wave-variants.html` — `wave-variants.html` — Interactive background effect prototyping sandbox variant. - -### Unknowns - -- Route `/backgrounds` in app.py L918 attempts to serve `static/backgrounds.html` which is missing from disk; variant templates `wave-variants.html` and `whirlpool-variants.html` exist. diff --git a/docs/discovery/domains/media.md b/docs/discovery/domains/media.md deleted file mode 100644 index cc0622ceb..000000000 --- a/docs/discovery/domains/media.md +++ /dev/null @@ -1,170 +0,0 @@ -# Media - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## MEDIA-001 — Gallery Image Library & Album Operations - -- **Domain**: `media` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Organizes images into custom albums, provides grid browsing, tagging, and album metadata management. - -### Evidence summary - -- `routes/gallery/gallery_routes.py` — `@router.get('/api/gallery/library')` — Fetches image library list with tag filters. -- `static/js/gallery.js` — `initGallery` — Main gallery grid renderer and uploader. - -### Unknowns - -- Thumbnail generation overhead for high-resolution RAW camera images. - -## MEDIA-002 — Image Processing, AI Upscaling & Style Transfer - -- **Domain**: `media` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Executes local image enhancement, background removal, face sharpening, and AI upscaling operations. - -### Evidence summary - -- `routes/gallery/gallery_routes.py` — `@router.post('/api/gallery/ai-upscale')` — Runs RealESRGAN image upscaling. -- `routes/gallery/gallery_routes.py` — `@router.post('/api/image/remove-bg')` — Executes background removal pass. - -### Unknowns - -- High GPU memory allocation when upscaling 4K images. - -## MEDIA-003 — Interactive Image Canvas Editor & Persisted Drafts - -- **Domain**: `media` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides full multi-layer raster canvas editor, brush tools, transforms, masks, and draft project persistence. - -### Evidence summary - -- `routes/editor_draft_routes.py` — `setup_editor_draft_routes` — API routes for saving and loading canvas project drafts. -- `static/js/editor/history-panel.js` — `HistoryManager` — Canvas undo/redo stack manager. - -### Unknowns - -- Browser memory leak if multi-gigabyte layer undo buffers are kept indefinitely. - -## MEDIA-004 — Text-to-Speech (TTS) Synthesis Service - -- **Domain**: `media` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Synthesizes spoken audio from text using local Kokoro, EdgeTTS, or OpenAI TTS engines. - -### Evidence summary - -- `routes/tts_routes.py` — `@router.post('/synthesize')` — Synthesizes TTS audio clip. -- `services/tts/tts_service.py` — `TTSService` — Provider abstraction layer for audio speech generation. - -### Unknowns - -- Audio synthesis latency on CPU-only hardware setups. - -## MEDIA-005 — Speech-to-Text (STT) Audio Transcription Service - -- **Domain**: `media` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Transcribes user audio recordings into text using faster-whisper or local speech models. - -### Evidence summary - -- `routes/stt_routes.py` — `@router.post('/transcribe')` — Accepts multipart audio file and returns transcription text. -- `services/stt/stt_service.py` — `STTService` — Whisper audio transcription engine wrapper. - -### Unknowns - -- Missing ffmpeg system dependency prevents audio format decoding. - -## MEDIA-006 — Digital Signature Stamp Storage & Placement - -- **Domain**: `media` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Stores transparent PNG user signatures and stamps for placement onto PDF forms and documents. - -### Evidence summary - -- `routes/signature_routes.py` — `setup_signature_routes` — CRUD endpoints for managing user signature PNG stamps. - -### Unknowns - -- Cross-site scripting if signature image titles contain unescaped user input. - -## MEDIA-007 — Generated Image Artifact Route & MCP Integration - -- **Domain**: `media` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Serves generated AI artwork artifacts and integrates with image generation MCP server. - -### Evidence summary - -- `app.py` — `serve_generated_image` — Serves generated image artifacts with cache headers. -- `src/generated_images.py` — `resolve_generated_image_path` — Confines requested image path within artifacts directory. - -### Unknowns - -- Path traversal vulnerability if filename parameter is un-sanitized. - -## MEDIA-008 — Native MLX Image Bridge (macOS Apple Silicon) - -- **Domain**: `media` -- **Status**: `experimental` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires Apple Silicon, macOS tooling, and the compiled MLX bridge. - -### Purpose - -Native Apple Swift bridge for hardware-accelerated diffusion and MLX image colorization on macOS. - -### Evidence summary - -- `swift/odysseus-mlx-image-bridge/Package.swift` — `Package` — Swift package manifest for native MLX image bridge. -- `scripts/mlx_image_server.py` — `main` — Python daemon wrapping native Swift MLX binary. - -### Unknowns - -- Binary build requires Xcode command line tools build step (`build-macos-app.sh`). diff --git a/docs/discovery/domains/memory.md b/docs/discovery/domains/memory.md deleted file mode 100644 index 22b9f8b78..000000000 --- a/docs/discovery/domains/memory.md +++ /dev/null @@ -1,24 +0,0 @@ -# Memory - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## MEMORY-001 — Persistent Long-Term Memory & Vector Indexing - -- **Domain**: `memory` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Extracts facts, user preferences, and temporal memories from chat sessions into vector/relational storage. - -### Evidence summary - -- `routes/memory/memory_routes.py` — `@router.get('')` — Fetches long-term user memory timeline. -- `services/memory/memory_extractor.py` — `MemoryExtractor` — LLM-driven fact extraction from conversation transcripts. - -### Unknowns - -- Conflicting memory facts extracted from contradictory user prompts. diff --git a/docs/discovery/domains/model.md b/docs/discovery/domains/model.md deleted file mode 100644 index 1855c0d3f..000000000 --- a/docs/discovery/domains/model.md +++ /dev/null @@ -1,177 +0,0 @@ -# Model - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## MODEL-001 — Multi-Provider LLM Model Discovery & Metadata Management - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Discovers models from OpenAI, Anthropic, Ollama, vLLM, LMStudio, OpenRouter, and Google AI Studio endpoints. - -### Evidence summary - -- `routes/model_routes.py` — `@router.get('/api/models')` — Returns unified list of available models across providers. -- `src/model_discovery.py` — `ModelDiscovery.discover_all` — Queries connected provider endpoints for available model IDs. - -### Unknowns - -- Remote endpoint timeouts may slow down full discovery refresh. - -## MODEL-002 — Model Capability & Context Limits Detection - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Detects vision, tool calling, reasoning, and context window limits for connected model endpoints. - -### Evidence summary - -- `src/model_capabilities.py` — `get_model_capabilities` — Maps model names to vision and tool support flags. -- `src/endpoint_resolver.py` — `resolve_endpoint_headers` — Resolves auth headers and target URLs for model endpoints. - -### Unknowns - -- Incorrect context limit metadata for unlisted custom fine-tunes. - -## MODEL-003 — LLM Core Provider Communication & Fallback Routing - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Provider dispatch, header injection and fallback advancement are only observable against a reachable LLM provider endpoint; not exercised in this documentation pass. - -### Purpose - -Manages HTTP request dispatching, authorization header injection, and fallback provider routing for LLM calls. - -### Evidence summary - -- `src/llm_core.py` — `llm_call_async` — Non-streaming provider request dispatcher: resolves the endpoint, injects authorization headers and executes the HTTP call. -- `src/llm_core.py` — `llm_call_async_with_fallback` — Ordered fallback wrapper that retries llm_call_async across the configured candidate endpoints. -- `src/llm_core.py` — `stream_llm_with_fallback` — Ordered fallback wrapper for the streaming path; advances to the next candidate when a provider yields an empty completion. - -### Unknowns - -- Unexpected API changes in upstream third-party model providers. - -## MODEL-004 — Model Selection & Display Ordering Preferences - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Allows pinning, sorting, and hiding specific models in the UI selection dropdown. - -### Evidence summary - -- `routes/model_routes.py` — `@router.post('/order')` — Saves custom model display order preference. - -### Unknowns - -- Stale model IDs in custom order lists after model endpoints are removed. - -## MODEL-005 — Side-by-Side Model Comparison (A/B Testing) - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Enables dual-model side-by-side response evaluation, arena scoring, and latency comparison. - -### Evidence summary - -- `routes/compare/compare_routes.py` — `@router.post('/start')` — Starts a parallel dual-model comparison stream. -- `static/js/compare/index.js` — `initCompareView` — Renders side-by-side model chat panes. - -### Unknowns - -- High memory and network usage when streaming two model responses simultaneously. - -## MODEL-006 — GitHub Copilot Device Flow Authentication - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E2` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires an interactive GitHub Copilot OAuth device-flow account. - -### Purpose - -Authenticates with GitHub Copilot via OAuth device flow to use Copilot models directly. - -### Evidence summary - -- `routes/copilot_routes.py` — `setup_copilot_routes` — Builds the Copilot device-flow router at prefix /api/copilot, wiring _start_device_flow and _poll_device_flow. -- `routes/device_flow.py` — `create_device_flow_router` — Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix. -- `src/copilot.py` — `request_device_code` — Issues the GitHub device-code request that begins the Copilot OAuth device flow. -- `src/copilot.py` — `poll_access_token` — Polls GitHub for the access token once the user has authorized the device code. -- `tests/test_provider_device_flow_js.py` — `test_copilot_success_uses_complete_verification_uri` — Inspected unit test asserting the Copilot device-flow runner surfaces the complete verification URI returned by the backend. - -### Unknowns - -- Token expiration requires manual device re-authentication. - -## MODEL-007 — ChatGPT Subscription Device Flow Authentication - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E2` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires an interactive ChatGPT subscription OAuth flow. - -### Purpose - -Authenticates with ChatGPT Pro/Plus subscription tokens via device login flow. - -### Evidence summary - -- `routes/chatgpt_subscription_routes.py` — `setup_chatgpt_subscription_routes` — Builds the ChatGPT subscription device-flow router at prefix /api/chatgpt-subscription. -- `routes/device_flow.py` — `create_device_flow_router` — Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix. -- `src/chatgpt_subscription.py` — `request_device_code` — Issues the ChatGPT device-authorization request that begins the subscription OAuth device flow. -- `src/chatgpt_subscription.py` — `poll_device_auth` — Polls the ChatGPT device-authorization endpoint for completion using the stored device_auth_id and user_code. -- `tests/test_provider_device_flow_js.py` — `test_chatgpt_success_uses_plain_verification_uri` — Inspected unit test asserting the ChatGPT device-flow runner uses the plain verification URI rather than the Copilot complete-URI form. - -### Unknowns - -- Changes in OpenAI auth endpoint security challenges. - -## MODEL-008 — Embedding Model Lane & Vector Provider Setup - -- **Domain**: `model` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Configures local sentence-transformers, FastEmbed, or remote OpenAI embedding model lanes. - -### Evidence summary - -- `routes/embedding_routes.py` — `setup_embedding_routes` — Provides embedding provider configuration endpoints. -- `src/embeddings.py` — `EmbeddingManager` — Generates dense vector embeddings for RAG and memory. - -### Unknowns - -- First-time download of heavy PyTorch model weights on CPU-only machines. diff --git a/docs/discovery/domains/note.md b/docs/discovery/domains/note.md deleted file mode 100644 index e0c197428..000000000 --- a/docs/discovery/domains/note.md +++ /dev/null @@ -1,24 +0,0 @@ -# Note - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## NOTE-001 — Interactive Notes & Checklist Management - -- **Domain**: `note` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides Google Keep-style notes, rich markdown text, checklist items, pinning, color tags, and reminders. - -### Evidence summary - -- `routes/note/note_routes.py` — `@router.get('')` — Lists all user notes with pin and archive states. -- `static/js/notes.js` — `initNotesView` — Main interactive notes grid and modal manager. - -### Unknowns - -- Concurrent edits on note item checkboxes. diff --git a/docs/discovery/domains/platform.md b/docs/discovery/domains/platform.md deleted file mode 100644 index 037edddde..000000000 --- a/docs/discovery/domains/platform.md +++ /dev/null @@ -1,213 +0,0 @@ -# Platform - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## PLATFORM-001 — Application Initialization & Lifespan Management - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Orchestrates server startup, database table migration, background daemon initialization, and clean shutdown. - -### Evidence summary - -- `app.py` — `_lifespan` — FastAPI lifespan context manager executing startup tasks. -- `src/app_initializer.py` — `initialize_app` — Initializes app directories, DB schemas, and logging. - -### Unknowns - -- Un-handled exceptions during startup halt application launch. - -## PLATFORM-002 — System Health, Readiness & Version Monitoring APIs - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Exposes Liveness (/api/health), Readiness (/api/ready), App Version (/api/version), and Client Perf APIs. - -### Evidence summary - -- `app.py` — `readiness_check` — Performs system component integrity check. -- `src/readiness.py` — `check_readiness` — Checks database, storage, and key paths for read/write access. - -### Unknowns - -- Readiness check delays if verifying connectivity to offline remote endpoints. - -## PLATFORM-003 — Database Schema, Migrations & SQLite Persistence - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Defines core relational tables (users, tokens, tasks, sessions) and executes automated SQLite schema upgrades. - -### Evidence summary - -- `core/database.py` — `init_db` — Creates ORM tables and establishes connection pool. -- `scripts/update_database.py` — `run_migrations` — Applies missing schema columns and indices. - -### Unknowns - -- SQLite file lock contention under high concurrent write loads. - -## PLATFORM-004 — User Data Export & Import Backup Infrastructure - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Exports complete user workspace state (sessions, memory, skills, notes, presets) into a zip archive. - -### Evidence summary - -- `routes/backup_routes.py` — `setup_backup_routes` — Handles workspace data export and import upload unpack. -- `docs/backup-restore.md` — `Documentation` — Backup and restore operational documentation. - -### Unknowns - -- Corrupt archive files causing partial data restore. - -## PLATFORM-005 — File Cleanup & Storage Maintenance Engine - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Scans data directories for orphaned files, old uploads, temporary vision images, and frees disk space. - -### Evidence summary - -- `routes/cleanup/cleanup_routes.py` — `@router.get('/preview')` — Previews reclaimable disk space across storage directories. -- `src/cleanup_service.py` — `CleanupService` — Executes filesystem purge of orphaned asset files. - -### Unknowns - -- Deletes files uploaded in active sessions if retention window is set too short. - -## PLATFORM-006 — System Health & RAG Diagnostic Suite - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Executes real-time integrity diagnostics across ChromaDB, SearXNG, local models, and network interfaces. - -### Evidence summary - -- `routes/diagnostics_routes.py` — `setup_diagnostics_routes` — Runs subsystem health check suite. -- `src/service_health.py` — `collect_health_status` — Inspects vector database, email, search, and local provider status. - -### Unknowns - -- Diagnostic timeout if external search provider is unreachable. - -## PLATFORM-007 — Desktop CLI Utilities & Shell Integration Tools - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides command-line interface tools (`odysseus`, `odysseus-mcp`, `odysseus-mail`) for terminal usage. - -### Evidence summary - -- `scripts/_lib/cli.py` — `main` — Shared CLI framework for terminal helper commands. -- `scripts/odysseus` — `odysseus` — Main terminal launcher script. - -### Unknowns - -- Outdated CLI scripts if backend API schemas change. - -## PLATFORM-008 — Desktop Companion App Integration - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides API routes and pairing mechanisms for the native macOS/desktop menu bar companion app. - -### Evidence summary - -- `companion/routes.py` — `setup_companion_routes` — Endpoints for pairing and status sync with desktop companion. -- `companion/pairing.py` — `PairingManager` — Generates and validates companion pairing codes. - -### Unknowns - -- Pairing code expiration timing window. - -## PLATFORM-009 — Docker Containerization & GPU Hardware Manifests - -- **Domain**: `platform` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires Docker GPU pass-through and compatible host drivers. - -### Purpose - -Provides multi-stage Dockerfile and Docker Compose manifests for CPU, NVIDIA CUDA, and AMD ROCm GPUs. - -### Evidence summary - -- `Dockerfile` — `multi-stage-build` — Multi-stage container build environment. -- `docker-compose.gpu-nvidia.yml` — `nvidia-gpu-manifest` — NVIDIA GPU pass-through container specification. -- `scripts/check-docker-gpu.sh` — `check-docker-gpu` — Automated diagnostic test script for host NVIDIA GPU passthrough. - -### Unknowns - -- Driver version incompatibility with host NVIDIA/AMD kernel drivers. - -## PLATFORM-010 — Legacy FAISS Vector Index Migration Script - -- **Domain**: `platform` -- **Status**: `legacy` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Legacy utility script to migrate older FAISS vector indices into ChromaDB. - -### Evidence summary - -- `scripts/migrate_faiss_to_chroma.py` — `migrate_faiss` — Reads FAISS vector index files and writes to ChromaDB collection. - -### Unknowns - -- Superseded by native ChromaDB vector index pipeline. diff --git a/docs/discovery/domains/research.md b/docs/discovery/domains/research.md deleted file mode 100644 index 2df5a4b75..000000000 --- a/docs/discovery/domains/research.md +++ /dev/null @@ -1,86 +0,0 @@ -# Research - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## RESEARCH-001 — Deep Research Execution Engine & SSE Progress Streaming - -- **Domain**: `research` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Executes multi-step recursive deep research tasks, web page scraping, synthesis, and streams live progress. - -### Evidence summary - -- `routes/research/research_routes.py` — `@router.post('/api/research/start')` — Initiates deep research job. -- `src/deep_research.py` — `DeepResearchEngine` — Recursive search and summary crawler. - -### Unknowns - -- High memory consumption when parsing multi-megabyte HTML target pages. - -## RESEARCH-002 — Research Library, Detail View & Image Controls - -- **Domain**: `research` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Stores completed research reports, generated diagrams, reference links, and manages image visibility. - -### Evidence summary - -- `routes/research/research_routes.py` — `@router.get('/api/research/library')` — Returns all saved research reports. - -### Unknowns - -- Orphaned report files if storage directory is modified out-of-band. - -## RESEARCH-003 — Web Search Engine Integration (SearXNG & Multi-Provider) - -- **Domain**: `research` -- **Status**: `verified` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires an active SearXNG instance or external search API provider. - -### Purpose - -Queries SearXNG, DuckDuckGo, or Google Search instances to retrieve web search snippets. - -### Evidence summary - -- `routes/search_routes.py` — `setup_search_routes` — Defines /api/search, /api/search/config, and /api/search/query endpoints. -- `src/search/core.py` — `SearchEngine` — Compatibility module aliasing services.search.core. -- `tests/test_search_ranking.py` — `test_news_queries_prefer_news_sources_over_sports_and_social_results` — Tests search result domain ranking and scoring. - -### Unknowns - -- Search provider IP throttling or rate-limiting. - -## RESEARCH-004 — Research Result Peeking & Topic Spinoff Generation - -- **Domain**: `research` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Extracts preliminary research snippets and spawns child research sessions focused on specific sub-topics. - -### Evidence summary - -- `routes/research/research_routes.py` — `@router.post('/api/research/spinoff/{session_id}')` — Spawns child research session for specific query. - -### Unknowns - -- Deep recursion tree depth when spawning multiple nested spinoffs. diff --git a/docs/discovery/domains/security.md b/docs/discovery/domains/security.md deleted file mode 100644 index 2d07dae22..000000000 --- a/docs/discovery/domains/security.md +++ /dev/null @@ -1,154 +0,0 @@ -# Security - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## SECURITY-001 — Authentication, Session Cookies & User Management - -- **Domain**: `security` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Handles bcrypt password hashing, session cookie issuance, authentication enforcement, and user administration. - -### Evidence summary - -- `routes/auth_routes.py` — `@router.post('/login')` — Authenticates credentials and sets session cookie. -- `core/auth.py` — `AuthManager` — Handles user creation, password verification, and session tokens. - -### Unknowns - -- Cookie session hijack if deployed over unencrypted HTTP without HTTPS cookie flags. - -## SECURITY-002 — System Vault Encrypted Secret Storage - -- **Domain**: `security` -- **Status**: `verified` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: pending — Requires installed Bitwarden CLI (`bw`) executable. - -### Purpose - -Encrypts API keys, passphrases, and third-party secrets on disk using AES-GCM / PBKDF2 key derivation. - -### Evidence summary - -- `routes/vault_routes.py` — `setup_vault_routes` — Admin routes for vault configuration, login, unlock, lock, and logout. -- `src/secret_storage.py` — `SecretStorage` — Fernet symmetric key DB secret encryption. -- `tests/test_vault_password_not_in_argv.py` — `test_bw_password_not_in_argv` — Verifies master password is fed via stdin and never appears in process argv. - -### Unknowns - -- Loss of vault master passphrase renders all encrypted secrets permanently unrecoverable. - -## SECURITY-003 — API Token Management & Scope Access Control - -- **Domain**: `security` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Generates scoped API bearer tokens (read/write/admin) for external tool and script authentication. - -### Evidence summary - -- `routes/api_token_routes.py` — `setup_api_token_routes` — Exposes API token creation, scope assignment, and revocation. -- `core/database.py` — `ApiToken` — SQLAlchemy ORM schema for API tokens and permissions. - -### Unknowns - -- Leaked API bearer tokens with excessive permission scopes. - -## SECURITY-004 — Prompt Security & Injection Defense Engine - -- **Domain**: `security` -- **Status**: `verified` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Scans system prompts and external inputs for prompt injection attempts, jailbreaks, and sensitive data leaks. - -### Evidence summary - -- `src/prompt_security.py` — `untrusted_context_message` — Wraps untrusted context with guard delimiters and sets metadata.trusted = False. -- `src/tool_security.py` — `NON_ADMIN_BLOCKED_TOOLS` — Enforces tool execution safety for non-admin user roles. -- `tests/test_skill_index_prompt_injection.py` — `test_skill_index` — Verifies skill index descriptions cannot leak into trusted system prompts. -- `tests/test_tool_output_prompt_injection.py` — `test_tool_output` — Tool output injection guards. - -### Unknowns - -- False positives blocking legitimate complex coding or security prompts. - -## SECURITY-005 — URL & Path Confinement Security Guards - -- **Domain**: `security` -- **Status**: `verified` -- **Evidence Maturity**: `E1` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Prevents SSRF attacks and path traversal by validating target IP addresses and resolving symlinks. - -### Evidence summary - -- `src/url_safety.py` — `check_outbound_url` — Rejects non-HTTP(S) schemes, link-local, cloud metadata SSRF addresses. -- `src/url_security.py` — `validate_public_http_url` — Validates public-facing endpoints. -- `tests/test_url_safety.py` — `test_url_safety` — Scheme validation, cloud metadata SSRF rejection, IP classification. -- `tests/test_tool_path_confinement.py` — `test_path_confinement` — Path traversal checks. -- `tests/test_workspace_confine.py` — `test_workspace_confine` — Workspace confinement checks. - -### Unknowns - -- DNS rebinding attacks if IP address is re-resolved post-validation. - -## SECURITY-006 — HTTP Security Headers Middleware - -- **Domain**: `security` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Injects standard OWASP HTTP security headers (CSP, HSTS, X-Content-Type-Options, X-Frame-Options). - -### Evidence summary - -- `core/middleware.py` — `SecurityHeadersMiddleware` — Sets strict security headers and CSP nonces on HTTP responses. - -### Unknowns - -- Strict Content Security Policy (CSP) blocking third-party embedded web resources. - -## SECURITY-007 — Admin System Data Wipe ('Danger Zone') - -- **Domain**: `security` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Provides administrative reset operations to wipe sessions, cache, uploaded files, or factory reset state. - -### Evidence summary - -- `routes/admin_wipe/admin_wipe_routes.py` — `@router.delete('/wipe/{kind}')` — Executes systemic data wipe based on requested scope. - -### Unknowns - -- Accidental catastrophic data loss if triggered without user confirmation. diff --git a/docs/discovery/domains/skill.md b/docs/discovery/domains/skill.md deleted file mode 100644 index 43fb72180..000000000 --- a/docs/discovery/domains/skill.md +++ /dev/null @@ -1,24 +0,0 @@ -# Skill - -Features in this document are generated from [`../feature-catalog.json`](../feature-catalog.json), the canonical inventory. - -## SKILL-001 — Dynamic Skill Management & Code Execution Engine - -- **Domain**: `skill` -- **Status**: `verified` -- **Evidence Maturity**: `E0` -- **Commit Verified**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Runtime Validation**: not-required — No separate environment-dependent runtime validation was identified during this documentation pass. - -### Purpose - -Allows users to create, import, edit, test, and execute custom Python/Markdown skills dynamically. - -### Evidence summary - -- `routes/skills_routes.py` — `setup_skills_routes` — Exposes CRUD and remote import routes for user skills. -- `services/memory/skills.py` — `SkillsManager` — Handles skill storage, parsing, and execution. - -### Unknowns - -- Arbitrary code execution risks if skill import URL is untrusted. diff --git a/docs/discovery/feature-catalog.json b/docs/discovery/feature-catalog.json deleted file mode 100644 index 93cdc6fd0..000000000 --- a/docs/discovery/feature-catalog.json +++ /dev/null @@ -1,4797 +0,0 @@ -[ - { - "id": "CHAT-001", - "domain": "chat", - "name": "Core Chat Streaming & SSE Message Generation", - "purpose": "Handles real-time Server-Sent Events (SSE) chat streaming, token rendering, and model response generation.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chatStream.js", - "static/js/chat.js", - "static/js/streamingRenderer.js" - ], - "backend_entrypoints": [ - "routes/chat_routes.py:chat_stream", - "routes/chat_helpers.py:build_chat_context", - "src/chat_handler.py:ChatHandler.preprocess_message", - "src/chat_processor.py:ChatProcessor.build_context_preface", - "src/llm_core.py:stream_llm_with_fallback", - "src/agent_loop.py:stream_agent_loop" - ], - "routes": [ - "/api/chat_stream", - "/api/chat" - ], - "configuration": [ - "OPENAI_API_KEY", - "OLLAMA_BASE_URL", - "REQUEST_TIMEOUT" - ], - "persistence": [ - "SESSIONS_FILE", - "DATA_DIR/sessions/" - ], - "dependencies": [ - "fastapi", - "starlette.responses.StreamingResponse", - "httpx" - ], - "tests": [ - "tests/test_chat_metrics.py", - "tests/test_resend_message_nondestructive.py" - ], - "documentation": [ - "docs/chat.webm", - "README.md" - ], - "risks": [ - "Stream interruption on connection drops requires retry logic." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/chat_routes.py", - "symbol": "chat_stream", - "line_range": "L702-L1870", - "explanation": "POST /api/chat_stream SSE endpoint; builds the shared chat context, then dispatches to the chat-mode or agent-mode streaming path." - }, - { - "path": "routes/chat_helpers.py", - "symbol": "build_chat_context", - "line_range": "L669-L863", - "explanation": "Shared context builder invoked by chat_stream; runs message preprocessing and assembles the memory/RAG/web context preface." - }, - { - "path": "src/chat_handler.py", - "symbol": "ChatHandler.preprocess_message", - "line_range": "L123-L315", - "explanation": "Message preprocessing (attachments, URLs, tool preprocessing) reached from build_chat_context via routes/chat_helpers.py:preprocess." - }, - { - "path": "src/chat_processor.py", - "symbol": "ChatProcessor.build_context_preface", - "line_range": "L263-L493", - "explanation": "Builds the retrieval and web-source context preface injected into the streamed request." - }, - { - "path": "src/llm_core.py", - "symbol": "stream_llm_with_fallback", - "line_range": "L2794-L2895", - "explanation": "Chat-mode streaming dispatcher called from chat_stream; wraps stream_llm with an ordered provider fallback chain." - }, - { - "path": "src/llm_core.py", - "symbol": "stream_llm", - "line_range": "L2131-L2151", - "explanation": "Per-request streaming entry wrapped by stream_llm_with_fallback; acquires the local model slot and delegates to _stream_llm_inner." - }, - { - "path": "src/agent_loop.py", - "symbol": "stream_agent_loop", - "line_range": "L3079-L5248", - "explanation": "Agent-mode streaming path called from chat_stream when the request selects agent mode." - }, - { - "path": "tests/test_chat_metrics.py", - "symbol": "test_stream_llm_passes_through_llamacpp_timings", - "line_range": "L129-L136", - "explanation": "Inspected unit test asserting stream_llm forwards backend generation timings into the emitted metrics chunk." - }, - { - "path": "tests/test_resend_message_nondestructive.py", - "symbol": "test_resend_message_does_not_truncate_by_default", - "line_range": "L23-L36", - "explanation": "Inspected unit test asserting the frontend resend path does not truncate prior conversation turns." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E2", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires access to a live LLM provider endpoint (OpenAI API key or local Ollama server)." - } - }, - { - "id": "CHAT-002", - "domain": "chat", - "name": "Session Management & Conversation State", - "purpose": "Manages session creation, listing, switching, renaming, and persistence of conversation metadata.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/sessions.js", - "static/js/sidebar-layout.js" - ], - "backend_entrypoints": [ - "routes/session_routes.py:setup_session_routes", - "core/session_manager.py:SessionManager" - ], - "routes": [ - "/api/sessions", - "/api/sessions/{session_id}" - ], - "configuration": [ - "SESSIONS_FILE", - "DATA_DIR/odysseus.db" - ], - "persistence": [ - "DATA_DIR/sessions.json", - "SQLite session table" - ], - "dependencies": [ - "sqlite3", - "pydantic" - ], - "tests": [ - "tests/test_session_manager.py", - "tests/test_session_routes_utcnow.py" - ], - "documentation": [ - "docs/setup.md" - ], - "risks": [ - "Concurrent file writes to sessions.json under high load." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/session_routes.py", - "symbol": "@router.get('/api/sessions')", - "line_range": "L150-L210", - "explanation": "Lists active sessions filtered by user owner scope." - }, - { - "path": "core/session_manager.py", - "symbol": "SessionManager", - "line_range": "L30-L150", - "explanation": "Provides thread-safe session storage operations." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CHAT-003", - "domain": "chat", - "name": "Chat History & Message Editing/Truncation", - "purpose": "Provides history retrieval, message content updating, message deletion, and history branch truncation.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chat.js", - "static/js/sessions.js" - ], - "backend_entrypoints": [ - "routes/history/history_routes.py:setup_history_routes" - ], - "routes": [ - "/api/history/{session_id}", - "/api/session/{session_id}/truncate", - "/api/session/{session_id}/edit-message" - ], - "configuration": [ - "MAX_HISTORY_MESSAGES" - ], - "persistence": [ - "DATA_DIR/sessions/" - ], - "dependencies": [ - "sqlite3", - "fastapi" - ], - "tests": [ - "tests/test_history_db_fallback_hidden.py", - "tests/test_truncate_message_count_regression.py" - ], - "documentation": [ - "docs/attachments.md" - ], - "risks": [ - "Truncating messages re-indexes context window and clears cached tool calls." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/history/history_routes.py", - "symbol": "@router.get('/api/history/{session_id}')", - "line_range": "L178-L230", - "explanation": "Fetches message history timeline for a session." - }, - { - "path": "routes/history_routes.py", - "symbol": "_sys.modules[__name__] = _canonical", - "line_range": "L1-L17", - "explanation": "Backward-compatibility shim module." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CHAT-004", - "domain": "chat", - "name": "File & Multimodal Attachment Handling", - "purpose": "Handles uploading, mime validation, image preview, vision encoding, and file attachments in chat messages.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/fileHandler.js", - "static/js/chat.js" - ], - "backend_entrypoints": [ - "routes/upload_routes.py:setup_upload_routes", - "src/upload_handler.py:UploadHandler" - ], - "routes": [ - "/api/upload", - "/api/upload/{file_id}/vision", - "/api/upload/cleanup" - ], - "configuration": [ - "MAX_UPLOAD_SIZE_MB", - "ALLOWED_UPLOAD_EXTENSIONS" - ], - "persistence": [ - "DATA_DIR/uploads/" - ], - "dependencies": [ - "pillow", - "python-magic" - ], - "tests": [ - "tests/test_upload_handler_atomicity.py", - "tests/test_upload_routes_owner_scope.py" - ], - "documentation": [ - "docs/attachments.md" - ], - "risks": [ - "Large file uploads may consume server disk space if cleanup task fails." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/upload_routes.py", - "symbol": "@router.post('')", - "line_range": "L257-L310", - "explanation": "Accepts multi-part file uploads and generates vision metadata." - }, - { - "path": "src/upload_handler.py", - "symbol": "UploadHandler.save_file", - "line_range": "L50-L140", - "explanation": "Validates upload size and atomicity on disk." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CHAT-005", - "domain": "chat", - "name": "Chat Message Search", - "purpose": "Enables full-text keyword search across stored chat messages and sessions.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/search-chat.js", - "static/js/search.js" - ], - "backend_entrypoints": [ - "routes/search_routes.py:setup_search_routes", - "src/session_search.py:search_sessions" - ], - "routes": [ - "/api/search/chat" - ], - "configuration": [ - "SEARCH_INDEX_CACHE_TTL" - ], - "persistence": [ - "SQLite FTS / session indices" - ], - "dependencies": [ - "sqlite3" - ], - "tests": [ - "tests/test_session_search.py", - "tests/test_session_search_batch_fetch.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Full table scans on un-indexed text columns for very large databases." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/search_routes.py", - "symbol": "setup_search_routes", - "line_range": "L30-L80", - "explanation": "Registers chat message search endpoint." - }, - { - "path": "src/session_search.py", - "symbol": "search_sessions", - "line_range": "L20-L90", - "explanation": "Executes query matching against session transcripts." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CHAT-006", - "domain": "chat", - "name": "System Prompts & Preset Management", - "purpose": "Provides creation, selection, and customization of system prompt presets for chat sessions.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/presets.js", - "static/js/settings.js" - ], - "backend_entrypoints": [ - "routes/preset_routes.py:setup_preset_routes", - "src/preset_manager.py:PresetManager" - ], - "routes": [ - "/api/presets", - "/api/presets/{preset_id}" - ], - "configuration": [ - "PRESETS_FILE" - ], - "persistence": [ - "DATA_DIR/presets.json" - ], - "dependencies": [ - "pydantic", - "json" - ], - "tests": [ - "tests/cli/test_preset_cli_store.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Invalid JSON syntax in user presets file can corrupt preset loading." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/preset_routes.py", - "symbol": "setup_preset_routes", - "line_range": "L30-L110", - "explanation": "API routes for listing and modifying system prompt presets." - }, - { - "path": "src/preset_manager.py", - "symbol": "PresetManager", - "line_range": "L15-L100", - "explanation": "Disk-backed manager for prompt presets." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CHAT-007", - "domain": "chat", - "name": "Emoji Rendering & Twemoji SVG Proxy", - "purpose": "Proxies Twemoji SVG icons locally to render flat SVG emojis in message text without external CDN dependencies.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/emojiPicker.js", - "static/js/emojiShortcodes.js" - ], - "backend_entrypoints": [ - "routes/emoji_routes.py:setup_emoji_routes" - ], - "routes": [ - "/api/emoji/{code}.svg" - ], - "configuration": [ - "EMOJI_CACHE_DIR" - ], - "persistence": [ - "DATA_DIR/emoji_cache/" - ], - "dependencies": [ - "httpx", - "fastapi.responses.Response" - ], - "tests": [ - "tests/test_censor_pref_js.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "First request fetches SVG from remote CDN before caching locally." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/emoji_routes.py", - "symbol": "setup_emoji_routes", - "line_range": "L20-L100", - "explanation": "Serves locally cached Twemoji SVGs." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CHAT-008", - "domain": "chat", - "name": "Input History Recall (Arrow Up)", - "purpose": "Allows users to cycle through previously sent prompt messages in the chat composer input using Arrow-Up/Down keys.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/composerArrowUpRecall.js", - "static/js/chat.js" - ], - "backend_entrypoints": [ - "None (Pure client-side state feature)" - ], - "routes": [], - "configuration": [ - "localStorage key: chat_composer_history" - ], - "persistence": [ - "Browser localStorage" - ], - "dependencies": [ - "Vanilla JavaScript DOM keyboard listeners" - ], - "tests": [], - "documentation": [ - "README.md" - ], - "risks": [ - "Client-side browser storage limits." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/js/composerArrowUpRecall.js", - "symbol": "initComposerRecall", - "line_range": "L1-L80", - "explanation": "Listens for ArrowUp keypress on composer textarea." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CHAT-009", - "domain": "chat", - "name": "Context Window Compaction & Truncation", - "purpose": "Compacts session transcript history when prompt size exceeds context limits using summarization.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chat.js" - ], - "backend_entrypoints": [ - "routes/history/history_routes.py:compact_session", - "src/context_compactor.py:compact_context" - ], - "routes": [ - "/api/session/{session_id}/compact", - "/api/session/{session_id}/context" - ], - "configuration": [ - "MAX_CONTEXT_TOKENS", - "COMPACTION_THRESHOLD" - ], - "persistence": [ - "DATA_DIR/sessions/" - ], - "dependencies": [ - "tiktoken", - "fastapi" - ], - "tests": [ - "tests/test_context_compactor.py" - ], - "documentation": [ - "specs/architecture-runtime-inventory.md" - ], - "risks": [ - "Aggressive compaction may discard subtle user instructions." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/history/history_routes.py", - "symbol": "@router.post('/api/session/{session_id}/compact')", - "line_range": "L751-L790", - "explanation": "Triggers context summarization and compaction." - }, - { - "path": "src/context_compactor.py", - "symbol": "compact_context", - "line_range": "L25-L110", - "explanation": "Executes context token pruning and summary generation." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "AGENT-001", - "domain": "agent", - "name": "Autonomous Agent Loop & Tool Execution Engine", - "purpose": "Executes multi-step agent reasoning loops, tool invocation parsing, and automated response generation.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chatStream.js" - ], - "backend_entrypoints": [ - "src/agent_loop.py:run_agent_loop", - "src/tool_execution.py:execute_tool_call" - ], - "routes": [ - "/api/chat/stream" - ], - "configuration": [ - "MAX_AGENT_STEPS", - "ENABLE_TOOL_EXECUTION" - ], - "persistence": [ - "Session transcript tool calls" - ], - "dependencies": [ - "asyncio", - "pydantic" - ], - "tests": [ - "tests/test_tool_policy.py", - "tests/test_unknown_tool_calls.py" - ], - "documentation": [ - "specs/architecture-runtime-inventory.md" - ], - "risks": [ - "Infinite tool loop if termination condition fails." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/agent_loop.py", - "symbol": "run_agent_loop", - "line_range": "L40-L210", - "explanation": "Core loop evaluating model tool requests and executing handlers." - }, - { - "path": "src/tool_execution.py", - "symbol": "execute_tool_call", - "line_range": "L30-L150", - "explanation": "Dispatches tool invocation requests to underlying tool handlers." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "AGENT-002", - "domain": "agent", - "name": "Scheduled Tasks & Event Bus Dispatcher", - "purpose": "Schedules background recurring or delayed tasks, emits event bus triggers, and executes automated flows.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/tasks.js" - ], - "backend_entrypoints": [ - "routes/task_routes.py:setup_task_routes", - "src/task_scheduler.py:TaskScheduler", - "src/event_bus.py" - ], - "routes": [ - "/api/tasks", - "/api/tasks/{task_id}/run", - "/api/tasks/{task_id}/pause", - "/api/tasks/{task_id}/resume" - ], - "configuration": [ - "TASK_SCHEDULER_INTERVAL" - ], - "persistence": [ - "DATA_DIR/tasks.db" - ], - "dependencies": [ - "apscheduler", - "sqlite3" - ], - "tests": [ - "tests/test_task_scheduler_cancel.py", - "tests/test_task_chain_owner_scope.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Task execution failure handling on system restart." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/task_routes.py", - "symbol": "@router.get('')", - "line_range": "L341-L370", - "explanation": "Fetches active scheduled tasks." - }, - { - "path": "src/task_scheduler.py", - "symbol": "TaskScheduler", - "line_range": "L40-L280", - "explanation": "Async task scheduler dispatching cron and delay triggers." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "AGENT-003", - "domain": "agent", - "name": "Webhook Event Subscriptions & Trigger Processing", - "purpose": "Manages incoming/outgoing webhook subscriptions, endpoint authentication tokens, and event triggers.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/admin.js" - ], - "backend_entrypoints": [ - "routes/webhook_routes.py:setup_webhook_routes", - "src/webhook_manager.py:WebhookManager" - ], - "routes": [ - "/api/webhooks", - "/v1/chat", - "/api/webhooks/{webhook_id}/test" - ], - "configuration": [ - "WEBHOOK_SECRET_KEY" - ], - "persistence": [ - "DATA_DIR/webhooks.db" - ], - "dependencies": [ - "httpx", - "sqlite3" - ], - "tests": [ - "tests/test_webhook_ssrf_resilience.py", - "tests/test_webhook_trigger_auth_exempt.py" - ], - "documentation": [ - "docs/pr-blocker-audit.md" - ], - "risks": [ - "SSRF risks when contacting external webhook URLs if unvalidated." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/webhook_routes.py", - "symbol": "@router.get('/webhooks')", - "line_range": "L71-L94", - "explanation": "Returns list of registered webhooks." - }, - { - "path": "src/webhook_manager.py", - "symbol": "WebhookManager", - "line_range": "L30-L160", - "explanation": "Handles payload delivery and signature verification." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "AGENT-004", - "domain": "agent", - "name": "Assistant Settings, Task Check-Ins & Background Job Monitor", - "purpose": "Manages per-user assistant sessions and scheduled check-in settings, drains background job completions, and retains a legacy no-op activity logging shim.", - "status": "partial", - "frontend_entrypoints": [ - "static/js/assistant.js" - ], - "backend_entrypoints": [ - "routes/assistant_routes.py:setup_assistant_routes", - "src/assistant_log.py", - "src/bg_monitor.py" - ], - "routes": [ - "/api/assistant/session", - "/api/assistant/settings", - "/api/assistant/run/{task_id}", - "/api/assistant/run-status/{task_id}", - "/api/assistant/available-timezones" - ], - "configuration": [ - "CrewMember.timezone (IANA tz name driving check-in scheduling)", - "CrewMember.model", - "CrewMember.enabled_tools", - "ScheduledTask.scheduled_time / ScheduledTask.status", - "src/bg_monitor.py:POLL_INTERVAL_S", - "src/bg_monitor.py:_FOLLOWUP_MAX_ROUNDS" - ], - "persistence": [ - "CrewMember (core/database.py, is_default_assistant singleton per owner)", - "Session pinned via CrewMember.session_id", - "ScheduledTask (per-owner check-in rows)", - "TaskRun (most recent run status for run-status polling)" - ], - "dependencies": [ - "logging", - "asyncio" - ], - "tests": [ - "tests/cli/test_logs_cli_resolve_nonstring.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Assistant seeding is owner-scoped: routes/assistant_routes.py documents that reaching an /api/assistant route under a synthetic owner previously seeded a duplicate CrewMember plus Morning/Midday/Evening tasks that double-fired; correctness now depends on RESERVED_USERNAMES staying complete.", - "Check-in next_run values are recomputed from CrewMember.timezone through compute_next_run, so a missing or stale IANA timezone shifts when check-ins fire.", - "Background follow-up continuation is retry-until-success: _run_followup returns False to defer and mark_followed_up() runs only after a successful agent run, so a persistently failing continuation is retried every POLL_INTERVAL_S tick without a bounded attempt count." - ], - "unknowns": [ - "Route `/api/assistant/logs` cited in legacy docs is absent from assistant router.", - "Existing unit test `tests/cli/test_logs_cli_resolve_nonstring.py` tests CLI target-name resolution logic, not active assistant routes or bg_monitor execution loop." - ], - "evidence": [ - { - "path": "routes/assistant_routes.py", - "symbol": "setup_assistant_routes", - "line_range": "L80-L326", - "explanation": "Active assistant session, settings, manual check-in, run-status and timezone-list endpoints, including the owner-scoping guards." - }, - { - "path": "src/bg_monitor.py", - "symbol": "_drain_agent", - "line_range": "L28-L72", - "explanation": "Runs the agent loop headless against a session to produce the background-job follow-up turn." - }, - { - "path": "src/bg_monitor.py", - "symbol": "_run_followup", - "line_range": "L75-L132", - "explanation": "Drains completed background jobs and auto-continues the owning session, deferring while a live turn is in progress." - }, - { - "path": "src/assistant_log.py", - "symbol": "log_to_assistant", - "line_range": "L34-L48", - "explanation": "Legacy no-op activity logging shim retained for existing callers; documented as inactive rather than as current behaviour." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "AGENT-005", - "domain": "agent", - "name": "Model Context Protocol (MCP) Server Integration", - "purpose": "Integrates external MCP servers over stdio/SSE to expand agent capabilities dynamically.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/settings.js", - "static/js/admin.js" - ], - "backend_entrypoints": [ - "routes/mcp_routes.py:setup_mcp_routes", - "src/mcp_manager.py:McpManager", - "mcp_servers/" - ], - "routes": [ - "/api/mcp/servers", - "/api/mcp/tools", - "/api/mcp/connect" - ], - "configuration": [ - "MCP_CONFIG_PATH" - ], - "persistence": [ - "DATA_DIR/mcp_config.json" - ], - "dependencies": [ - "mcp", - "asyncio" - ], - "tests": [ - "tests/test_mcp_param_hint_hardening.py", - "tests/cli/test_mcp_cli_json.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Subprocess leaks if external MCP server process fails to terminate clean." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/mcp_routes.py", - "symbol": "setup_mcp_routes", - "line_range": "L60-L240", - "explanation": "Exposes management endpoints for external MCP servers." - }, - { - "path": "src/mcp_manager.py", - "symbol": "McpManager", - "line_range": "L40-L300", - "explanation": "Manages MCP server subprocess lifecycles." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "AGENT-006", - "domain": "agent", - "name": "AI Interaction Tools & Pipeline Orchestration", - "purpose": "Provides specialized AI interaction tools for agent self-debugging, debate, and multi-model collaboration.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chatStream.js" - ], - "backend_entrypoints": [ - "src/ai_interaction.py", - "src/action_intents.py", - "src/builtin_actions.py" - ], - "routes": [ - "Implicit agent tool calls" - ], - "configuration": [ - "MAX_PIPELINE_TURNS" - ], - "persistence": [ - "Session transcript state" - ], - "dependencies": [ - "asyncio" - ], - "tests": [ - "tests/test_builtin_actions_nonstring.py", - "tests/test_model_interaction_registry.py" - ], - "documentation": [ - "specs/architecture-runtime-inventory.md" - ], - "risks": [ - "High API token consumption during extended agent debates." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/ai_interaction.py", - "symbol": "init_ai_interaction_tools", - "line_range": "L30-L140", - "explanation": "Registers specialized multi-agent interaction primitives." - }, - { - "path": "src/builtin_actions.py", - "symbol": "execute_builtin_action", - "line_range": "L20-L90", - "explanation": "Executes pre-built action intent sequences." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "AGENT-007", - "domain": "agent", - "name": "Subprocess & Background Job Execution Tools", - "purpose": "Provides sandboxed bash/shell tool execution capabilities with output streaming and background tracking.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chatStream.js", - "static/js/tasks.js" - ], - "backend_entrypoints": [ - "src/agent_tools/subprocess_tools.py", - "src/agent_tools/bg_job_tools.py", - "src/bg_jobs.py" - ], - "routes": [ - "Implicit agent tool calls" - ], - "configuration": [ - "ALLOW_SHELL_EXECUTION", - "SANDBOX_DIR" - ], - "persistence": [ - "DATA_DIR/bg_jobs/" - ], - "dependencies": [ - "subprocess", - "asyncio" - ], - "tests": [ - "tests/test_bg_job_tools.py", - "tests/test_task_shell_tools.py" - ], - "documentation": [ - "THREAT_MODEL.md" - ], - "risks": [ - "Arbitrary shell command execution permissions if sandbox confinement fails." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/agent_tools/subprocess_tools.py", - "symbol": "run_command", - "line_range": "L25-L120", - "explanation": "Executes shell commands in background/foreground." - }, - { - "path": "src/bg_jobs.py", - "symbol": "JobManager", - "line_range": "L30-L150", - "explanation": "Tracks async background subprocess tasks." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MODEL-001", - "domain": "model", - "name": "Multi-Provider LLM Model Discovery & Metadata Management", - "purpose": "Discovers models from OpenAI, Anthropic, Ollama, vLLM, LMStudio, OpenRouter, and Google AI Studio endpoints.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/models.js", - "static/js/modelPicker.js" - ], - "backend_entrypoints": [ - "routes/model_routes.py:setup_model_routes", - "src/model_discovery.py:ModelDiscovery" - ], - "routes": [ - "/api/models", - "/api/models/active", - "/api/model-endpoints" - ], - "configuration": [ - "OPENAI_API_KEY", - "OLLAMA_BASE_URL", - "ANTHROPIC_API_KEY" - ], - "persistence": [ - "DATA_DIR/model_endpoints.json" - ], - "dependencies": [ - "httpx", - "pydantic" - ], - "tests": [ - "tests/test_model_routes.py", - "tests/test_provider_classification.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Remote endpoint timeouts may slow down full discovery refresh." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/model_routes.py", - "symbol": "@router.get('/api/models')", - "line_range": "L100-L180", - "explanation": "Returns unified list of available models across providers." - }, - { - "path": "src/model_discovery.py", - "symbol": "ModelDiscovery.discover_all", - "line_range": "L45-L210", - "explanation": "Queries connected provider endpoints for available model IDs." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MODEL-002", - "domain": "model", - "name": "Model Capability & Context Limits Detection", - "purpose": "Detects vision, tool calling, reasoning, and context window limits for connected model endpoints.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/models.js" - ], - "backend_entrypoints": [ - "src/model_capabilities.py", - "src/model_context.py", - "src/endpoint_resolver.py" - ], - "routes": [ - "/api/model-endpoints/{ep_id}/probe" - ], - "configuration": [ - "MODEL_CAPABILITY_OVERRODES" - ], - "persistence": [ - "In-memory capabilities cache" - ], - "dependencies": [ - "pydantic" - ], - "tests": [ - "tests/test_endpoint_resolver_headers.py", - "tests/test_vision_model_detection.py" - ], - "documentation": [ - "specs/architecture-runtime-inventory.md" - ], - "risks": [ - "Incorrect context limit metadata for unlisted custom fine-tunes." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/model_capabilities.py", - "symbol": "get_model_capabilities", - "line_range": "L30-L120", - "explanation": "Maps model names to vision and tool support flags." - }, - { - "path": "src/endpoint_resolver.py", - "symbol": "resolve_endpoint_headers", - "line_range": "L20-L80", - "explanation": "Resolves auth headers and target URLs for model endpoints." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MODEL-003", - "domain": "model", - "name": "LLM Core Provider Communication & Fallback Routing", - "purpose": "Manages HTTP request dispatching, authorization header injection, and fallback provider routing for LLM calls.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chatStream.js" - ], - "backend_entrypoints": [ - "src/llm_core.py:llm_call_async", - "src/llm_core.py:llm_call_async_with_fallback", - "src/llm_core.py:stream_llm_with_fallback" - ], - "routes": [ - "Implicit backend calls" - ], - "configuration": [ - "LLM_RETRY_ATTEMPTS", - "LLM_TIMEOUT" - ], - "persistence": [ - "Session state" - ], - "dependencies": [ - "httpx", - "asyncio" - ], - "tests": [ - "tests/test_llm_core_concurrency.py", - "tests/test_llm_core_fallback.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Unexpected API changes in upstream third-party model providers." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/llm_core.py", - "symbol": "llm_call_async", - "line_range": "L1949-L2118", - "explanation": "Non-streaming provider request dispatcher: resolves the endpoint, injects authorization headers and executes the HTTP call." - }, - { - "path": "src/llm_core.py", - "symbol": "llm_call_async_with_fallback", - "line_range": "L1932-L1946", - "explanation": "Ordered fallback wrapper that retries llm_call_async across the configured candidate endpoints." - }, - { - "path": "src/llm_core.py", - "symbol": "stream_llm_with_fallback", - "line_range": "L2794-L2895", - "explanation": "Ordered fallback wrapper for the streaming path; advances to the next candidate when a provider yields an empty completion." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Provider dispatch, header injection and fallback advancement are only observable against a reachable LLM provider endpoint; not exercised in this documentation pass." - } - }, - { - "id": "MODEL-004", - "domain": "model", - "name": "Model Selection & Display Ordering Preferences", - "purpose": "Allows pinning, sorting, and hiding specific models in the UI selection dropdown.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/modelSort.js", - "static/js/modelPicker.js" - ], - "backend_entrypoints": [ - "routes/model_routes.py:save_model_order" - ], - "routes": [ - "/api/models/order", - "/api/models/order/reset" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/model_order.json" - ], - "dependencies": [ - "json" - ], - "tests": [ - "tests/test_model_defaults.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Stale model IDs in custom order lists after model endpoints are removed." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/model_routes.py", - "symbol": "@router.post('/order')", - "line_range": "L500-L550", - "explanation": "Saves custom model display order preference." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MODEL-005", - "domain": "model", - "name": "Side-by-Side Model Comparison (A/B Testing)", - "purpose": "Enables dual-model side-by-side response evaluation, arena scoring, and latency comparison.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/compare/index.js", - "static/js/compare/panes.js" - ], - "backend_entrypoints": [ - "routes/compare/compare_routes.py:setup_compare_routes" - ], - "routes": [ - "/api/compare/start", - "/api/compare/{comp_id}/vote", - "/api/compare/history" - ], - "configuration": [ - "COMPARE_ENABLED" - ], - "persistence": [ - "DATA_DIR/compare_history.db" - ], - "dependencies": [ - "sqlite3" - ], - "tests": [ - "tests/test_endpoint_owner_scope_followup.py" - ], - "documentation": [ - "docs/compare.webm" - ], - "risks": [ - "High memory and network usage when streaming two model responses simultaneously." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/compare/compare_routes.py", - "symbol": "@router.post('/start')", - "line_range": "L70-L150", - "explanation": "Starts a parallel dual-model comparison stream." - }, - { - "path": "static/js/compare/index.js", - "symbol": "initCompareView", - "line_range": "L1-L100", - "explanation": "Renders side-by-side model chat panes." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MODEL-006", - "domain": "model", - "name": "GitHub Copilot Device Flow Authentication", - "purpose": "Authenticates with GitHub Copilot via OAuth device flow to use Copilot models directly.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/providerDeviceFlow.js" - ], - "backend_entrypoints": [ - "routes/copilot_routes.py:setup_copilot_routes", - "routes/device_flow.py:create_device_flow_router", - "src/copilot.py:request_device_code", - "src/copilot.py:poll_access_token" - ], - "routes": [ - "/api/copilot/device/start", - "/api/copilot/device/poll" - ], - "configuration": [ - "COPILOT_CLIENT_ID" - ], - "persistence": [ - "DATA_DIR/copilot_auth.json" - ], - "dependencies": [ - "httpx" - ], - "tests": [ - "tests/test_provider_device_flow_js.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Token expiration requires manual device re-authentication." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/copilot_routes.py", - "symbol": "setup_copilot_routes", - "line_range": "L166-L173", - "explanation": "Builds the Copilot device-flow router at prefix /api/copilot, wiring _start_device_flow and _poll_device_flow." - }, - { - "path": "routes/device_flow.py", - "symbol": "create_device_flow_router", - "line_range": "L135-L193", - "explanation": "Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix." - }, - { - "path": "src/copilot.py", - "symbol": "request_device_code", - "line_range": "L150-L161", - "explanation": "Issues the GitHub device-code request that begins the Copilot OAuth device flow." - }, - { - "path": "src/copilot.py", - "symbol": "poll_access_token", - "line_range": "L164-L180", - "explanation": "Polls GitHub for the access token once the user has authorized the device code." - }, - { - "path": "tests/test_provider_device_flow_js.py", - "symbol": "test_copilot_success_uses_complete_verification_uri", - "line_range": "L28-L65", - "explanation": "Inspected unit test asserting the Copilot device-flow runner surfaces the complete verification URI returned by the backend." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E2", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires an interactive GitHub Copilot OAuth device-flow account." - } - }, - { - "id": "MODEL-007", - "domain": "model", - "name": "ChatGPT Subscription Device Flow Authentication", - "purpose": "Authenticates with ChatGPT Pro/Plus subscription tokens via device login flow.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/providerDeviceFlow.js" - ], - "backend_entrypoints": [ - "routes/chatgpt_subscription_routes.py:setup_chatgpt_subscription_routes", - "routes/device_flow.py:create_device_flow_router", - "src/chatgpt_subscription.py:request_device_code", - "src/chatgpt_subscription.py:poll_device_auth" - ], - "routes": [ - "/api/chatgpt-subscription/device/start", - "/api/chatgpt-subscription/device/poll" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/chatgpt_auth.json" - ], - "dependencies": [ - "httpx" - ], - "tests": [ - "tests/test_provider_device_flow_js.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Changes in OpenAI auth endpoint security challenges." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/chatgpt_subscription_routes.py", - "symbol": "setup_chatgpt_subscription_routes", - "line_range": "L163-L170", - "explanation": "Builds the ChatGPT subscription device-flow router at prefix /api/chatgpt-subscription." - }, - { - "path": "routes/device_flow.py", - "symbol": "create_device_flow_router", - "line_range": "L135-L193", - "explanation": "Shared factory registering POST /device/start and POST /device/poll under the caller-supplied prefix." - }, - { - "path": "src/chatgpt_subscription.py", - "symbol": "request_device_code", - "line_range": "L168-L181", - "explanation": "Issues the ChatGPT device-authorization request that begins the subscription OAuth device flow." - }, - { - "path": "src/chatgpt_subscription.py", - "symbol": "poll_device_auth", - "line_range": "L184-L193", - "explanation": "Polls the ChatGPT device-authorization endpoint for completion using the stored device_auth_id and user_code." - }, - { - "path": "tests/test_provider_device_flow_js.py", - "symbol": "test_chatgpt_success_uses_plain_verification_uri", - "line_range": "L68-L95", - "explanation": "Inspected unit test asserting the ChatGPT device-flow runner uses the plain verification URI rather than the Copilot complete-URI form." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E2", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires an interactive ChatGPT subscription OAuth flow." - } - }, - { - "id": "MODEL-008", - "domain": "model", - "name": "Embedding Model Lane & Vector Provider Setup", - "purpose": "Configures local sentence-transformers, FastEmbed, or remote OpenAI embedding model lanes.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/settings.js" - ], - "backend_entrypoints": [ - "routes/embedding_routes.py:setup_embedding_routes", - "src/embeddings.py", - "src/embedding_lanes.py" - ], - "routes": [ - "/api/embeddings/active", - "/api/embeddings/test" - ], - "configuration": [ - "EMBEDDING_PROVIDER", - "EMBEDDING_MODEL" - ], - "persistence": [ - "DATA_DIR/embeddings_config.json" - ], - "dependencies": [ - "fastembed", - "sentence-transformers" - ], - "tests": [ - "tests/test_embedding_lane_ndarray_restore.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "First-time download of heavy PyTorch model weights on CPU-only machines." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/embedding_routes.py", - "symbol": "setup_embedding_routes", - "line_range": "L30-L120", - "explanation": "Provides embedding provider configuration endpoints." - }, - { - "path": "src/embeddings.py", - "symbol": "EmbeddingManager", - "line_range": "L40-L200", - "explanation": "Generates dense vector embeddings for RAG and memory." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "COOKBOOK-001", - "domain": "cookbook", - "name": "Local Model Download & Recipe Lifecycle Management", - "purpose": "Downloads HuggingFace models, configures execution parameters, and manages local GGUF/MLX model servers.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/cookbook.js", - "static/js/cookbookServe.js", - "static/js/cookbookDownload.js" - ], - "backend_entrypoints": [ - "routes/cookbook_routes.py:setup_cookbook_routes", - "src/tools/cookbook.py", - "src/cookbook_serve_lifecycle.py" - ], - "routes": [ - "/api/cookbook/download", - "/api/cookbook/serve", - "/api/cookbook/status" - ], - "configuration": [ - "COOKBOOK_MODELS_DIR" - ], - "persistence": [ - "DATA_DIR/models/" - ], - "dependencies": [ - "huggingface_hub", - "subprocess" - ], - "tests": [ - "tests/test_cookbook_endpoint_registration.py", - "tests/test_cookbook_port_parsing_js.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Disk space exhaustion during multi-gigabyte GGUF weights downloads." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/cookbook_routes.py", - "symbol": "setup_cookbook_routes", - "line_range": "L100-L300", - "explanation": "Exposes model downloading and process serving endpoints." - }, - { - "path": "static/js/cookbook.js", - "symbol": "initCookbook", - "line_range": "L1-L150", - "explanation": "UI manager for local model library." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "COOKBOOK-002", - "domain": "cookbook", - "name": "Hardware Model Fitting ('What Fits?') Analysis Engine", - "purpose": "Calculates RAM/VRAM requirements, quantized size, and context overhead to determine model compatibility.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/cookbook-hwfit.js" - ], - "backend_entrypoints": [ - "routes/hwfit_routes.py:setup_hwfit_routes", - "services/hwfit/fit.py:calculate_fit" - ], - "routes": [ - "/api/hwfit/fit", - "/api/hwfit/profile" - ], - "configuration": [ - "FORCE_GPU_VENDOR" - ], - "persistence": [ - "System hardware specs" - ], - "dependencies": [ - "psutil", - "torch" - ], - "tests": [ - "tests/test_hwfit_gemma4_12b.py", - "tests/test_hwfit_bandwidth_nonstring.py", - "tests/test_hwfit_gpu_count_nonnumeric.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Inaccurate VRAM estimation for non-standard KV-cache quantization." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/hwfit_routes.py", - "symbol": "setup_hwfit_routes", - "line_range": "L40-L120", - "explanation": "Calculates hardware model compatibility." - }, - { - "path": "services/hwfit/fit.py", - "symbol": "calculate_fit", - "line_range": "L30-L140", - "explanation": "Performs parameter and memory fit calculations." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "COOKBOOK-003", - "domain": "cookbook", - "name": "HuggingFace & MLX Model Discovery Services", - "purpose": "Searches HuggingFace Hub and local MLX model repositories for compatible GGUF and MLX weights.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/cookbook.js" - ], - "backend_entrypoints": [ - "routes/hwfit_routes.py:hf_search", - "services/hwfit/hf_discovery.py" - ], - "routes": [ - "/api/hwfit/hf-search", - "/api/hwfit/mlx-models" - ], - "configuration": [ - "HF_TOKEN" - ], - "persistence": [ - "Cache directory" - ], - "dependencies": [ - "huggingface_hub" - ], - "tests": [ - "tests/test_cookbook_hf_token.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "HuggingFace API rate limits when searching without an API token." - ], - "unknowns": [], - "evidence": [ - { - "path": "services/hwfit/hf_discovery.py", - "symbol": "search_hf_models", - "line_range": "L20-L90", - "explanation": "Queries HuggingFace API for model tags and files." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "COOKBOOK-004", - "domain": "cookbook", - "name": "Host Docker Access for Inference Container Runtimes", - "purpose": "Detects and connects to host Docker engine to launch containerized Ollama, vLLM, or SGLang runtimes.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/cookbookServe.js" - ], - "backend_entrypoints": [ - "src/host_docker_access.py:HostDockerAccess" - ], - "routes": [ - "Implicit local docker socket API calls" - ], - "configuration": [ - "DOCKER_HOST" - ], - "persistence": [ - "/var/run/docker.sock" - ], - "dependencies": [ - "docker" - ], - "tests": [ - "tests/test_cookbook_docker_access.py" - ], - "documentation": [ - "docker/host-docker.yml" - ], - "risks": [ - "Permission denied accessing docker socket on non-root setups." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/host_docker_access.py", - "symbol": "HostDockerAccess", - "line_range": "L50-L62", - "explanation": "Interacts with host docker daemon." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires Docker access and supported physical GPU hardware." - } - }, - { - "id": "RESEARCH-001", - "domain": "research", - "name": "Deep Research Execution Engine & SSE Progress Streaming", - "purpose": "Executes multi-step recursive deep research tasks, web page scraping, synthesis, and streams live progress.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/research/panel.js", - "static/js/research/jobs.js", - "static/js/researchSynapse.js" - ], - "backend_entrypoints": [ - "routes/research/research_routes.py:start_research", - "src/deep_research.py", - "services/research/service.py" - ], - "routes": [ - "/api/research/start", - "/api/research/stream/{session_id}", - "/api/research/active" - ], - "configuration": [ - "MAX_RESEARCH_DEPTH", - "SEARXNG_URL" - ], - "persistence": [ - "DATA_DIR/research/" - ], - "dependencies": [ - "beautifulsoup4", - "httpx", - "asyncio" - ], - "tests": [ - "tests/test_research_owner_scope_routes.py", - "tests/test_services_research_low_quality_sources.py" - ], - "documentation": [ - "docs/research.webm" - ], - "risks": [ - "High memory consumption when parsing multi-megabyte HTML target pages." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/research/research_routes.py", - "symbol": "@router.post('/api/research/start')", - "line_range": "L492-L550", - "explanation": "Initiates deep research job." - }, - { - "path": "src/deep_research.py", - "symbol": "DeepResearchEngine", - "line_range": "L40-L300", - "explanation": "Recursive search and summary crawler." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "RESEARCH-002", - "domain": "research", - "name": "Research Library, Detail View & Image Controls", - "purpose": "Stores completed research reports, generated diagrams, reference links, and manages image visibility.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/research/panel.js" - ], - "backend_entrypoints": [ - "routes/research/research_routes.py:get_library" - ], - "routes": [ - "/api/research/library", - "/api/research/detail/{session_id}", - "/api/research/{session_id}/hide-image" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/research/library.json" - ], - "dependencies": [ - "pydantic" - ], - "tests": [ - "tests/cli/test_research_cli_preview.py", - "tests/test_research_routes_path_confinement.py" - ], - "documentation": [ - "docs/research.webm" - ], - "risks": [ - "Orphaned report files if storage directory is modified out-of-band." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/research/research_routes.py", - "symbol": "@router.get('/api/research/library')", - "line_range": "L366-L415", - "explanation": "Returns all saved research reports." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "RESEARCH-003", - "domain": "research", - "name": "Web Search Engine Integration (SearXNG & Multi-Provider)", - "purpose": "Queries SearXNG, DuckDuckGo, or Google Search instances to retrieve web search snippets.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chat.js", - "static/js/search.js" - ], - "backend_entrypoints": [ - "routes/search_routes.py:setup_search_routes", - "src/search/core.py", - "services/search/service.py" - ], - "routes": [ - "/api/search" - ], - "configuration": [ - "SEARXNG_URL", - "SEARCH_PROVIDER" - ], - "persistence": [ - "In-memory search cache" - ], - "dependencies": [ - "httpx" - ], - "tests": [ - "tests/test_search_ranking.py", - "tests/test_searxng_image_pinned.py" - ], - "documentation": [ - "config/searxng/settings.yml" - ], - "risks": [ - "Search provider IP throttling or rate-limiting." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/search_routes.py", - "symbol": "setup_search_routes", - "line_range": "L39-L100", - "explanation": "Defines /api/search, /api/search/config, and /api/search/query endpoints." - }, - { - "path": "src/search/core.py", - "symbol": "SearchEngine", - "line_range": "L1-L12", - "explanation": "Compatibility module aliasing services.search.core." - }, - { - "path": "tests/test_search_ranking.py", - "symbol": "test_news_queries_prefer_news_sources_over_sports_and_social_results", - "line_range": "L1-L39", - "explanation": "Tests search result domain ranking and scoring." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires an active SearXNG instance or external search API provider." - } - }, - { - "id": "RESEARCH-004", - "domain": "research", - "name": "Research Result Peeking & Topic Spinoff Generation", - "purpose": "Extracts preliminary research snippets and spawns child research sessions focused on specific sub-topics.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/research/panel.js" - ], - "backend_entrypoints": [ - "routes/research/research_routes.py:peek_result", - "routes/research/research_routes.py:spinoff_research" - ], - "routes": [ - "/api/research/result-peek/{session_id}", - "/api/research/spinoff/{session_id}" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/research/" - ], - "dependencies": [ - "fastapi" - ], - "tests": [ - "tests/test_research_routes_path_confinement.py" - ], - "documentation": [ - "docs/research.webm" - ], - "risks": [ - "Deep recursion tree depth when spawning multiple nested spinoffs." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/research/research_routes.py", - "symbol": "@router.post('/api/research/spinoff/{session_id}')", - "line_range": "L635-L680", - "explanation": "Spawns child research session for specific query." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "DOCUMENT-001", - "domain": "document", - "name": "Document & Canvas Artifact Management", - "purpose": "Renders dynamic canvas documents, handles live editing, markdown preview, and side-by-side artifact display.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/document.js", - "static/js/documentLibrary.js" - ], - "backend_entrypoints": [ - "routes/document_routes.py:setup_document_routes", - "src/document_actions.py", - "src/document_processor.py" - ], - "routes": [ - "/api/document", - "/api/document/{id}" - ], - "configuration": [ - "MAX_DOCUMENT_SIZE_MB" - ], - "persistence": [ - "DATA_DIR/documents/" - ], - "dependencies": [ - "pydantic", - "fastapi" - ], - "tests": [ - "tests/test_document_actions_nonstring.py", - "tests/test_document_diff_discard_on_update_js.py" - ], - "documentation": [ - "docs/document.webm" - ], - "risks": [ - "Concurrent edits on the same document artifact." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/document_routes.py", - "symbol": "setup_document_routes", - "line_range": "L100-L300", - "explanation": "Registers document artifact CRUD routes." - }, - { - "path": "static/js/document.js", - "symbol": "initDocumentView", - "line_range": "L1-L150", - "explanation": "Renders interactive canvas document panel." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "DOCUMENT-002", - "domain": "document", - "name": "PDF Form Processing & High-Fidelity Rendering", - "purpose": "Extracts form fields from PDF files, fills dynamic values, and generates PDF previews.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/document.js" - ], - "backend_entrypoints": [ - "src/pdf_runtime.py", - "src/pdf_forms.py", - "src/pdf_form_doc.py" - ], - "routes": [ - "Implicit PDF document processing" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/documents/pdf/" - ], - "dependencies": [ - "pypdf", - "fitz" - ], - "tests": [ - "tests/test_document_pdf_marker.py", - "tests/test_security_headers_pdf_preview.py" - ], - "documentation": [ - "docs/attachments.md" - ], - "risks": [ - "Complex XFA PDF forms may not extract cleanly with standard pdf parsers." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/pdf_runtime.py", - "symbol": "load_pymupdf_for_pdf_viewer", - "line_range": "L9-L15", - "explanation": "Loads optional PyMuPDF runtime for PDF viewing." - }, - { - "path": "src/pdf_forms.py", - "symbol": "extract_form_fields", - "line_range": "L1-L100", - "explanation": "Handles PDF form field extraction and filling." - }, - { - "path": "tests/test_document_pdf_marker.py", - "symbol": "test_marker_removed_without_eating_following_text", - "line_range": "L1-L30", - "explanation": "Tests PDF text extraction wrapper stripping without content corruption." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires optional PyMuPDF (`fitz`) or pypdf runtime dependency." - } - }, - { - "id": "DOCUMENT-003", - "domain": "document", - "name": "Personal Document Indexing & RAG Retrieval", - "purpose": "Indexes local user documents (PDF, DOCX, TXT) into ChromaDB for semantic vector retrieval.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/rag.js" - ], - "backend_entrypoints": [ - "routes/personal_routes.py:setup_personal_routes", - "src/personal_docs.py", - "src/rag_manager.py" - ], - "routes": [ - "/api/personal/documents", - "/api/personal/search", - "/api/personal/index" - ], - "configuration": [ - "CHROMADB_DIR" - ], - "persistence": [ - "DATA_DIR/chroma/" - ], - "dependencies": [ - "chromadb", - "pypdf", - "docx" - ], - "tests": [ - "tests/test_personal_docs_pdf_index.py", - "tests/test_rag_index_hidden_dirs.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Slow vector embedding indexing step for massive multi-thousand page documents." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/personal_routes.py", - "symbol": "setup_personal_routes", - "line_range": "L50-L160", - "explanation": "Personal document RAG indexing and search API endpoints." - }, - { - "path": "src/personal_docs.py", - "symbol": "PersonalDocsManager", - "line_range": "L30-L180", - "explanation": "Handles file text chunking and vector storage." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "DOCUMENT-004", - "domain": "document", - "name": "Document Conversion & Text Extraction Engine", - "purpose": "Converts office formats (.docx, .xlsx, .pptx) and HTML into clean Markdown text representations.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/fileHandler.js" - ], - "backend_entrypoints": [ - "src/markitdown_runtime.py", - "src/office_doc.py" - ], - "routes": [ - "Implicit file conversion calls" - ], - "configuration": [], - "persistence": [ - "Temporary conversion cache" - ], - "dependencies": [ - "markitdown", - "python-docx" - ], - "tests": [ - "tests/cli/test_docs_cli_content_length.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Formatting loss when parsing legacy binary doc/xls files." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/markitdown_runtime.py", - "symbol": "convert_to_markdown", - "line_range": "L15-L80", - "explanation": "Converts binary office documents into structured Markdown text." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "DOCUMENT-005", - "domain": "document", - "name": "Document Library UI Navigation", - "purpose": "Provides dedicated UI view for browsing, filtering, and organizing saved user documents.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/documentLibrary.js" - ], - "backend_entrypoints": [ - "app.py:serve_library" - ], - "routes": [ - "/library" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/documents/" - ], - "dependencies": [ - "Vanilla JS" - ], - "tests": [ - "tests/test_document_close_clears_active_route.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Large folder trees may cause initial DOM render slowdown." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/js/documentLibrary.js", - "symbol": "initDocumentLibrary", - "line_range": "L1-L100", - "explanation": "Renders document library navigation grid." - }, - { - "path": "app.py", - "symbol": "serve_library", - "line_range": "L911-L914", - "explanation": "Serves SPA shell for /library route." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "EMAIL-001", - "domain": "email", - "name": "Email Account Setup, IMAP/SMTP Connection & Polling", - "purpose": "Configures IMAP/SMTP email accounts, validates TLS certificates, and polls background inbox updates.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/emailInbox.js" - ], - "backend_entrypoints": [ - "routes/email_routes.py:setup_email_routes", - "routes/email_pollers.py" - ], - "routes": [ - "/api/email/accounts", - "/api/email/poll" - ], - "configuration": [ - "EMAIL_POLL_INTERVAL" - ], - "persistence": [ - "DATA_DIR/email_accounts.json" - ], - "dependencies": [ - "imaplib", - "smtplib" - ], - "tests": [ - "tests/test_service_health_email.py", - "tests/test_active_email_reply_guard.py" - ], - "documentation": [ - "docs/email-outlook.md" - ], - "risks": [ - "Account lockouts if bad credentials are repeatedly polled." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/email_routes.py", - "symbol": "setup_email_routes", - "line_range": "L1453-L1500", - "explanation": "Sets up email account management and synchronization routes." - }, - { - "path": "routes/email_pollers.py", - "symbol": "_start_poller", - "line_range": "L1-L100", - "explanation": "Background poller for email inbox synchronization." - }, - { - "path": "tests/test_service_health_email.py", - "symbol": "test_email_ok_all_connect", - "line_range": "L1-L80", - "explanation": "Tests IMAP connection health probing and status reporting." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires a controlled live IMAP account and network access." - } - }, - { - "id": "EMAIL-002", - "domain": "email", - "name": "Email Searching, Threading & Message Operations", - "purpose": "Parses email headers, folds signatures, groups messages into threads, and executes full-text email search.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/emailLibrary.js", - "static/js/emailLibrary/state.js" - ], - "backend_entrypoints": [ - "routes/email_routes.py:search_email", - "src/email_thread_parser.py" - ], - "routes": [ - "/api/email/search", - "/api/email/threads", - "/api/email/messages" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/email_cache.db" - ], - "dependencies": [ - "sqlite3", - "email" - ], - "tests": [ - "tests/test_reply_recipients_js.py", - "tests/test_signature_fold_js.py" - ], - "documentation": [ - "docs/email-outlook.md" - ], - "risks": [ - "Malformed MIME email structures failing HTML sanitization." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/email_routes.py", - "symbol": "@router.get('/search')", - "line_range": "L250-L320", - "explanation": "Executes search across cached email headers and text." - }, - { - "path": "src/email_thread_parser.py", - "symbol": "parse_email_thread", - "line_range": "L20-L100", - "explanation": "Builds conversation tree from Message-ID and In-Reply-To headers." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "EMAIL-003", - "domain": "email", - "name": "Email Composition, Draft Management & Sending", - "purpose": "Creates, saves, and dispatches HTML/plaintext email messages via SMTP.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/emailShared.js" - ], - "backend_entrypoints": [ - "routes/email_routes.py:send_email" - ], - "routes": [ - "/api/email/draft", - "/api/email/send" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/email_drafts.json" - ], - "dependencies": [ - "smtplib" - ], - "tests": [ - "tests/test_schedule_email_offset_normalization.py" - ], - "documentation": [ - "docs/email-outlook.md" - ], - "risks": [ - "SMTP connection drop mid-send causing unsent mail state." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/email_routes.py", - "symbol": "@router.post('/send')", - "line_range": "L450-L520", - "explanation": "Sends email message via user SMTP credentials." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires a controlled live SMTP account and network access." - } - }, - { - "id": "EMAIL-004", - "domain": "email", - "name": "Email MCP Server & Codex Integration Bridge", - "purpose": "Exposes constrained email reading and draft capabilities to external Codex / MCP agents with scope checks.", - "status": "verified", - "frontend_entrypoints": [ - "integrations/codex/scripts/odysseus_api.py" - ], - "backend_entrypoints": [ - "mcp_servers/email_server.py", - "routes/codex_routes.py:setup_codex_routes" - ], - "routes": [ - "/api/codex/email/search", - "/api/codex/email/send" - ], - "configuration": [ - "CODEX_API_KEY" - ], - "persistence": [ - "API token scopes" - ], - "dependencies": [ - "mcp", - "fastapi" - ], - "tests": [ - "tests/cli/test_mail_cli_recipients.py" - ], - "documentation": [ - "integrations/codex/README.md" - ], - "risks": [ - "Unauthorized mail sending if token scopes are improperly scoped." - ], - "unknowns": [], - "evidence": [ - { - "path": "mcp_servers/email_server.py", - "symbol": "EmailMcpServer", - "line_range": "L20-L120", - "explanation": "MCP server exposing email tools over stdio/SSE." - }, - { - "path": "routes/codex_routes.py", - "symbol": "setup_codex_routes", - "line_range": "L30-L110", - "explanation": "Bridge endpoints for external Codex plugin integration." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CALENDAR-001", - "domain": "calendar", - "name": "CalDAV Calendar Synchronization & Account Setup", - "purpose": "Connects to remote CalDAV servers (Apple iCloud, Nextcloud, Google) to sync calendar event feeds.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/calendar.js" - ], - "backend_entrypoints": [ - "routes/calendar_routes.py:setup_calendar_routes", - "src/caldav_sync.py:CalDavSync" - ], - "routes": [ - "/api/calendar/config", - "/api/calendar/sync" - ], - "configuration": [ - "CALDAV_URL", - "CALDAV_USERNAME" - ], - "persistence": [ - "DATA_DIR/calendar_config.json" - ], - "dependencies": [ - "caldav", - "vobject" - ], - "tests": [ - "tests/test_caldav_url_hardening.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Invalid SSL certificates on self-hosted CalDAV servers." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/calendar_routes.py", - "symbol": "setup_calendar_routes", - "line_range": "L50-L180", - "explanation": "Exposes CalDAV setup and manual sync trigger routes." - }, - { - "path": "src/caldav_sync.py", - "symbol": "CalDavSync", - "line_range": "L30-L160", - "explanation": "Fetches and parses remote iCalendar VEVENT objects." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires a controlled external CalDAV server." - } - }, - { - "id": "CALENDAR-002", - "domain": "calendar", - "name": "Calendar Event Operations & iCalendar Parsing", - "purpose": "Creates, updates, deletes, and displays calendar events with timezone conversion and reminder notifications.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/calendar/reminders.js", - "static/js/calendar/utils.js" - ], - "backend_entrypoints": [ - "routes/calendar_routes.py:create_event", - "src/caldav_writeback.py", - "src/tools/calendar.py" - ], - "routes": [ - "/api/calendar/events", - "/api/calendar/events/{event_id}" - ], - "configuration": [ - "USER_TIMEZONE" - ], - "persistence": [ - "DATA_DIR/calendar_events.db" - ], - "dependencies": [ - "sqlite3", - "icalendar" - ], - "tests": [ - "tests/test_calendar_parse_dt_time_first.py", - "tests/test_calendar_update_event_tz.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Recurring RRULE event expansion calculation bugs across leap years." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/calendar_routes.py", - "symbol": "@router.get('/events')", - "line_range": "L220-L310", - "explanation": "Fetches calendar events for requested date window." - }, - { - "path": "src/tools/calendar.py", - "symbol": "CalendarTool", - "line_range": "L25-L120", - "explanation": "Agent tool for creating and modifying calendar entries." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-001", - "domain": "media", - "name": "Gallery Image Library & Album Operations", - "purpose": "Organizes images into custom albums, provides grid browsing, tagging, and album metadata management.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/gallery.js" - ], - "backend_entrypoints": [ - "routes/gallery/gallery_routes.py:setup_gallery_routes" - ], - "routes": [ - "/api/gallery/library", - "/api/gallery/albums", - "/api/gallery/upload" - ], - "configuration": [ - "GALLERY_STORAGE_DIR" - ], - "persistence": [ - "DATA_DIR/gallery/" - ], - "dependencies": [ - "pillow", - "sqlite3" - ], - "tests": [ - "tests/test_gallery_image_endpoint_owner_scope.py", - "tests/test_gallery_owner_filter_single_user.py" - ], - "documentation": [ - "docs/gallery.webm" - ], - "risks": [ - "Thumbnail generation overhead for high-resolution RAW camera images." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/gallery/gallery_routes.py", - "symbol": "@router.get('/api/gallery/library')", - "line_range": "L657-L720", - "explanation": "Fetches image library list with tag filters." - }, - { - "path": "static/js/gallery.js", - "symbol": "initGallery", - "line_range": "L1-L180", - "explanation": "Main gallery grid renderer and uploader." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-002", - "domain": "media", - "name": "Image Processing, AI Upscaling & Style Transfer", - "purpose": "Executes local image enhancement, background removal, face sharpening, and AI upscaling operations.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/galleryEditor.js", - "static/js/editor/ai-tool-runner.js" - ], - "backend_entrypoints": [ - "routes/gallery/gallery_routes.py:ai_upscale", - "routes/gallery/gallery_routes.py:remove_bg" - ], - "routes": [ - "/api/gallery/ai-upscale", - "/api/gallery/style-transfer", - "/api/image/inpaint", - "/api/image/remove-bg" - ], - "configuration": [ - "REALESRGAN_MODEL_PATH" - ], - "persistence": [ - "DATA_DIR/gallery/processed/" - ], - "dependencies": [ - "onnxruntime", - "opencv-python" - ], - "tests": [ - "tests/test_sanitize_multimodal_merge.py" - ], - "documentation": [ - "docs/gallery.webm" - ], - "risks": [ - "High GPU memory allocation when upscaling 4K images." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/gallery/gallery_routes.py", - "symbol": "@router.post('/api/gallery/ai-upscale')", - "line_range": "L544-L580", - "explanation": "Runs RealESRGAN image upscaling." - }, - { - "path": "routes/gallery/gallery_routes.py", - "symbol": "@router.post('/api/image/remove-bg')", - "line_range": "L1950-L2010", - "explanation": "Executes background removal pass." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-003", - "domain": "media", - "name": "Interactive Image Canvas Editor & Persisted Drafts", - "purpose": "Provides full multi-layer raster canvas editor, brush tools, transforms, masks, and draft project persistence.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/galleryEditor.js", - "static/js/editor/history-panel.js" - ], - "backend_entrypoints": [ - "routes/editor_draft_routes.py:setup_editor_draft_routes" - ], - "routes": [ - "/api/editor/drafts", - "/api/editor/drafts/{draft_id}" - ], - "configuration": [ - "MAX_DRAFT_PROJECTS" - ], - "persistence": [ - "DATA_DIR/editor_drafts/" - ], - "dependencies": [ - "HTML5 Canvas API", - "pydantic" - ], - "tests": [ - "tests/test_canvas_coords_empty_touches_js.py", - "tests/test_snap_other_layers_nonarray_js.py" - ], - "documentation": [ - "docs/gallery.webm" - ], - "risks": [ - "Browser memory leak if multi-gigabyte layer undo buffers are kept indefinitely." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/editor_draft_routes.py", - "symbol": "setup_editor_draft_routes", - "line_range": "L30-L110", - "explanation": "API routes for saving and loading canvas project drafts." - }, - { - "path": "static/js/editor/history-panel.js", - "symbol": "HistoryManager", - "line_range": "L1-L90", - "explanation": "Canvas undo/redo stack manager." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-004", - "domain": "media", - "name": "Text-to-Speech (TTS) Synthesis Service", - "purpose": "Synthesizes spoken audio from text using local Kokoro, EdgeTTS, or OpenAI TTS engines.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/tts-ai.js" - ], - "backend_entrypoints": [ - "routes/tts_routes.py:setup_tts_routes", - "services/tts/tts_service.py:TTSService" - ], - "routes": [ - "/api/tts/synthesize", - "/api/tts/stats" - ], - "configuration": [ - "TTS_PROVIDER", - "TTS_VOICE" - ], - "persistence": [ - "DATA_DIR/tts_cache/" - ], - "dependencies": [ - "soundfile", - "edge-tts" - ], - "tests": [ - "tests/test_tts_available_nonstring_provider.py", - "tests/test_tts_cache_stats.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Audio synthesis latency on CPU-only hardware setups." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/tts_routes.py", - "symbol": "@router.post('/synthesize')", - "line_range": "L30-L75", - "explanation": "Synthesizes TTS audio clip." - }, - { - "path": "services/tts/tts_service.py", - "symbol": "TTSService", - "line_range": "L25-L140", - "explanation": "Provider abstraction layer for audio speech generation." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-005", - "domain": "media", - "name": "Speech-to-Text (STT) Audio Transcription Service", - "purpose": "Transcribes user audio recordings into text using faster-whisper or local speech models.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/voiceRecorder.js" - ], - "backend_entrypoints": [ - "routes/stt_routes.py:setup_stt_routes", - "services/stt/stt_service.py:STTService" - ], - "routes": [ - "/api/stt/transcribe", - "/api/stt/stats" - ], - "configuration": [ - "STT_PROVIDER", - "WHISPER_MODEL_SIZE" - ], - "persistence": [ - "Temporary audio buffer" - ], - "dependencies": [ - "faster-whisper", - "ffmpeg-python" - ], - "tests": [ - "tests/test_stt_leak.py", - "tests/test_speech_service_toggles.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Missing ffmpeg system dependency prevents audio format decoding." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/stt_routes.py", - "symbol": "@router.post('/transcribe')", - "line_range": "L25-L55", - "explanation": "Accepts multipart audio file and returns transcription text." - }, - { - "path": "services/stt/stt_service.py", - "symbol": "STTService", - "line_range": "L20-L110", - "explanation": "Whisper audio transcription engine wrapper." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-006", - "domain": "media", - "name": "Digital Signature Stamp Storage & Placement", - "purpose": "Stores transparent PNG user signatures and stamps for placement onto PDF forms and documents.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/signature.js" - ], - "backend_entrypoints": [ - "routes/signature_routes.py:setup_signature_routes" - ], - "routes": [ - "/api/signatures", - "/api/signatures/{sig_id}" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/signatures/" - ], - "dependencies": [ - "pillow" - ], - "tests": [ - "tests/test_signature_route_hardening.py", - "tests/test_signature_settings_dom_xss.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Cross-site scripting if signature image titles contain unescaped user input." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/signature_routes.py", - "symbol": "setup_signature_routes", - "line_range": "L30-L120", - "explanation": "CRUD endpoints for managing user signature PNG stamps." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-007", - "domain": "media", - "name": "Generated Image Artifact Route & MCP Integration", - "purpose": "Serves generated AI artwork artifacts and integrates with image generation MCP server.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chat.js" - ], - "backend_entrypoints": [ - "app.py:serve_generated_image", - "src/generated_images.py", - "mcp_servers/image_gen_server.py" - ], - "routes": [ - "/api/generated-image/{filename}" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/generated_images/" - ], - "dependencies": [ - "pillow", - "mcp" - ], - "tests": [ - "tests/test_image_models_nondict_system.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Path traversal vulnerability if filename parameter is un-sanitized." - ], - "unknowns": [], - "evidence": [ - { - "path": "app.py", - "symbol": "serve_generated_image", - "line_range": "L499-L520", - "explanation": "Serves generated image artifacts with cache headers." - }, - { - "path": "src/generated_images.py", - "symbol": "resolve_generated_image_path", - "line_range": "L20-L32", - "explanation": "Confines requested image path within artifacts directory." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEDIA-008", - "domain": "media", - "name": "Native MLX Image Bridge (macOS Apple Silicon)", - "purpose": "Native Apple Swift bridge for hardware-accelerated diffusion and MLX image colorization on macOS.", - "status": "experimental", - "frontend_entrypoints": [ - "static/js/editor/ai-tools-misc.js" - ], - "backend_entrypoints": [ - "swift/odysseus-mlx-image-bridge/Package.swift", - "scripts/mlx_image_server.py", - "scripts/diffusion_server.py" - ], - "routes": [ - "/api/image/mlx-colorize" - ], - "configuration": [ - "ENABLE_MLX_ACCELERATION" - ], - "persistence": [ - "Build artifact binary" - ], - "dependencies": [ - "Swift", - "MLX Framework" - ], - "tests": [ - "tests/helpers/import_state.py" - ], - "documentation": [ - "swift/odysseus-mlx-image-bridge/Package.swift" - ], - "risks": [ - "Requires macOS host with Apple Silicon M-series chip and compiled Swift binary." - ], - "unknowns": [ - "Binary build requires Xcode command line tools build step (`build-macos-app.sh`)." - ], - "evidence": [ - { - "path": "swift/odysseus-mlx-image-bridge/Package.swift", - "symbol": "Package", - "line_range": "L1-L30", - "explanation": "Swift package manifest for native MLX image bridge." - }, - { - "path": "scripts/mlx_image_server.py", - "symbol": "main", - "line_range": "L20-L90", - "explanation": "Python daemon wrapping native Swift MLX binary." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires Apple Silicon, macOS tooling, and the compiled MLX bridge." - } - }, - { - "id": "SECURITY-001", - "domain": "security", - "name": "Authentication, Session Cookies & User Management", - "purpose": "Handles bcrypt password hashing, session cookie issuance, authentication enforcement, and user administration.", - "status": "verified", - "frontend_entrypoints": [ - "static/login.html", - "static/js/admin.js" - ], - "backend_entrypoints": [ - "routes/auth_routes.py:setup_auth_routes", - "core/auth.py:AuthManager" - ], - "routes": [ - "/api/auth/login", - "/api/auth/me", - "/api/auth/users", - "/api/auth/logout" - ], - "configuration": [ - "AUTH_ENABLED", - "SESSION_COOKIE_NAME" - ], - "persistence": [ - "DATA_DIR/auth.db" - ], - "dependencies": [ - "bcrypt", - "itsdangerous" - ], - "tests": [ - "tests/test_rename_user_owner_sync.py", - "tests/test_reserved_username_admin_escalation.py" - ], - "documentation": [ - "THREAT_MODEL.md", - "SECURITY.md" - ], - "risks": [ - "Cookie session hijack if deployed over unencrypted HTTP without HTTPS cookie flags." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/auth_routes.py", - "symbol": "@router.post('/login')", - "line_range": "L100-L180", - "explanation": "Authenticates credentials and sets session cookie." - }, - { - "path": "core/auth.py", - "symbol": "AuthManager", - "line_range": "L40-L260", - "explanation": "Handles user creation, password verification, and session tokens." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "SECURITY-002", - "domain": "security", - "name": "System Vault Encrypted Secret Storage", - "purpose": "Encrypts API keys, passphrases, and third-party secrets on disk using AES-GCM / PBKDF2 key derivation.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/settings.js" - ], - "backend_entrypoints": [ - "routes/vault_routes.py:setup_vault_routes", - "src/secret_storage.py:SecretVault", - "src/tools/vault.py" - ], - "routes": [ - "/api/vault/config", - "/api/vault/unlock", - "/api/vault/lock" - ], - "configuration": [ - "VAULT_SALT" - ], - "persistence": [ - "DATA_DIR/vault.enc" - ], - "dependencies": [ - "cryptography" - ], - "tests": [ - "tests/test_vault_password_not_in_argv.py" - ], - "documentation": [ - "THREAT_MODEL.md" - ], - "risks": [ - "Loss of vault master passphrase renders all encrypted secrets permanently unrecoverable." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/vault_routes.py", - "symbol": "setup_vault_routes", - "line_range": "L126-L229", - "explanation": "Admin routes for vault configuration, login, unlock, lock, and logout." - }, - { - "path": "src/secret_storage.py", - "symbol": "SecretStorage", - "line_range": "L57-L87", - "explanation": "Fernet symmetric key DB secret encryption." - }, - { - "path": "tests/test_vault_password_not_in_argv.py", - "symbol": "test_bw_password_not_in_argv", - "line_range": "L1-L117", - "explanation": "Verifies master password is fed via stdin and never appears in process argv." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires installed Bitwarden CLI (`bw`) executable." - } - }, - { - "id": "SECURITY-003", - "domain": "security", - "name": "API Token Management & Scope Access Control", - "purpose": "Generates scoped API bearer tokens (read/write/admin) for external tool and script authentication.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/settings.js" - ], - "backend_entrypoints": [ - "routes/api_token_routes.py:setup_api_token_routes", - "core/database.py:ApiToken" - ], - "routes": [ - "/api/tokens", - "/api/tokens/{token_id}" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/odysseus.db (api_tokens table)" - ], - "dependencies": [ - "sqlalchemy", - "secrets" - ], - "tests": [ - "tests/test_api_key_file_permissions.py" - ], - "documentation": [ - "SECURITY.md" - ], - "risks": [ - "Leaked API bearer tokens with excessive permission scopes." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/api_token_routes.py", - "symbol": "setup_api_token_routes", - "line_range": "L30-L150", - "explanation": "Exposes API token creation, scope assignment, and revocation." - }, - { - "path": "core/database.py", - "symbol": "ApiToken", - "line_range": "L50-L90", - "explanation": "SQLAlchemy ORM schema for API tokens and permissions." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "SECURITY-004", - "domain": "security", - "name": "Prompt Security & Injection Defense Engine", - "purpose": "Scans system prompts and external inputs for prompt injection attempts, jailbreaks, and sensitive data leaks.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/chat.js" - ], - "backend_entrypoints": [ - "src/prompt_security.py:sanitize_prompt", - "src/tool_security.py:validate_tool_input" - ], - "routes": [ - "Implicit security inspection" - ], - "configuration": [ - "ENABLE_PROMPT_SANITY_CHECK" - ], - "persistence": [], - "dependencies": [ - "re" - ], - "tests": [ - "tests/test_skill_index_prompt_injection.py", - "tests/test_tool_output_prompt_injection.py" - ], - "documentation": [ - "THREAT_MODEL.md" - ], - "risks": [ - "False positives blocking legitimate complex coding or security prompts." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/prompt_security.py", - "symbol": "untrusted_context_message", - "line_range": "L64-L86", - "explanation": "Wraps untrusted context with guard delimiters and sets metadata.trusted = False." - }, - { - "path": "src/tool_security.py", - "symbol": "NON_ADMIN_BLOCKED_TOOLS", - "line_range": "L42-L78", - "explanation": "Enforces tool execution safety for non-admin user roles." - }, - { - "path": "tests/test_skill_index_prompt_injection.py", - "symbol": "test_skill_index", - "line_range": "L1-L208", - "explanation": "Verifies skill index descriptions cannot leak into trusted system prompts." - }, - { - "path": "tests/test_tool_output_prompt_injection.py", - "symbol": "test_tool_output", - "line_range": "L1-L50", - "explanation": "Tool output injection guards." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "SECURITY-005", - "domain": "security", - "name": "URL & Path Confinement Security Guards", - "purpose": "Prevents SSRF attacks and path traversal by validating target IP addresses and resolving symlinks.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/fileHandler.js" - ], - "backend_entrypoints": [ - "src/url_security.py:validate_url", - "src/url_safety.py", - "core/log_safety.py" - ], - "routes": [ - "Implicit guard functions" - ], - "configuration": [ - "ALLOWED_DOMAINS_WHITELIST" - ], - "persistence": [], - "dependencies": [ - "ipaddress", - "urllib.parse" - ], - "tests": [ - "tests/test_url_safety.py", - "tests/test_tool_path_confinement.py", - "tests/test_workspace_confine.py" - ], - "documentation": [ - "THREAT_MODEL.md" - ], - "risks": [ - "DNS rebinding attacks if IP address is re-resolved post-validation." - ], - "unknowns": [], - "evidence": [ - { - "path": "src/url_safety.py", - "symbol": "check_outbound_url", - "line_range": "L60-L108", - "explanation": "Rejects non-HTTP(S) schemes, link-local, cloud metadata SSRF addresses." - }, - { - "path": "src/url_security.py", - "symbol": "validate_public_http_url", - "line_range": "L81-L94", - "explanation": "Validates public-facing endpoints." - }, - { - "path": "tests/test_url_safety.py", - "symbol": "test_url_safety", - "line_range": "L1-L117", - "explanation": "Scheme validation, cloud metadata SSRF rejection, IP classification." - }, - { - "path": "tests/test_tool_path_confinement.py", - "symbol": "test_path_confinement", - "line_range": "L1-L50", - "explanation": "Path traversal checks." - }, - { - "path": "tests/test_workspace_confine.py", - "symbol": "test_workspace_confine", - "line_range": "L1-L50", - "explanation": "Workspace confinement checks." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "SECURITY-006", - "domain": "security", - "name": "HTTP Security Headers Middleware", - "purpose": "Injects standard OWASP HTTP security headers (CSP, HSTS, X-Content-Type-Options, X-Frame-Options).", - "status": "verified", - "frontend_entrypoints": [ - "static/app.js" - ], - "backend_entrypoints": [ - "core/middleware.py:SecurityHeadersMiddleware" - ], - "routes": [ - "Applies globally to all routes" - ], - "configuration": [ - "CSP_NONCE_ENABLED" - ], - "persistence": [], - "dependencies": [ - "starlette.middleware.base" - ], - "tests": [ - "tests/test_security_headers_middleware.py", - "tests/test_security_headers_pdf_preview.py" - ], - "documentation": [ - "SECURITY.md" - ], - "risks": [ - "Strict Content Security Policy (CSP) blocking third-party embedded web resources." - ], - "unknowns": [], - "evidence": [ - { - "path": "core/middleware.py", - "symbol": "SecurityHeadersMiddleware", - "line_range": "L40-L110", - "explanation": "Sets strict security headers and CSP nonces on HTTP responses." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "SECURITY-007", - "domain": "security", - "name": "Admin System Data Wipe ('Danger Zone')", - "purpose": "Provides administrative reset operations to wipe sessions, cache, uploaded files, or factory reset state.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/admin.js" - ], - "backend_entrypoints": [ - "routes/admin_wipe/admin_wipe_routes.py:setup_admin_wipe_routes" - ], - "routes": [ - "/api/admin/wipe/{kind}" - ], - "configuration": [], - "persistence": [ - "ALL storage locations" - ], - "dependencies": [ - "os", - "shutil" - ], - "tests": [ - "tests/test_session_ghost_delete.py" - ], - "documentation": [ - "THREAT_MODEL.md" - ], - "risks": [ - "Accidental catastrophic data loss if triggered without user confirmation." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/admin_wipe/admin_wipe_routes.py", - "symbol": "@router.delete('/wipe/{kind}')", - "line_range": "L71-L130", - "explanation": "Executes systemic data wipe based on requested scope." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "FRONTEND-001", - "domain": "frontend", - "name": "Single Page Application Shell & Client Router", - "purpose": "Main HTML5 SPA shell, DOM lifecycle initializers, tab navigation, and deep-link route handlers.", - "status": "verified", - "frontend_entrypoints": [ - "static/index.html", - "static/app.js", - "static/js/init.js" - ], - "backend_entrypoints": [ - "app.py:serve_index" - ], - "routes": [ - "/", - "/notes", - "/calendar", - "/cookbook", - "/email", - "/memory", - "/gallery", - "/tasks", - "/library" - ], - "configuration": [], - "persistence": [ - "static/index.html" - ], - "dependencies": [ - "Vanilla HTML5/JS" - ], - "tests": [ - "tests/test_app_static_mime.py", - "tests/test_serve_html_with_nonce.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Stale browser static cache if asset hashing is omitted during deployment." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/index.html", - "symbol": "index.html", - "line_range": "L1-L200", - "explanation": "Main SPA entry point containing modal roots and CSS bundles." - }, - { - "path": "app.py", - "symbol": "serve_index", - "line_range": "L867-L878", - "explanation": "Serves index.html with dynamically generated CSP nonces." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "FRONTEND-002", - "domain": "frontend", - "name": "Dynamic Theme, Color System & Custom Fonts", - "purpose": "Supports dark/light themes, custom CSS variables, color picker controls, and user font uploads.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/theme.js", - "static/style.css", - "static/js/colorPicker.js" - ], - "backend_entrypoints": [ - "routes/font_routes.py:setup_font_routes", - "routes/prefs_routes.py:setup_prefs_routes" - ], - "routes": [ - "/api/font/custom", - "/api/prefs" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/user_prefs.json", - "static/fonts/custom/" - ], - "dependencies": [ - "Vanilla CSS", - "pydantic" - ], - "tests": [ - "tests/test_prefs_atomic_write.py", - "tests/test_select_dropdown_theme_css.py" - ], - "documentation": [ - "docs/theme.webm" - ], - "risks": [ - "Flash of unstyled content (FOUC) on slow connections." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/js/theme.js", - "symbol": "applyTheme", - "line_range": "L1-L90", - "explanation": "Applies custom HSL theme variables to DOM document root." - }, - { - "path": "routes/font_routes.py", - "symbol": "setup_font_routes", - "line_range": "L20-L55", - "explanation": "Allows uploading and serving custom WOFF2 font files." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "FRONTEND-003", - "domain": "frontend", - "name": "Window Manager, Tile Layout & Modal Control System", - "purpose": "Manages draggable tool windows, snapped multi-tile viewports, modal dialog Z-ordering, and ESC key stacks.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/modalManager.js", - "static/js/tileManager.js", - "static/js/windowDrag.js", - "static/js/windowResize.js", - "static/js/escMenuStack.js" - ], - "backend_entrypoints": [ - "None (Client UI subsystem)" - ], - "routes": [], - "configuration": [ - "localStorage: tile_layout_state" - ], - "persistence": [ - "Browser localStorage" - ], - "dependencies": [ - "Vanilla JS DOM event listeners" - ], - "tests": [ - "tests/test_portal_dropdown_z_js.py", - "tests/test_tile_manager_snap_zones_js.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Overlap artifacts when opening many simultaneous tool floating windows." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/js/modalManager.js", - "symbol": "ModalManager", - "line_range": "L1-L140", - "explanation": "Controls modal open/close transitions and focus trapping." - }, - { - "path": "static/js/tileManager.js", - "symbol": "TileManager", - "line_range": "L1-L180", - "explanation": "Handles viewport split-pane grid arrangements." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "FRONTEND-004", - "domain": "frontend", - "name": "Global Keyboard Shortcuts & Accessibility Controls", - "purpose": "Provides configurable hotkeys (Ctrl+K search, Esc close, Alt+1-9 tabs) and high-contrast accessibility options.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/keyboard-shortcuts.js", - "static/js/a11y.js" - ], - "backend_entrypoints": [ - "None (Client UI subsystem)" - ], - "routes": [], - "configuration": [ - "localStorage: user_hotkeys" - ], - "persistence": [ - "Browser localStorage" - ], - "dependencies": [ - "Vanilla JS" - ], - "tests": [ - "tests/test_keyboard_shortcuts.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Browser keybinding collisions with browser default hotkeys." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/js/keyboard-shortcuts.js", - "symbol": "initShortcuts", - "line_range": "L1-L110", - "explanation": "Binds global keydown handlers for system shortcuts." - }, - { - "path": "static/js/a11y.js", - "symbol": "initA11y", - "line_range": "L1-L80", - "explanation": "Applies ARIA roles and dyslexic font toggles." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "FRONTEND-005", - "domain": "frontend", - "name": "Markdown, LaTeX & Code Block Streaming Renderer", - "purpose": "Parses incoming SSE markdown streams, renders KaTeX math formulas, syntax-highlighted code, and interactive runners.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/markdown.js", - "static/js/streamingRenderer.js", - "static/js/streamingSegmenter.js", - "static/js/codeRunner.js" - ], - "backend_entrypoints": [ - "None (Client rendering subsystem)" - ], - "routes": [], - "configuration": [], - "persistence": [ - "Browser DOM" - ], - "dependencies": [ - "highlight.js", - "katex" - ], - "tests": [ - "tests/streaming/segmenter.test.mjs", - "tests/test_streaming_segmenter_js.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "DOM thrashing if streaming segmenter updates UI too frequently." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/js/markdown.js", - "symbol": "renderMarkdown", - "line_range": "L1-L200", - "explanation": "Converts markdown prose to HTML nodes with syntax highlighting." - }, - { - "path": "static/js/streamingSegmenter.js", - "symbol": "Segmenter", - "line_range": "L1-L150", - "explanation": "Parses un-closed markdown fences during live stream." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "FRONTEND-006", - "domain": "frontend", - "name": "Interactive Tour & Guided Onboarding System", - "purpose": "Presents interactive step-by-step feature tours and UI tooltip hints for new users.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/tourHints.js", - "static/js/tourAutoplay.js" - ], - "backend_entrypoints": [ - "None (Client UI subsystem)" - ], - "routes": [], - "configuration": [ - "localStorage: tour_completed" - ], - "persistence": [ - "Browser localStorage" - ], - "dependencies": [ - "Vanilla JS" - ], - "tests": [ - "tests/test_task_routes.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Tour step misalignment if window is resized mid-tour." - ], - "unknowns": [], - "evidence": [ - { - "path": "static/js/tourHints.js", - "symbol": "startTour", - "line_range": "L1-L120", - "explanation": "Renders guided feature tour overlays over target UI elements." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "FRONTEND-007", - "domain": "frontend", - "name": "Background Effects Prototyping Sandbox", - "purpose": "Standalone sandbox page for prototyping visual background animations, waves, and whirlpool effects.", - "status": "dead-code-candidate", - "frontend_entrypoints": [ - "static/wave-variants.html", - "static/whirlpool-variants.html" - ], - "backend_entrypoints": [ - "app.py:serve_backgrounds" - ], - "routes": [ - "/backgrounds" - ], - "configuration": [], - "persistence": [ - "None" - ], - "dependencies": [ - "HTML5 Canvas", - "WebGL" - ], - "tests": [], - "documentation": [ - "README.md" - ], - "risks": [ - "High GPU utilization when rendering complex shader wave animations." - ], - "unknowns": [ - "Route `/backgrounds` in app.py L918 attempts to serve `static/backgrounds.html` which is missing from disk; variant templates `wave-variants.html` and `whirlpool-variants.html` exist." - ], - "evidence": [ - { - "path": "app.py", - "symbol": "serve_backgrounds", - "line_range": "L915-L918", - "explanation": "Serves visual background sandbox HTML page route." - }, - { - "path": "static/wave-variants.html", - "symbol": "wave-variants.html", - "line_range": "L1-L150", - "explanation": "Interactive background effect prototyping sandbox variant." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E1", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-001", - "domain": "platform", - "name": "Application Initialization & Lifespan Management", - "purpose": "Orchestrates server startup, database table migration, background daemon initialization, and clean shutdown.", - "status": "verified", - "frontend_entrypoints": [ - "app.py" - ], - "backend_entrypoints": [ - "app.py:_lifespan", - "src/app_initializer.py:initialize_app" - ], - "routes": [ - "App startup lifespan" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/" - ], - "dependencies": [ - "asyncio", - "logging" - ], - "tests": [ - "tests/test_app_helpers.py" - ], - "documentation": [ - "docs/setup.md" - ], - "risks": [ - "Un-handled exceptions during startup halt application launch." - ], - "unknowns": [], - "evidence": [ - { - "path": "app.py", - "symbol": "_lifespan", - "line_range": "L996-L1030", - "explanation": "FastAPI lifespan context manager executing startup tasks." - }, - { - "path": "src/app_initializer.py", - "symbol": "initialize_app", - "line_range": "L29-L125", - "explanation": "Initializes app directories, DB schemas, and logging." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-002", - "domain": "platform", - "name": "System Health, Readiness & Version Monitoring APIs", - "purpose": "Exposes Liveness (/api/health), Readiness (/api/ready), App Version (/api/version), and Client Perf APIs.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/admin.js" - ], - "backend_entrypoints": [ - "app.py:health_check", - "app.py:readiness_check", - "app.py:get_version", - "src/readiness.py" - ], - "routes": [ - "/api/health", - "/api/ready", - "/api/version", - "/api/runtime", - "/api/client-perf" - ], - "configuration": [], - "persistence": [], - "dependencies": [ - "fastapi" - ], - "tests": [ - "tests/test_readiness.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Readiness check delays if verifying connectivity to offline remote endpoints." - ], - "unknowns": [], - "evidence": [ - { - "path": "app.py", - "symbol": "readiness_check", - "line_range": "L963-L973", - "explanation": "Performs system component integrity check." - }, - { - "path": "src/readiness.py", - "symbol": "check_readiness", - "line_range": "L15-L61", - "explanation": "Checks database, storage, and key paths for read/write access." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-003", - "domain": "platform", - "name": "Database Schema, Migrations & SQLite Persistence", - "purpose": "Defines core relational tables (users, tokens, tasks, sessions) and executes automated SQLite schema upgrades.", - "status": "verified", - "frontend_entrypoints": [ - "scripts/update_database.py" - ], - "backend_entrypoints": [ - "core/database.py:init_db", - "scripts/update_database.py:run_migrations", - "src/database.py" - ], - "routes": [ - "Implicit DB queries" - ], - "configuration": [ - "DATABASE_URL" - ], - "persistence": [ - "DATA_DIR/odysseus.db" - ], - "dependencies": [ - "sqlalchemy", - "alembic", - "sqlite3" - ], - "tests": [ - "tests/test_sqlite_foreign_keys.py", - "tests/test_update_database_script.py", - "tests/test_app_db_permissions.py" - ], - "documentation": [ - "docs/setup.md" - ], - "risks": [ - "SQLite file lock contention under high concurrent write loads." - ], - "unknowns": [], - "evidence": [ - { - "path": "core/database.py", - "symbol": "init_db", - "line_range": "L40-L120", - "explanation": "Creates ORM tables and establishes connection pool." - }, - { - "path": "scripts/update_database.py", - "symbol": "run_migrations", - "line_range": "L20-L110", - "explanation": "Applies missing schema columns and indices." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-004", - "domain": "platform", - "name": "User Data Export & Import Backup Infrastructure", - "purpose": "Exports complete user workspace state (sessions, memory, skills, notes, presets) into a zip archive.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/settings.js" - ], - "backend_entrypoints": [ - "routes/backup_routes.py:setup_backup_routes", - "scripts/odysseus-backup" - ], - "routes": [ - "/api/backup/export", - "/api/backup/import" - ], - "configuration": [], - "persistence": [ - "ZIP archive files" - ], - "dependencies": [ - "zipfile", - "json" - ], - "tests": [ - "tests/cli/test_preset_cli_store.py" - ], - "documentation": [ - "docs/backup-restore.md" - ], - "risks": [ - "Corrupt archive files causing partial data restore." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/backup_routes.py", - "symbol": "setup_backup_routes", - "line_range": "L30-L140", - "explanation": "Handles workspace data export and import upload unpack." - }, - { - "path": "docs/backup-restore.md", - "symbol": "Documentation", - "line_range": "L1-L50", - "explanation": "Backup and restore operational documentation." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-005", - "domain": "platform", - "name": "File Cleanup & Storage Maintenance Engine", - "purpose": "Scans data directories for orphaned files, old uploads, temporary vision images, and frees disk space.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/storage.js" - ], - "backend_entrypoints": [ - "routes/cleanup/cleanup_routes.py:setup_cleanup_routes", - "src/cleanup_service.py", - "src/session_image_cleanup.py" - ], - "routes": [ - "/api/cleanup/preview", - "/api/cleanup" - ], - "configuration": [ - "CLEANUP_RETENTION_DAYS" - ], - "persistence": [ - "DATA_DIR/uploads/" - ], - "dependencies": [ - "os", - "shutil" - ], - "tests": [ - "tests/test_session_actions_cleanup.py", - "tests/test_session_image_cleanup.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Deletes files uploaded in active sessions if retention window is set too short." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/cleanup/cleanup_routes.py", - "symbol": "@router.get('/preview')", - "line_range": "L22-L37", - "explanation": "Previews reclaimable disk space across storage directories." - }, - { - "path": "src/cleanup_service.py", - "symbol": "CleanupService", - "line_range": "L25-L120", - "explanation": "Executes filesystem purge of orphaned asset files." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-006", - "domain": "platform", - "name": "System Health & RAG Diagnostic Suite", - "purpose": "Executes real-time integrity diagnostics across ChromaDB, SearXNG, local models, and network interfaces.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/settings.js" - ], - "backend_entrypoints": [ - "routes/diagnostics_routes.py:setup_diagnostics_routes", - "src/service_health.py" - ], - "routes": [ - "/api/diagnostics" - ], - "configuration": [], - "persistence": [], - "dependencies": [ - "httpx", - "chromadb" - ], - "tests": [ - "tests/test_service_health_collect.py", - "tests/test_service_health_chromadb.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Diagnostic timeout if external search provider is unreachable." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/diagnostics_routes.py", - "symbol": "setup_diagnostics_routes", - "line_range": "L30-L100", - "explanation": "Runs subsystem health check suite." - }, - { - "path": "src/service_health.py", - "symbol": "collect_health_status", - "line_range": "L20-L150", - "explanation": "Inspects vector database, email, search, and local provider status." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-007", - "domain": "platform", - "name": "Desktop CLI Utilities & Shell Integration Tools", - "purpose": "Provides command-line interface tools (`odysseus`, `odysseus-mcp`, `odysseus-mail`) for terminal usage.", - "status": "verified", - "frontend_entrypoints": [ - "scripts/odysseus", - "scripts/odysseus-mcp", - "scripts/odysseus-mail", - "scripts/odysseus-calendar" - ], - "backend_entrypoints": [ - "scripts/_lib/cli.py:main" - ], - "routes": [ - "CLI commands" - ], - "configuration": [], - "persistence": [ - "CLI configuration" - ], - "dependencies": [ - "urllib", - "json" - ], - "tests": [ - "tests/cli/test_sessions_cli.py", - "tests/cli/test_mail_cli_recipients.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Outdated CLI scripts if backend API schemas change." - ], - "unknowns": [], - "evidence": [ - { - "path": "scripts/_lib/cli.py", - "symbol": "main", - "line_range": "L15-L110", - "explanation": "Shared CLI framework for terminal helper commands." - }, - { - "path": "scripts/odysseus", - "symbol": "odysseus", - "line_range": "L1-L50", - "explanation": "Main terminal launcher script." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-008", - "domain": "platform", - "name": "Desktop Companion App Integration", - "purpose": "Provides API routes and pairing mechanisms for the native macOS/desktop menu bar companion app.", - "status": "verified", - "frontend_entrypoints": [ - "companion/README.md" - ], - "backend_entrypoints": [ - "companion/pairing.py", - "companion/routes.py:setup_companion_routes" - ], - "routes": [ - "/companion/pair", - "/companion/status" - ], - "configuration": [ - "COMPANION_SECRET_KEY" - ], - "persistence": [ - "companion_pairing.json" - ], - "dependencies": [ - "fastapi" - ], - "tests": [ - "tests/helpers/import_state.py" - ], - "documentation": [ - "companion/README.md" - ], - "risks": [ - "Pairing code expiration timing window." - ], - "unknowns": [], - "evidence": [ - { - "path": "companion/routes.py", - "symbol": "setup_companion_routes", - "line_range": "L20-L120", - "explanation": "Endpoints for pairing and status sync with desktop companion." - }, - { - "path": "companion/pairing.py", - "symbol": "PairingManager", - "line_range": "L15-L80", - "explanation": "Generates and validates companion pairing codes." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "PLATFORM-009", - "domain": "platform", - "name": "Docker Containerization & GPU Hardware Manifests", - "purpose": "Provides multi-stage Dockerfile and Docker Compose manifests for CPU, NVIDIA CUDA, and AMD ROCm GPUs.", - "status": "verified", - "frontend_entrypoints": [ - "Dockerfile", - "docker-compose.yml", - "docker-compose.gpu-nvidia.yml", - "docker-compose.gpu-amd.yml" - ], - "backend_entrypoints": [ - "docker/entrypoint.sh" - ], - "routes": [ - "Containerized application" - ], - "configuration": [ - "DOCKER_GPU_VENDOR" - ], - "persistence": [ - "Container volumes" - ], - "dependencies": [ - "Docker", - "NVIDIA Container Toolkit" - ], - "tests": [ - "scripts/check-docker-gpu.sh" - ], - "documentation": [ - "docs/setup.md" - ], - "risks": [ - "Driver version incompatibility with host NVIDIA/AMD kernel drivers." - ], - "unknowns": [], - "evidence": [ - { - "path": "Dockerfile", - "symbol": "multi-stage-build", - "line_range": "L1-L113", - "explanation": "Multi-stage container build environment." - }, - { - "path": "docker-compose.gpu-nvidia.yml", - "symbol": "nvidia-gpu-manifest", - "line_range": "L1-L179", - "explanation": "NVIDIA GPU pass-through container specification." - }, - { - "path": "scripts/check-docker-gpu.sh", - "symbol": "check-docker-gpu", - "line_range": "L1-L615", - "explanation": "Automated diagnostic test script for host NVIDIA GPU passthrough." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": true, - "status": "pending", - "reason": "Requires Docker GPU pass-through and compatible host drivers." - } - }, - { - "id": "PLATFORM-010", - "domain": "platform", - "name": "Legacy FAISS Vector Index Migration Script", - "purpose": "Legacy utility script to migrate older FAISS vector indices into ChromaDB.", - "status": "legacy", - "frontend_entrypoints": [ - "None" - ], - "backend_entrypoints": [ - "scripts/migrate_faiss_to_chroma.py" - ], - "routes": [ - "CLI Script" - ], - "configuration": [], - "persistence": [ - "Legacy FAISS index files" - ], - "dependencies": [ - "faiss", - "chromadb" - ], - "tests": [ - "tests/helpers/import_state.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Fails if legacy FAISS index files do not exist." - ], - "unknowns": [ - "Superseded by native ChromaDB vector index pipeline." - ], - "evidence": [ - { - "path": "scripts/migrate_faiss_to_chroma.py", - "symbol": "migrate_faiss", - "line_range": "L15-L80", - "explanation": "Reads FAISS vector index files and writes to ChromaDB collection." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "NOTE-001", - "domain": "note", - "name": "Interactive Notes & Checklist Management", - "purpose": "Provides Google Keep-style notes, rich markdown text, checklist items, pinning, color tags, and reminders.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/notes.js" - ], - "backend_entrypoints": [ - "routes/note/note_routes.py:setup_note_routes", - "src/tools/notes.py" - ], - "routes": [ - "/api/notes", - "/api/notes/{note_id}", - "/api/notes/{note_id}/pin", - "/api/notes/reorder" - ], - "configuration": [], - "persistence": [ - "DATA_DIR/notes.db" - ], - "dependencies": [ - "sqlite3", - "pydantic" - ], - "tests": [ - "tests/test_notes_fail_closed_auth.py", - "tests/test_manage_notes_owner_gate.py" - ], - "documentation": [ - "docs/notes.webm" - ], - "risks": [ - "Concurrent edits on note item checkboxes." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/note/note_routes.py", - "symbol": "@router.get('')", - "line_range": "L623-L650", - "explanation": "Lists all user notes with pin and archive states." - }, - { - "path": "static/js/notes.js", - "symbol": "initNotesView", - "line_range": "L1-L160", - "explanation": "Main interactive notes grid and modal manager." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "SKILL-001", - "domain": "skill", - "name": "Dynamic Skill Management & Code Execution Engine", - "purpose": "Allows users to create, import, edit, test, and execute custom Python/Markdown skills dynamically.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/skills.js" - ], - "backend_entrypoints": [ - "routes/skills_routes.py:setup_skills_routes", - "services/memory/skills.py:SkillsManager" - ], - "routes": [ - "/api/skills", - "/api/skills/{skill_id}/invoke", - "/api/skills/import-from-url" - ], - "configuration": [ - "SKILLS_DIR" - ], - "persistence": [ - "DATA_DIR/skills/" - ], - "dependencies": [ - "pydantic", - "httpx" - ], - "tests": [ - "tests/test_skills_routes_owner_update.py", - "tests/test_skill_importer.py", - "tests/test_skill_save_no_rename.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "Arbitrary code execution risks if skill import URL is untrusted." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/skills_routes.py", - "symbol": "setup_skills_routes", - "line_range": "L100-L300", - "explanation": "Exposes CRUD and remote import routes for user skills." - }, - { - "path": "services/memory/skills.py", - "symbol": "SkillsManager", - "line_range": "L40-L220", - "explanation": "Handles skill storage, parsing, and execution." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "CONTACT-001", - "domain": "contact", - "name": "CardDAV Contact Management & Address Book Integration", - "purpose": "Connects to CardDAV servers, imports VCard contacts, and provides contact lookup for email/calendar autocomplete.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/emailLibrary.js" - ], - "backend_entrypoints": [ - "routes/contacts/contacts_routes.py:setup_contacts_routes", - "src/tools/contacts.py" - ], - "routes": [ - "/api/contacts/list", - "/api/contacts/search", - "/api/contacts/add", - "/api/contacts/config" - ], - "configuration": [ - "CARDDAV_URL" - ], - "persistence": [ - "DATA_DIR/contacts.db" - ], - "dependencies": [ - "vobject", - "sqlite3" - ], - "tests": [ - "tests/test_contacts_carddav_security.py", - "tests/cli/test_contacts_cli_rows.py" - ], - "documentation": [ - "README.md" - ], - "risks": [ - "VCard 3.0 vs 4.0 property parsing mismatches." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/contacts/contacts_routes.py", - "symbol": "@router.get('/list')", - "line_range": "L741-L764", - "explanation": "Returns contact list filtered by search query." - }, - { - "path": "src/tools/contacts.py", - "symbol": "ContactsTool", - "line_range": "L20-L110", - "explanation": "Agent tool for querying user address book contacts." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - }, - { - "id": "MEMORY-001", - "domain": "memory", - "name": "Persistent Long-Term Memory & Vector Indexing", - "purpose": "Extracts facts, user preferences, and temporal memories from chat sessions into vector/relational storage.", - "status": "verified", - "frontend_entrypoints": [ - "static/js/memory.js" - ], - "backend_entrypoints": [ - "routes/memory/memory_routes.py:setup_memory_routes", - "services/memory/service.py", - "mcp_servers/memory_server.py" - ], - "routes": [ - "/api/memory", - "/api/memory/search", - "/api/memory/extract", - "/api/memory/audit" - ], - "configuration": [ - "MEMORY_AUTO_EXTRACT" - ], - "persistence": [ - "DATA_DIR/memory.db", - "DATA_DIR/memory_chroma/" - ], - "dependencies": [ - "sqlite3", - "chromadb" - ], - "tests": [ - "tests/test_memory_routes_session_owner.py", - "tests/test_consolidate_memory_explicit_drops.py" - ], - "documentation": [ - "docs/theme.webm" - ], - "risks": [ - "Conflicting memory facts extracted from contradictory user prompts." - ], - "unknowns": [], - "evidence": [ - { - "path": "routes/memory/memory_routes.py", - "symbol": "@router.get('')", - "line_range": "L132-L150", - "explanation": "Fetches long-term user memory timeline." - }, - { - "path": "services/memory/memory_extractor.py", - "symbol": "MemoryExtractor", - "line_range": "L30-L160", - "explanation": "LLM-driven fact extraction from conversation transcripts." - } - ], - "verified_at_commit": "d8a2059df8e53bc7275c45339849d14c8651e73c", - "evidence_maturity": "E0", - "runtime_validation": { - "required": false, - "status": "not-required", - "reason": "No separate environment-dependent runtime validation was identified during this documentation pass." - } - } -] diff --git a/docs/discovery/feature-catalog.md b/docs/discovery/feature-catalog.md deleted file mode 100644 index 9a93c0317..000000000 --- a/docs/discovery/feature-catalog.md +++ /dev/null @@ -1,85 +0,0 @@ -# Feature Catalog - -This is a human-readable index derived from [`feature-catalog.json`](feature-catalog.json). The JSON file is canonical. - -| ID | Feature | Domain | Status | Evidence | Runtime | -|---|---|---|---|---|---| -| `AGENT-001` | Autonomous Agent Loop & Tool Execution Engine | `agent` | `verified` | `E0` | not required | -| `AGENT-002` | Scheduled Tasks & Event Bus Dispatcher | `agent` | `verified` | `E0` | not required | -| `AGENT-003` | Webhook Event Subscriptions & Trigger Processing | `agent` | `verified` | `E0` | not required | -| `AGENT-004` | Assistant Settings, Task Check-Ins & Background Job Monitor | `agent` | `partial` | `E1` | not required | -| `AGENT-005` | Model Context Protocol (MCP) Server Integration | `agent` | `verified` | `E0` | not required | -| `AGENT-006` | AI Interaction Tools & Pipeline Orchestration | `agent` | `verified` | `E0` | not required | -| `AGENT-007` | Subprocess & Background Job Execution Tools | `agent` | `verified` | `E0` | not required | -| `CALENDAR-001` | CalDAV Calendar Synchronization & Account Setup | `calendar` | `verified` | `E0` | pending | -| `CALENDAR-002` | Calendar Event Operations & iCalendar Parsing | `calendar` | `verified` | `E0` | not required | -| `CHAT-001` | Core Chat Streaming & SSE Message Generation | `chat` | `verified` | `E2` | pending | -| `CHAT-002` | Session Management & Conversation State | `chat` | `verified` | `E0` | not required | -| `CHAT-003` | Chat History & Message Editing/Truncation | `chat` | `verified` | `E0` | not required | -| `CHAT-004` | File & Multimodal Attachment Handling | `chat` | `verified` | `E0` | not required | -| `CHAT-005` | Chat Message Search | `chat` | `verified` | `E0` | not required | -| `CHAT-006` | System Prompts & Preset Management | `chat` | `verified` | `E0` | not required | -| `CHAT-007` | Emoji Rendering & Twemoji SVG Proxy | `chat` | `verified` | `E0` | not required | -| `CHAT-008` | Input History Recall (Arrow Up) | `chat` | `verified` | `E0` | not required | -| `CHAT-009` | Context Window Compaction & Truncation | `chat` | `verified` | `E0` | not required | -| `CONTACT-001` | CardDAV Contact Management & Address Book Integration | `contact` | `verified` | `E0` | not required | -| `COOKBOOK-001` | Local Model Download & Recipe Lifecycle Management | `cookbook` | `verified` | `E0` | not required | -| `COOKBOOK-002` | Hardware Model Fitting ('What Fits?') Analysis Engine | `cookbook` | `verified` | `E0` | not required | -| `COOKBOOK-003` | HuggingFace & MLX Model Discovery Services | `cookbook` | `verified` | `E0` | not required | -| `COOKBOOK-004` | Host Docker Access for Inference Container Runtimes | `cookbook` | `verified` | `E0` | pending | -| `DOCUMENT-001` | Document & Canvas Artifact Management | `document` | `verified` | `E0` | not required | -| `DOCUMENT-002` | PDF Form Processing & High-Fidelity Rendering | `document` | `verified` | `E1` | pending | -| `DOCUMENT-003` | Personal Document Indexing & RAG Retrieval | `document` | `verified` | `E0` | not required | -| `DOCUMENT-004` | Document Conversion & Text Extraction Engine | `document` | `verified` | `E0` | not required | -| `DOCUMENT-005` | Document Library UI Navigation | `document` | `verified` | `E0` | not required | -| `EMAIL-001` | Email Account Setup, IMAP/SMTP Connection & Polling | `email` | `verified` | `E1` | pending | -| `EMAIL-002` | Email Searching, Threading & Message Operations | `email` | `verified` | `E0` | not required | -| `EMAIL-003` | Email Composition, Draft Management & Sending | `email` | `verified` | `E0` | pending | -| `EMAIL-004` | Email MCP Server & Codex Integration Bridge | `email` | `verified` | `E0` | not required | -| `FRONTEND-001` | Single Page Application Shell & Client Router | `frontend` | `verified` | `E0` | not required | -| `FRONTEND-002` | Dynamic Theme, Color System & Custom Fonts | `frontend` | `verified` | `E0` | not required | -| `FRONTEND-003` | Window Manager, Tile Layout & Modal Control System | `frontend` | `verified` | `E0` | not required | -| `FRONTEND-004` | Global Keyboard Shortcuts & Accessibility Controls | `frontend` | `verified` | `E0` | not required | -| `FRONTEND-005` | Markdown, LaTeX & Code Block Streaming Renderer | `frontend` | `verified` | `E0` | not required | -| `FRONTEND-006` | Interactive Tour & Guided Onboarding System | `frontend` | `verified` | `E0` | not required | -| `FRONTEND-007` | Background Effects Prototyping Sandbox | `frontend` | `dead-code-candidate` | `E1` | not required | -| `MEDIA-001` | Gallery Image Library & Album Operations | `media` | `verified` | `E0` | not required | -| `MEDIA-002` | Image Processing, AI Upscaling & Style Transfer | `media` | `verified` | `E0` | not required | -| `MEDIA-003` | Interactive Image Canvas Editor & Persisted Drafts | `media` | `verified` | `E0` | not required | -| `MEDIA-004` | Text-to-Speech (TTS) Synthesis Service | `media` | `verified` | `E0` | not required | -| `MEDIA-005` | Speech-to-Text (STT) Audio Transcription Service | `media` | `verified` | `E0` | not required | -| `MEDIA-006` | Digital Signature Stamp Storage & Placement | `media` | `verified` | `E0` | not required | -| `MEDIA-007` | Generated Image Artifact Route & MCP Integration | `media` | `verified` | `E0` | not required | -| `MEDIA-008` | Native MLX Image Bridge (macOS Apple Silicon) | `media` | `experimental` | `E0` | pending | -| `MEMORY-001` | Persistent Long-Term Memory & Vector Indexing | `memory` | `verified` | `E0` | not required | -| `MODEL-001` | Multi-Provider LLM Model Discovery & Metadata Management | `model` | `verified` | `E0` | not required | -| `MODEL-002` | Model Capability & Context Limits Detection | `model` | `verified` | `E0` | not required | -| `MODEL-003` | LLM Core Provider Communication & Fallback Routing | `model` | `verified` | `E0` | pending | -| `MODEL-004` | Model Selection & Display Ordering Preferences | `model` | `verified` | `E0` | not required | -| `MODEL-005` | Side-by-Side Model Comparison (A/B Testing) | `model` | `verified` | `E0` | not required | -| `MODEL-006` | GitHub Copilot Device Flow Authentication | `model` | `verified` | `E2` | pending | -| `MODEL-007` | ChatGPT Subscription Device Flow Authentication | `model` | `verified` | `E2` | pending | -| `MODEL-008` | Embedding Model Lane & Vector Provider Setup | `model` | `verified` | `E0` | not required | -| `NOTE-001` | Interactive Notes & Checklist Management | `note` | `verified` | `E0` | not required | -| `PLATFORM-001` | Application Initialization & Lifespan Management | `platform` | `verified` | `E0` | not required | -| `PLATFORM-002` | System Health, Readiness & Version Monitoring APIs | `platform` | `verified` | `E0` | not required | -| `PLATFORM-003` | Database Schema, Migrations & SQLite Persistence | `platform` | `verified` | `E0` | not required | -| `PLATFORM-004` | User Data Export & Import Backup Infrastructure | `platform` | `verified` | `E0` | not required | -| `PLATFORM-005` | File Cleanup & Storage Maintenance Engine | `platform` | `verified` | `E0` | not required | -| `PLATFORM-006` | System Health & RAG Diagnostic Suite | `platform` | `verified` | `E0` | not required | -| `PLATFORM-007` | Desktop CLI Utilities & Shell Integration Tools | `platform` | `verified` | `E0` | not required | -| `PLATFORM-008` | Desktop Companion App Integration | `platform` | `verified` | `E0` | not required | -| `PLATFORM-009` | Docker Containerization & GPU Hardware Manifests | `platform` | `verified` | `E0` | pending | -| `PLATFORM-010` | Legacy FAISS Vector Index Migration Script | `platform` | `legacy` | `E0` | not required | -| `RESEARCH-001` | Deep Research Execution Engine & SSE Progress Streaming | `research` | `verified` | `E0` | not required | -| `RESEARCH-002` | Research Library, Detail View & Image Controls | `research` | `verified` | `E0` | not required | -| `RESEARCH-003` | Web Search Engine Integration (SearXNG & Multi-Provider) | `research` | `verified` | `E1` | pending | -| `RESEARCH-004` | Research Result Peeking & Topic Spinoff Generation | `research` | `verified` | `E0` | not required | -| `SECURITY-001` | Authentication, Session Cookies & User Management | `security` | `verified` | `E0` | not required | -| `SECURITY-002` | System Vault Encrypted Secret Storage | `security` | `verified` | `E1` | pending | -| `SECURITY-003` | API Token Management & Scope Access Control | `security` | `verified` | `E0` | not required | -| `SECURITY-004` | Prompt Security & Injection Defense Engine | `security` | `verified` | `E1` | not required | -| `SECURITY-005` | URL & Path Confinement Security Guards | `security` | `verified` | `E1` | not required | -| `SECURITY-006` | HTTP Security Headers Middleware | `security` | `verified` | `E0` | not required | -| `SECURITY-007` | Admin System Data Wipe ('Danger Zone') | `security` | `verified` | `E0` | not required | -| `SKILL-001` | Dynamic Skill Management & Code Execution Engine | `skill` | `verified` | `E0` | not required | diff --git a/docs/discovery/references/source-provenance.md b/docs/discovery/references/source-provenance.md deleted file mode 100644 index c7a9a581d..000000000 --- a/docs/discovery/references/source-provenance.md +++ /dev/null @@ -1,28 +0,0 @@ -# Source Provenance & Audit Baseline - -## Target Repository & Snapshot - -- **Repository**: `odysseus-dev/odysseus` -- **Branch**: `discovery` -- **Pinned Commit SHA**: `d8a2059df8e53bc7275c45339849d14c8651e73c` -- **Snapshot Date**: `2026-07-23T14:49:02Z` - -## Discovery Package Organization - -The public discovery documentation package under `docs/discovery/` is structured as follows: - -- `feature-catalog.json`: Canonical machine-readable JSON catalog containing 79 feature records. -- `feature-catalog.md`: Human-readable summary derived from `feature-catalog.json`. -- `BASELINE-STATUS.md`: Publication status, evidence-validation snapshot, and durable maintainer guidance. -- `audit-method.md`: Audit rules, scope, and evidence maturity definitions (E0 to E4). -- `domains/`: 16 functional domain markdown files detailing feature implementations. -- `references/`: Audit provenance and repository snapshot metadata. -- `tools/`: Structural, consistency, and evidence validators with focused evidence-validator tests. - -## Exclusion Principles - -This public documentation package explicitly excludes: -- Internal planning artifacts or private meeting notes. -- Machine-specific filesystem paths or user environments. -- API keys, credentials, or private service endpoints. -- Application code or automated test mutations. diff --git a/docs/discovery/tools/test_validate_discovery_evidence.py b/docs/discovery/tools/test_validate_discovery_evidence.py deleted file mode 100644 index 5e3144fe0..000000000 --- a/docs/discovery/tools/test_validate_discovery_evidence.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python3 -"""Focused negative tests for validate_discovery_evidence.py.""" - -from __future__ import annotations - -import importlib.util -import shutil -import sys -import tempfile -import unittest -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("validate_discovery_evidence.py") -SPEC = importlib.util.spec_from_file_location("validate_discovery_evidence", MODULE_PATH) -assert SPEC and SPEC.loader -validator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = validator -SPEC.loader.exec_module(validator) - - -class FakeJavascriptParser: - supported = True - reason = "test parser" - - def __init__(self, symbols: list[validator.Located] | None = None) -> None: - self.symbols = symbols or [] - - def parse(self, path: Path) -> list[validator.Located]: - return self.symbols - - -class UnsupportedJavascriptParser: - supported = False - reason = "no repository-local parser" - - -class EvidenceNegativeTests(unittest.TestCase): - def setUp(self) -> None: - self.temp = tempfile.TemporaryDirectory() - self.root = Path(self.temp.name) - (self.root / "sample.py").write_text( - "from fastapi import APIRouter\n" - "router = APIRouter(prefix='/api')\n" - "\n" - "class ChatHandler:\n" - " def preprocess_message(self):\n" - " return True\n" - "\n" - "@router.post('/chat')\n" - "def chat_stream():\n" - " return True\n", - encoding="utf-8", - ) - (self.root / "sample.js").write_text( - "export const present = () => true;\n", encoding="utf-8" - ) - (self.root / "sample.sh").write_text( - "#!/usr/bin/env bash\nreal_function() {\n return 0\n}\n", - encoding="utf-8", - ) - self.backup = self.root / "copied-backups" - self.backup.mkdir() - for path in self.root.glob("sample.*"): - shutil.copy2(path, self.backup / path.name) - - def tearDown(self) -> None: - for backup in self.backup.iterdir(): - target = self.root / backup.name - shutil.copy2(backup, target) - self.assertEqual(target.read_bytes(), backup.read_bytes()) - self.temp.cleanup() - - def validate( - self, - evidence: dict[str, str], - javascript_parser: object | None = None, - ) -> validator.Validation: - return validator.validate_evidence( - self.root, - "TEST-001", - 0, - evidence, - javascript_parser or FakeJavascriptParser(), - ) - - def test_missing_python_symbol(self) -> None: - result = self.validate( - { - "path": "sample.py", - "kind": "python-function", - "locator": "fabricated", - "line_range": "L1-L1", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-locator") - - def test_incorrect_qualified_method(self) -> None: - result = self.validate( - { - "path": "sample.py", - "kind": "python-method", - "locator": "WrongHandler.preprocess_message", - "line_range": "L5-L6", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-locator") - - def test_symbol_outside_cited_range(self) -> None: - result = self.validate( - { - "path": "sample.py", - "kind": "python-method", - "locator": "ChatHandler.preprocess_message", - "line_range": "L1-L2", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "locator-outside-range") - - def test_fabricated_test_function(self) -> None: - result = self.validate( - { - "path": "sample.py", - "kind": "test-function", - "locator": "test_fabricated", - "line_range": "L1-L2", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-locator") - - def test_nonexistent_javascript_symbol_with_parser(self) -> None: - result = self.validate( - { - "path": "sample.js", - "kind": "javascript-function", - "locator": "missing", - "line_range": "L1-L1", - "explanation": "negative fixture", - }, - FakeJavascriptParser( - [validator.Located("present", "javascript-export", 1, 1)] - ), - ) - self.assertEqual(result.result, "invalid-locator") - - def test_unsupported_javascript_parser(self) -> None: - result = self.validate( - { - "path": "sample.js", - "kind": "javascript-function", - "locator": "present", - "line_range": "L1-L1", - "explanation": "negative fixture", - }, - UnsupportedJavascriptParser(), - ) - self.assertEqual(result.result, "unsupported") - - def test_route_path_mismatch(self) -> None: - result = self.validate( - { - "path": "sample.py", - "kind": "python-route", - "locator": "POST /api/wrong -> chat_stream", - "line_range": "L9-L10", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-locator") - self.assertIn("path", result.problem or "") - - def test_http_method_mismatch(self) -> None: - result = self.validate( - { - "path": "sample.py", - "kind": "python-route", - "locator": "GET /api/chat -> chat_stream", - "line_range": "L9-L10", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-locator") - self.assertIn("method", result.problem or "") - - def test_shell_function_mismatch(self) -> None: - result = self.validate( - { - "path": "sample.sh", - "kind": "shell-function", - "locator": "fabricated", - "line_range": "L1-L4", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-locator") - - def test_invalid_file_level_evidence(self) -> None: - result = self.validate( - { - "path": "missing.file", - "kind": "file", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-path") - - def test_file_level_evidence_rejects_fake_symbol(self) -> None: - result = self.validate( - { - "path": "sample.sh", - "kind": "file", - "locator": "whole-script", - "explanation": "negative fixture", - } - ) - self.assertEqual(result.result, "invalid-locator") - - -if __name__ == "__main__": - unittest.main() diff --git a/docs/discovery/tools/validate_discovery_consistency.py b/docs/discovery/tools/validate_discovery_consistency.py deleted file mode 100755 index 953b093e1..000000000 --- a/docs/discovery/tools/validate_discovery_consistency.py +++ /dev/null @@ -1,592 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import json -import re -import sys -from collections import Counter, defaultdict -from pathlib import Path -from typing import Any - - -ROOT = Path(__file__).resolve().parents[1] -CATALOG_PATH = ROOT / "feature-catalog.json" -CATALOG_MD_PATH = ROOT / "feature-catalog.md" -DOMAINS_DIR = ROOT / "domains" -REVIEWS_DIR = ROOT / "reviews" - -EXPECTED_COMMIT = "d8a2059df8e53bc7275c45339849d14c8651e73c" -EXPECTED_FEATURES = 79 -EXPECTED_DOMAINS = 16 - -VALID_STATUSES = { - "verified", - "partial", - "disabled", - "experimental", - "legacy", - "dead-code-candidate", - "unverified", -} - -VALID_MATURITY = {"E0", "E1", "E2", "E3", "E4"} - -VALID_RUNTIME = { - "not-required", - "pending", - "blocked", - "completed", -} - -REQUIRED_FIELDS = { - "id", - "domain", - "name", - "purpose", - "status", - "evidence_maturity", - "verified_at_commit", - "evidence", - "runtime_validation", -} - -FEATURE_ID_RE = re.compile(r"^[A-Z][A-Z0-9]*-\d{3}$") - -DOMAIN_HEADING_RE = re.compile( - r"^##\s+`?([A-Z][A-Z0-9]*-\d{3})`?" - r"\s+(?:—|-)\s+(.+?)\s*$" -) - -DOMAIN_FIELD_RE = re.compile( - r"^-\s+\*\*" - r"(Domain|Status|Evidence Maturity|Commit Verified)" - r"\*\*:\s*(.*?)\s*$" -) - -DOMAIN_RUNTIME_RE = re.compile( - r"^-\s+\*\*Runtime Validation\*\*:\s*(.*?)\s*$" -) - -REVIEW_HEADING_RE = re.compile( - r"^###\s+(?:\d+\.\s+)?" - r"([A-Z][A-Z0-9]*-\d{3})" - r"\s+(?:—|-)\s+.+$" -) - -REVIEW_RESULT_RE = re.compile( - r"^-\s+\*\*Resulting Status(?:\s+and|/)\s+Maturity\*\*:" - r"\s*`([^`]+)`\s*/\s*`([^`]+)`\s*$" -) - - -def clean(value: str) -> str: - value = value.strip() - - if ( - len(value) >= 2 - and value.startswith("`") - and value.endswith("`") - ): - return value[1:-1].strip() - - return value - - -def load_catalog(errors: list[str]) -> list[dict[str, Any]]: - try: - data = json.loads( - CATALOG_PATH.read_text(encoding="utf-8") - ) - except Exception as exc: - errors.append(f"Unable to read catalog: {exc}") - return [] - - if isinstance(data, list): - features = data - elif isinstance(data, dict) and isinstance(data.get("features"), list): - features = data["features"] - else: - errors.append( - "Catalog must be an array or contain a features array" - ) - return [] - - if not all(isinstance(feature, dict) for feature in features): - errors.append("Every catalog feature must be an object") - return [] - - return features - - -def validate_catalog( - features: list[dict[str, Any]], - errors: list[str], -) -> None: - if len(features) != EXPECTED_FEATURES: - errors.append( - f"Expected {EXPECTED_FEATURES} features, " - f"found {len(features)}" - ) - - ids = [feature.get("id") for feature in features] - - duplicates = sorted( - feature_id - for feature_id, count in Counter(ids).items() - if feature_id and count > 1 - ) - - if duplicates: - errors.append( - "Duplicate feature IDs: " + ", ".join(duplicates) - ) - - for index, feature in enumerate(features): - feature_id = feature.get("id") - label = ( - feature_id - if isinstance(feature_id, str) - else f"<index:{index}>" - ) - - missing = sorted( - field - for field in REQUIRED_FIELDS - if feature.get(field) in (None, "", []) - ) - - if missing: - errors.append( - f"{label}: missing fields: {', '.join(missing)}" - ) - - if ( - not isinstance(feature_id, str) - or not FEATURE_ID_RE.fullmatch(feature_id) - ): - errors.append(f"{label}: invalid feature ID") - - if feature.get("status") not in VALID_STATUSES: - errors.append( - f"{label}: invalid status " - f"{feature.get('status')!r}" - ) - - if feature.get("evidence_maturity") not in VALID_MATURITY: - errors.append( - f"{label}: invalid maturity " - f"{feature.get('evidence_maturity')!r}" - ) - - if feature.get("verified_at_commit") != EXPECTED_COMMIT: - errors.append( - f"{label}: incorrect verified_at_commit" - ) - - runtime = feature.get("runtime_validation") - - if not isinstance(runtime, dict): - errors.append( - f"{label}: runtime_validation must be an object" - ) - continue - - required = runtime.get("required") - runtime_status = runtime.get("status") - reason = runtime.get("reason") - - if not isinstance(required, bool): - errors.append( - f"{label}: runtime required must be boolean" - ) - - if runtime_status not in VALID_RUNTIME: - errors.append( - f"{label}: invalid runtime status " - f"{runtime_status!r}" - ) - - if not isinstance(reason, str) or not reason.strip(): - errors.append( - f"{label}: runtime reason is blank" - ) - - if required is False and runtime_status != "not-required": - errors.append( - f"{label}: required=false requires not-required" - ) - - if required is True and runtime_status == "not-required": - errors.append( - f"{label}: required=true cannot be not-required" - ) - - -def parse_domain( - path: Path, - errors: list[str], -) -> dict[str, dict[str, str]]: - records: dict[str, dict[str, str]] = {} - current_id: str | None = None - - for line_number, line in enumerate( - path.read_text(encoding="utf-8").splitlines(), - start=1, - ): - heading = DOMAIN_HEADING_RE.match(line) - - if heading: - current_id = heading.group(1) - - if current_id in records: - errors.append( - f"{path.relative_to(ROOT)}:{line_number}: " - f"duplicate heading {current_id}" - ) - - records[current_id] = { - "Name": heading.group(2).strip(), - } - continue - - field = DOMAIN_FIELD_RE.match(line) - - if field and current_id: - value = clean(field.group(2)) - - if not value: - errors.append( - f"{path.relative_to(ROOT)}:{line_number}: " - f"blank {field.group(1)}" - ) - - records[current_id][field.group(1)] = value - continue - - runtime = DOMAIN_RUNTIME_RE.match(line) - - if runtime and current_id: - value = clean(runtime.group(1)) - runtime_status = re.split( - r"\s+(?:—|-)\s+", - value, - maxsplit=1, - )[0] - records[current_id]["Runtime Validation"] = ( - runtime_status.strip("` ") - ) - - return records - - -def validate_domains( - features: list[dict[str, Any]], - errors: list[str], -) -> None: - catalog = { - feature["id"]: feature - for feature in features - if feature.get("id") - } - - expected_by_domain: defaultdict[str, set[str]] = defaultdict(set) - - for feature in features: - expected_by_domain[feature["domain"]].add(feature["id"]) - - paths = sorted(DOMAINS_DIR.glob("*.md")) - - if len(paths) != EXPECTED_DOMAINS: - errors.append( - f"Expected {EXPECTED_DOMAINS} domain files, " - f"found {len(paths)}" - ) - - all_found: set[str] = set() - - for path in paths: - domain = path.stem - records = parse_domain(path, errors) - found = set(records) - expected = expected_by_domain.get(domain, set()) - all_found.update(found) - - if found != expected: - missing = sorted(expected - found) - extra = sorted(found - expected) - - errors.append( - f"{domain}: missing={missing}, unexpected={extra}" - ) - - for feature_id, record in records.items(): - feature = catalog.get(feature_id) - - if feature is None: - continue - - expected_values = { - "Name": feature["name"], - "Domain": feature["domain"], - "Status": feature["status"], - "Evidence Maturity": feature["evidence_maturity"], - "Commit Verified": feature["verified_at_commit"], - "Runtime Validation": ( - feature["runtime_validation"]["status"] - ), - } - - for field, expected_value in expected_values.items(): - actual = record.get(field) - - if actual != expected_value: - errors.append( - f"{path.relative_to(ROOT)}: " - f"{feature_id} {field}: " - f"{actual!r} != {expected_value!r}" - ) - - if all_found != set(catalog): - errors.append( - "Domain Markdown IDs do not match catalog JSON" - ) - - -def validate_catalog_markdown( - features: list[dict[str, Any]], - errors: list[str], -) -> None: - expected = { - feature["id"]: feature - for feature in features - } - found: dict[str, list[str]] = {} - - for line in CATALOG_MD_PATH.read_text( - encoding="utf-8" - ).splitlines(): - if not line.startswith("|"): - continue - - cells = [ - cell.strip() - for cell in line.strip().strip("|").split("|") - ] - - if len(cells) < 6: - continue - - feature_id = clean(cells[0]) - - if FEATURE_ID_RE.fullmatch(feature_id): - found[feature_id] = cells - - if set(found) != set(expected): - errors.append( - "feature-catalog.md IDs do not match JSON" - ) - - for feature_id, cells in found.items(): - feature = expected[feature_id] - runtime = feature["runtime_validation"] - runtime_display = ( - runtime["status"] - if runtime["required"] - else "not required" - ) - - actual = { - "name": cells[1].replace("\\|", "|"), - "domain": clean(cells[2]), - "status": clean(cells[3]), - "maturity": clean(cells[4]), - "runtime": clean(cells[5]), - } - - wanted = { - "name": feature["name"], - "domain": feature["domain"], - "status": feature["status"], - "maturity": feature["evidence_maturity"], - "runtime": runtime_display, - } - - for field, expected_value in wanted.items(): - if actual[field] != expected_value: - errors.append( - f"feature-catalog.md: {feature_id} " - f"{field}: {actual[field]!r} " - f"!= {expected_value!r}" - ) - - -def validate_reviews( - features: list[dict[str, Any]], - errors: list[str], -) -> int: - catalog = { - feature["id"]: feature - for feature in features - } - checked = 0 - - for path in sorted( - REVIEWS_DIR.glob("evidence-sample-*.md") - ): - current_id: str | None = None - results: set[str] = set() - - for line_number, line in enumerate( - path.read_text(encoding="utf-8").splitlines(), - start=1, - ): - heading = REVIEW_HEADING_RE.match(line) - - if heading: - current_id = heading.group(1) - continue - - result = REVIEW_RESULT_RE.match(line) - - if not result or current_id is None: - continue - - status, maturity = result.groups() - results.add(current_id) - checked += 1 - - feature = catalog.get(current_id) - - if feature is None: - errors.append( - f"{path.relative_to(ROOT)}:{line_number}: " - f"unknown feature {current_id}" - ) - continue - - if feature["status"] != status: - errors.append( - f"{current_id}: review status {status!r} " - f"!= catalog {feature['status']!r}" - ) - - if feature["evidence_maturity"] != maturity: - errors.append( - f"{current_id}: review maturity {maturity!r} " - f"!= catalog " - f"{feature['evidence_maturity']!r}" - ) - - if path.name == "evidence-sample-01.md" and len(results) != 12: - errors.append( - f"{path.relative_to(ROOT)}: expected 12 " - f"review results, found {len(results)}" - ) - - return checked - - -def validate_whitespace(errors: list[str]) -> None: - for path in sorted(ROOT.rglob("*")): - if not path.is_file(): - continue - - if path.suffix not in {".md", ".json", ".py", ".txt"}: - continue - - for line_number, line in enumerate( - path.read_text( - encoding="utf-8", - errors="replace", - ).splitlines(), - start=1, - ): - if line != line.rstrip(" \t"): - errors.append( - f"{path.relative_to(ROOT)}:{line_number}: " - "trailing whitespace" - ) - - if any(ROOT.rglob("*.pyc")): - errors.append("Generated .pyc files exist") - - if any( - path.is_dir() - for path in ROOT.rglob("__pycache__") - ): - errors.append("__pycache__ exists") - - -def main() -> int: - errors: list[str] = [] - features = load_catalog(errors) - reviewed = 0 - - if features: - validate_catalog(features, errors) - validate_domains(features, errors) - validate_catalog_markdown(features, errors) - reviewed = validate_reviews(features, errors) - - validate_whitespace(errors) - - print(f"Catalog Features: {len(features)}") - print( - "Unique Feature IDs:", - len({feature.get("id") for feature in features}), - ) - print( - "Domain Files:", - len(list(DOMAINS_DIR.glob("*.md"))), - ) - - if features: - print( - "Statuses:", - dict( - Counter( - feature.get("status") - for feature in features - ) - ), - ) - print( - "Evidence Maturity:", - dict( - Counter( - feature.get("evidence_maturity") - for feature in features - ) - ), - ) - print( - "Runtime Validation:", - dict( - Counter( - ( - feature.get( - "runtime_validation", - {}, - ).get("required"), - feature.get( - "runtime_validation", - {}, - ).get("status"), - ) - for feature in features - ) - ), - ) - - print(f"Review Results Checked: {reviewed}") - print(f"Consistency Errors: {len(errors)}") - - for error in errors: - print(f"ERROR: {error}") - - return 1 if errors else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docs/discovery/tools/validate_discovery_docs.py b/docs/discovery/tools/validate_discovery_docs.py deleted file mode 100755 index 656b800ec..000000000 --- a/docs/discovery/tools/validate_discovery_docs.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python3 -"""Validate the Odysseus public discovery documentation package.""" - -from __future__ import annotations - -import argparse -import json -import re -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -REPO_ROOT = ROOT.parents[1] -CATALOG_PATH = ROOT / "feature-catalog.json" -DOMAINS_DIR = ROOT / "domains" - -EXPECTED_COMMIT = "d8a2059df8e53bc7275c45339849d14c8651e73c" -ALLOWED_STATUSES = { - "verified", "partial", "disabled", "experimental", "legacy", - "dead-code-candidate", "unverified" -} - -LINE_RANGE_RE = re.compile(r"^L([1-9]\d*)-L([1-9]\d*)$") -FEATURE_HEADING_RE = re.compile( - r"^##\s+`?([A-Z][A-Z0-9]*-\d{3})`?\s+(?:—|-)\s+.+$", - re.MULTILINE, -) -LINK_RE = re.compile(r"(?<!!)\[[^\]]*\]\(([^)]+)\)") -FORBIDDEN_TERMS_RE = re.compile( - r"(roadforge|kanban|matrix|owner link|editor link|github support|" - r"private maintainer|OD-AUD-|[A-Z]{2,10}-AUD-\d+|TASK-\d+)", - re.IGNORECASE, -) - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Validate Odysseus discovery docs.") - parser.add_argument( - "--repo-root", - type=Path, - default=REPO_ROOT, - help="Path to Odysseus repository root.", - ) - return parser.parse_args() - -def validate() -> int: - args = parse_args() - repo_root = args.repo_root.resolve() - errors: list[str] = [] - - # 1. Validate Catalog JSON existence and content - if not CATALOG_PATH.is_file(): - errors.append(f"Missing catalog file: {CATALOG_PATH}") - print(f"Errors: {len(errors)}") - for e in errors: - print(f"ERROR: {e}") - return 1 - - try: - catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8")) - except Exception as e: - errors.append(f"Failed to parse catalog JSON: {e}") - print(f"Errors: {len(errors)}") - for err in errors: - print(f"ERROR: {err}") - return 1 - - if not isinstance(catalog, list): - errors.append("feature-catalog.json must be a JSON array") - return 1 - - feature_ids = [item.get("id") for item in catalog if isinstance(item, dict)] - if len(feature_ids) != 79: - errors.append(f"Expected 79 unique feature IDs, found {len(feature_ids)}") - if len(feature_ids) != len(set(feature_ids)): - errors.append("Duplicate feature IDs found in catalog JSON") - - # Domain catalog counts - catalog_domain_counts: dict[str, int] = {} - for item in catalog: - if not isinstance(item, dict): - errors.append("Catalog item is not an object") - continue - fid = item.get("id", "<missing>") - status = item.get("status") - domain = item.get("domain", "").lower() - catalog_domain_counts[domain] = catalog_domain_counts.get(domain, 0) + 1 - - if status not in ALLOWED_STATUSES: - errors.append(f"{fid}: Invalid status '{status}'") - - evidence_list = item.get("evidence") - if not isinstance(evidence_list, list) or not evidence_list: - errors.append(f"{fid}: Missing or empty evidence list") - continue - - for ev in evidence_list: - if not isinstance(ev, dict): - errors.append(f"{fid}: Evidence item is not an object") - continue - path_str = ev.get("path") - lr_str = str(ev.get("line_range", "")) - if not path_str or Path(path_str).is_absolute() or ".." in Path(path_str).parts: - errors.append(f"{fid}: Unsafe or invalid path '{path_str}'") - continue - - # Check path exists in repo - target_path = repo_root / path_str - if not target_path.is_file(): - errors.append(f"{fid}: Referenced path '{path_str}' does not exist on disk") - continue - - # Check line range format & bounds - m = LINE_RANGE_RE.fullmatch(lr_str) - if not m: - errors.append(f"{fid}: Invalid line range format '{lr_str}' for path '{path_str}'") - continue - - start, end = int(m.group(1)), int(m.group(2)) - lines_cnt = len(target_path.read_text(encoding="utf-8", errors="ignore").splitlines()) - if start > end or end > lines_cnt or start < 1: - errors.append( - f"{fid}: Line range '{lr_str}' exceeds file length ({lines_cnt} lines) in '{path_str}'" - ) - - # 2. Check Domain Markdown files - md_feature_ids: list[str] = [] - domain_files = sorted(DOMAINS_DIR.glob("*.md")) - for df in domain_files: - domain_name = df.stem.lower() - content = df.read_text(encoding="utf-8") - found_ids = FEATURE_HEADING_RE.findall(content) - md_feature_ids.extend(found_ids) - if len(found_ids) != catalog_domain_counts.get(domain_name, 0): - errors.append( - f"Domain '{domain_name}' count mismatch: catalog has {catalog_domain_counts.get(domain_name, 0)}, Markdown has {len(found_ids)}" - ) - - if sorted(md_feature_ids) != sorted(feature_ids): - errors.append("Markdown domain feature IDs do not match catalog JSON feature IDs") - - # 3. Check for forbidden/private terms, sensitive credentials, and broken links across all docs - for md_file in ROOT.rglob("*.md"): - rel_md = md_file.relative_to(ROOT) - content = md_file.read_text(encoding="utf-8") - - # Forbidden terms scan - forbidden_matches = FORBIDDEN_TERMS_RE.findall(content) - if forbidden_matches: - errors.append( - f"{rel_md}: Found forbidden/internal terms: {set(forbidden_matches)}" - ) - - # Broken local link check - for target in LINK_RE.findall(content): - target = target.strip().strip("<>") - if not target or target.startswith(("#", "http://", "https://", "mailto:")): - continue - target_path = target.split("#", 1)[0] - resolved = (md_file.parent / target_path).resolve() - if not resolved.exists(): - errors.append(f"{rel_md}: Broken local link '{target}'") - - print(f"Catalog Features: {len(catalog)}") - print(f"Domain Files: {len(domain_files)}") - print(f"Validation Errors: {len(errors)}") - for err in errors: - print(f"ERROR: {err}") - - return 1 if errors else 0 - -if __name__ == "__main__": - sys.exit(validate()) diff --git a/docs/discovery/tools/validate_discovery_evidence.py b/docs/discovery/tools/validate_discovery_evidence.py deleted file mode 100644 index 4315478f7..000000000 --- a/docs/discovery/tools/validate_discovery_evidence.py +++ /dev/null @@ -1,1413 +0,0 @@ -#!/usr/bin/env python3 -"""Validate feature-catalog evidence locators against repository source. - -Evidence identity is resolved from a language parser where one is available. -The cited line range is treated as generated display metadata and checked only -after the locator has been resolved. -""" - -from __future__ import annotations - -import argparse -import ast -import difflib -import json -import re -import subprocess -import sys -from collections import Counter, defaultdict -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, Iterable - - -DISCOVERY_ROOT = Path(__file__).resolve().parents[1] -DEFAULT_REPO_ROOT = DISCOVERY_ROOT.parents[1] -DEFAULT_CATALOG = DISCOVERY_ROOT / "feature-catalog.json" - -RESULTS = { - "valid", - "invalid-path", - "invalid-locator", - "locator-outside-range", - "range-mismatch", - "ambiguous", - "unsupported", - "file-level-valid", -} - -PYTHON_KINDS = { - "python-function", - "python-method", - "python-class", - "python-variable", - "python-route", - "python-module", - "test-function", -} -JAVASCRIPT_KINDS = { - "javascript-function", - "javascript-class", - "javascript-export", - "javascript-event", -} -FILE_EXTENSIONS = { - ".html", - ".md", - ".swift", - ".yaml", - ".yml", -} -RANGE_RE = re.compile(r"^L([1-9]\d*)-L([1-9]\d*)$") -LEGACY_ROUTE_RE = re.compile( - r"^@(?P<router>[A-Za-z_]\w*)\." - r"(?P<method>get|post|put|patch|delete|options|head)" - r"\(\s*['\"](?P<path>[^'\"]*)['\"]\s*\)$", - re.IGNORECASE, -) -CANONICAL_ROUTE_RE = re.compile( - r"^(?P<method>GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\s+" - r"(?P<path>\S+)(?:\s+->\s+(?P<function>[\w.]+))?$" -) -SHELL_FUNCTION_RE = re.compile( - r"^\s*(?:function\s+)?(?P<name>[A-Za-z_]\w*)\s*" - r"(?:\(\s*\))?\s*\{" -) -MARKDOWN_HEADING_RE = re.compile(r"^(?P<marks>#{1,6})\s+(?P<title>.+?)\s*$") - - -@dataclass(frozen=True) -class Located: - name: str - kind: str - start: int - end: int - details: dict[str, Any] | None = None - - @property - def line_range(self) -> str: - return f"L{self.start}-L{self.end}" - - -@dataclass -class Validation: - feature_id: str - evidence_index: int - current_evidence: dict[str, Any] - inferred_kind: str - locator: str | None - result: str - problem: str | None = None - resolved: Located | None = None - suggested_kind: str | None = None - suggested_locator: str | None = None - generated_line_range: str | None = None - confidence: str = "high" - manual_review_reason: str | None = None - - def to_dict(self) -> dict[str, Any]: - data = asdict(self) - data["resolved"] = asdict(self.resolved) if self.resolved else None - return data - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Validate discovery evidence identities and line ranges." - ) - parser.add_argument("--repo-root", type=Path, default=DEFAULT_REPO_ROOT) - parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) - parser.add_argument( - "--output-dir", - type=Path, - default=DISCOVERY_ROOT, - help="Directory for the report and repair queue.", - ) - return parser.parse_args(argv) - - -def load_catalog(path: Path) -> list[dict[str, Any]]: - data = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(data, list) or not all(isinstance(item, dict) for item in data): - raise ValueError("catalog must be an array of feature objects") - return data - - -def safe_target(repo_root: Path, raw_path: Any) -> Path | None: - if not isinstance(raw_path, str) or not raw_path: - return None - candidate = Path(raw_path) - if candidate.is_absolute() or ".." in candidate.parts: - return None - resolved = (repo_root / candidate).resolve() - try: - resolved.relative_to(repo_root.resolve()) - except ValueError: - return None - return resolved - - -def parse_range(raw: Any) -> tuple[int, int] | None: - if raw in (None, ""): - return None - match = RANGE_RE.fullmatch(str(raw)) - if not match: - return None - return int(match.group(1)), int(match.group(2)) - - -def extension_for(path: str) -> str: - suffix = Path(path).suffix.lower() - return suffix or "[none]" - - -def legacy_locator(evidence: dict[str, Any]) -> str | None: - value = evidence.get("locator", evidence.get("symbol")) - return value if isinstance(value, str) and value.strip() else None - - -def infer_kind(evidence: dict[str, Any], target: Path | None) -> str: - explicit = evidence.get("kind") - if isinstance(explicit, str) and explicit: - return explicit - - path = str(evidence.get("path", "")) - suffix = Path(path).suffix.lower() - locator = legacy_locator(evidence) or "" - - if suffix == ".py": - if LEGACY_ROUTE_RE.fullmatch(locator) or CANONICAL_ROUTE_RE.fullmatch(locator): - return "python-route" - if path.startswith("tests/") and locator.startswith("test"): - return "test-function" - if "." in locator and all(part.isidentifier() for part in locator.split(".")): - return "python-method" - if locator.isupper(): - return "python-variable" - if target and target.is_file(): - try: - tree = ast.parse(target.read_text(encoding="utf-8")) - except (OSError, SyntaxError, UnicodeError): - return "manual-review" - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef) and node.name == locator: - return "python-class" - return "python-function" - if suffix == ".js": - if locator.startswith("on:") or locator.startswith("event:"): - return "javascript-event" - if locator and locator[:1].isupper(): - return "javascript-class" - return "javascript-function" - if suffix in {".sh", ".bash"}: - return "shell-function" - if suffix == ".md": - return "documentation-section" - if suffix in {".json", ".yaml", ".yml", ".toml", ".ini"}: - return "configuration-key" - if suffix in FILE_EXTENSIONS or Path(path).name in {"Dockerfile"}: - return "manual-review" - if not suffix: - return "manual-review" - return "manual-review" - - -class PythonIndex(ast.NodeVisitor): - """Collect Python declarations, assignments, and decorated routes.""" - - def __init__(self) -> None: - self.symbols: list[Located] = [] - self.routes: list[Located] = [] - self.scope: list[str] = [] - self.constants: dict[str, str] = {} - self.router_prefixes: dict[str, str | None] = {} - - @classmethod - def from_path(cls, path: Path) -> "PythonIndex": - index = cls() - tree = ast.parse(path.read_text(encoding="utf-8")) - index._collect_static_values(tree) - index.visit(tree) - return index - - def _collect_static_values(self, tree: ast.AST) -> None: - for node in getattr(tree, "body", []): - if isinstance(node, (ast.Assign, ast.AnnAssign)): - value = self._string_value(node.value) - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - for target in targets: - if isinstance(target, ast.Name) and value is not None: - self.constants[target.id] = value - if isinstance(target, ast.Name) and isinstance( - node.value, ast.Call - ): - call_name = self._call_name(node.value.func) - if call_name.endswith("APIRouter"): - prefix = self._router_prefix(node.value) - self.router_prefixes[target.id] = prefix - - def _qualified(self, name: str) -> str: - return ".".join([*self.scope, name]) - - @staticmethod - def _call_name(node: ast.AST) -> str: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - return f"{PythonIndex._call_name(node.value)}.{node.attr}".strip(".") - return "" - - def _string_value(self, node: ast.AST | None) -> str | None: - if isinstance(node, ast.Constant) and isinstance(node.value, str): - return node.value - if isinstance(node, ast.Name): - return self.constants.get(node.id) - if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): - left = self._string_value(node.left) - right = self._string_value(node.right) - return left + right if left is not None and right is not None else None - return None - - def _keyword_string(self, call: ast.Call, name: str) -> str | None: - for keyword in call.keywords: - if keyword.arg == name: - return self._string_value(keyword.value) - return None - - def _router_prefix(self, call: ast.Call) -> str | None: - if not any(keyword.arg == "prefix" for keyword in call.keywords): - return "" - return self._keyword_string(call, "prefix") - - def visit_ClassDef(self, node: ast.ClassDef) -> None: - self.symbols.append( - Located( - self._qualified(node.name), - "python-class", - node.lineno, - node.end_lineno or node.lineno, - ) - ) - self.scope.append(node.name) - self.generic_visit(node) - self.scope.pop() - - def visit_FunctionDef(self, node: ast.FunctionDef) -> None: - self._visit_function(node) - - def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: - self._visit_function(node) - - def _visit_function(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: - qualified = self._qualified(node.name) - parent_is_class = bool( - self.scope - and any( - item.kind == "python-class" and item.name == ".".join(self.scope) - for item in self.symbols - ) - ) - kind = ( - "test-function" - if node.name.startswith("test") - else "python-method" - if parent_is_class - else "python-function" - ) - located = Located( - qualified, kind, node.lineno, node.end_lineno or node.lineno - ) - self.symbols.append(located) - self._collect_routes(node, qualified) - self.scope.append(node.name) - self.generic_visit(node) - self.scope.pop() - - def _collect_routes( - self, node: ast.FunctionDef | ast.AsyncFunctionDef, qualified: str - ) -> None: - methods = {"get", "post", "put", "patch", "delete", "options", "head"} - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or not isinstance( - decorator.func, ast.Attribute - ): - continue - method = decorator.func.attr.lower() - if method not in methods: - continue - router = self._call_name(decorator.func.value) - local_path = ( - self._string_value(decorator.args[0]) if decorator.args else None - ) - if local_path is None: - continue - prefix = self.router_prefixes.get(router) - full_path = None if router in self.router_prefixes and prefix is None else ( - f"{prefix or ''}{local_path}" - ) - details = { - "method": method.upper(), - "path": local_path, - "full_path": full_path, - "function": qualified, - "router": router, - "router_prefix": prefix, - "prefix_resolved": full_path is not None, - } - route_name = f"{method.upper()} {full_path or local_path} -> {qualified}" - self.routes.append( - Located( - route_name, - "python-route", - node.lineno, - node.end_lineno or node.lineno, - details, - ) - ) - - def visit_Assign(self, node: ast.Assign) -> None: - if isinstance(node.value, ast.Call): - call_name = self._call_name(node.value.func) - if call_name.endswith("APIRouter"): - for target in node.targets: - if isinstance(target, ast.Name): - self.router_prefixes[target.id] = self._router_prefix(node.value) - for target in node.targets: - self._collect_assignment(target, node) - self.generic_visit(node) - - def visit_AnnAssign(self, node: ast.AnnAssign) -> None: - self._collect_assignment(node.target, node) - self.generic_visit(node) - - def _collect_assignment(self, target: ast.AST, node: ast.AST) -> None: - if isinstance(target, (ast.Tuple, ast.List)): - for item in target.elts: - self._collect_assignment(item, node) - return - if not isinstance(target, ast.Name): - return - self.symbols.append( - Located( - self._qualified(target.id), - "python-variable", - node.lineno, - node.end_lineno or node.lineno, - ) - ) - - -NODE_PARSER = r""" -const fs = require("fs"); -const parserName = process.argv[1]; -const parserEntry = process.argv[2]; -const filename = process.argv[3]; -const source = fs.readFileSync(filename, "utf8"); -const parser = require(parserEntry); -let tree; -if (parserName === "@babel/parser") { - tree = parser.parse(source, { - sourceType: "unambiguous", - plugins: ["jsx", "classProperties", "optionalChaining", "topLevelAwait"] - }); -} else if (parserName === "typescript") { - throw new Error("typescript parser normalization is not implemented"); -} else { - const parse = parser.parse || parser.Parser?.parse; - tree = parse.call(parser, source, { - ecmaVersion: "latest", sourceType: "module", locations: true - }); -} -const out = []; -function loc(node) { - const l = node.loc; - return {start: l.start.line, end: l.end.line}; -} -function add(name, kind, node, extra={}) { - if (!name || !node.loc) return; - out.push({name, kind, ...loc(node), details: extra}); -} -function walk(node, scope=[], exported=false) { - if (!node || typeof node !== "object") return; - if (node.type === "ExportNamedDeclaration" || node.type === "ExportDefaultDeclaration") { - if (node.declaration) walk(node.declaration, scope, true); - return; - } - if (node.type === "FunctionDeclaration") { - const name = node.id?.name; - add([...scope, name].filter(Boolean).join("."), exported ? "javascript-export" : "javascript-function", node); - scope = [...scope, name].filter(Boolean); - } else if (node.type === "ClassDeclaration") { - const name = node.id?.name; - add([...scope, name].filter(Boolean).join("."), exported ? "javascript-export" : "javascript-class", node); - scope = [...scope, name].filter(Boolean); - } else if (node.type === "MethodDefinition" || node.type === "ClassMethod") { - const name = node.key?.name || node.key?.value; - add([...scope, name].filter(Boolean).join("."), "javascript-function", node); - scope = [...scope, name].filter(Boolean); - } else if (node.type === "VariableDeclarator" && node.id?.type === "Identifier") { - const initType = node.init?.type || ""; - const isFunction = initType === "ArrowFunctionExpression" || initType === "FunctionExpression"; - if (isFunction || exported) { - add([...scope, node.id.name].join("."), exported ? "javascript-export" : "javascript-function", node); - } - } else if (node.type === "CallExpression" && node.callee?.type === "MemberExpression" - && node.callee.property?.name === "addEventListener" - && ["Literal", "StringLiteral"].includes(node.arguments?.[0]?.type)) { - add("event:" + node.arguments[0].value, "javascript-event", node, {event: node.arguments[0].value}); - } - for (const [key, value] of Object.entries(node)) { - if (key === "loc" || key === "start" || key === "end") continue; - if (Array.isArray(value)) value.forEach(child => walk(child, scope, exported)); - else if (value && typeof value === "object") walk(value, scope, exported); - } -} -walk(tree.program || tree); -process.stdout.write(JSON.stringify(out)); -""" - - -class JavascriptParser: - CANDIDATES = ("acorn", "espree", "@babel/parser", "typescript") - - def __init__(self, repo_root: Path) -> None: - self.repo_root = repo_root.resolve() - self.node = self._node_binary() - self.parser_name, self.parser_entry, self.reason = self._find_parser() - - @staticmethod - def _node_binary() -> str | None: - from shutil import which - - return which("node") - - def _package_roots(self) -> list[tuple[Path, set[str]]]: - roots: list[tuple[Path, set[str]]] = [] - for package in self.repo_root.rglob("package.json"): - if "node_modules" in package.parts: - continue - try: - data = json.loads(package.read_text(encoding="utf-8")) - except (OSError, ValueError): - continue - declared: set[str] = set() - for key in ("dependencies", "devDependencies", "peerDependencies"): - values = data.get(key) - if isinstance(values, dict): - declared.update(values) - roots.append((package.parent, declared)) - return roots - - def _find_parser(self) -> tuple[str | None, str | None, str]: - if not self.node: - return None, None, "Node.js is unavailable" - package_roots = self._package_roots() - for name in self.CANDIDATES: - for package_root, declared in package_roots: - local = package_root / "node_modules" / name - if name not in declared and not local.exists(): - continue - probe = subprocess.run( - [ - self.node, - "-e", - ( - "const p=require.resolve(process.argv[1]," - "{paths:[process.argv[2]]});process.stdout.write(p)" - ), - name, - str(package_root), - ], - cwd=package_root, - text=True, - capture_output=True, - check=False, - ) - if probe.returncode: - continue - resolved = Path(probe.stdout).resolve() - try: - resolved.relative_to(self.repo_root) - except ValueError: - continue - if name == "typescript": - return None, None, ( - "TypeScript is repository-local but its AST " - "normalization is not implemented" - ) - return name, str(resolved), f"repository-local parser: {name}" - return None, None, ( - "no repository-local Acorn, Espree, Babel parser, or supported " - "TypeScript parser is available" - ) - - @property - def supported(self) -> bool: - return bool(self.node and self.parser_name) - - def parse(self, path: Path) -> list[Located]: - if not self.supported: - raise RuntimeError(self.reason) - process = subprocess.run( - [ - self.node or "node", - "-e", - NODE_PARSER, - self.parser_name or "", - self.parser_entry or "", - str(path), - ], - cwd=self.repo_root, - text=True, - capture_output=True, - check=False, - ) - if process.returncode: - raise RuntimeError(process.stderr.strip() or "JavaScript parser failed") - return [Located(**item) for item in json.loads(process.stdout)] - - -def choose_symbol( - symbols: Iterable[Located], locator: str -) -> tuple[Located | None, list[Located]]: - candidates = list(symbols) - exact = [item for item in candidates if item.name == locator] - if not exact and "." not in locator: - exact = [item for item in candidates if item.name.split(".")[-1] == locator] - return (exact[0] if len(exact) == 1 else None), exact - - -def apply_range_result( - validation: Validation, located: Located, cited: tuple[int, int] | None -) -> Validation: - validation.resolved = located - validation.generated_line_range = located.line_range - validation.suggested_locator = located.name - validation.suggested_kind = located.kind - if cited is None: - if validation.current_evidence.get("line_range") not in (None, ""): - validation.result = "range-mismatch" - validation.problem = "line_range is malformed" - else: - validation.result = "valid" - return validation - start, end = cited - if start <= located.start <= end: - # Evidence ranges are excerpts. They must contain the declaration that - # gives the locator its identity, but need not contain the whole body. - validation.result = "valid" - else: - validation.result = "locator-outside-range" - validation.problem = ( - f"locator declaration at L{located.start} is outside cited " - f"L{start}-L{end}; parser-derived span is {located.line_range}" - ) - return validation - - -def suggest_nearby( - symbols: list[Located], locator: str, cited: tuple[int, int] | None -) -> tuple[Located | None, str]: - if cited: - overlapping = [ - item - for item in symbols - if cited[0] <= item.start <= cited[1] - or item.start <= cited[0] <= item.end - ] - if len(overlapping) == 1: - return overlapping[0], "high" - leaf_matches = [ - item - for item in overlapping - if item.name.split(".")[-1] == locator.split(".")[-1] - ] - if len(leaf_matches) == 1: - return leaf_matches[0], "high" - names = [item.name for item in symbols] - close = difflib.get_close_matches(locator, names, n=1, cutoff=0.72) - if close: - return next(item for item in symbols if item.name == close[0]), "medium" - return None, "low" - - -def validate_python( - validation: Validation, target: Path, cited: tuple[int, int] | None -) -> Validation: - try: - index = PythonIndex.from_path(target) - except (OSError, UnicodeError, SyntaxError) as exc: - validation.result = "unsupported" - validation.problem = f"Python AST parsing failed: {exc}" - validation.manual_review_reason = validation.problem - return validation - - locator = validation.locator or "" - if validation.inferred_kind == "python-module": - if locator: - validation.result = "invalid-locator" - validation.problem = "python-module evidence must not name a symbol" - else: - validation.result = "file-level-valid" - validation.suggested_kind = "python-module" - return validation - - if validation.inferred_kind == "python-route": - return validate_route(validation, index.routes, cited) - - located, exact = choose_symbol(index.symbols, locator) - if len(exact) > 1: - validation.result = "ambiguous" - validation.problem = f"locator matches {len(exact)} declarations" - validation.manual_review_reason = validation.problem - validation.confidence = "low" - return validation - if not located: - validation.result = "invalid-locator" - validation.problem = f"Python locator {locator!r} does not exist" - suggestion, confidence = suggest_nearby(index.symbols, locator, cited) - validation.confidence = confidence - if suggestion: - validation.suggested_kind = suggestion.kind - validation.suggested_locator = suggestion.name - validation.generated_line_range = suggestion.line_range - return validation - return apply_range_result(validation, located, cited) - - -def validate_route( - validation: Validation, routes: list[Located], cited: tuple[int, int] | None -) -> Validation: - locator = validation.locator or "" - legacy = LEGACY_ROUTE_RE.fullmatch(locator) - canonical = CANONICAL_ROUTE_RE.fullmatch(locator) - if not legacy and not canonical: - validation.result = "invalid-locator" - validation.problem = ( - "route locator must be '@router.method(\"/path\")' or " - "'METHOD /path -> qualified.function'" - ) - return validation - expected_method = (legacy or canonical).group("method").upper() - expected_path = (legacy or canonical).group("path") - expected_function = canonical.group("function") if canonical else None - method_matches = [ - route - for route in routes - if route.details and route.details["method"] == expected_method - ] - path_matches = [ - route - for route in method_matches - if route.details - and expected_path in {route.details["path"], route.details["full_path"]} - ] - if not method_matches: - validation.result = "invalid-locator" - validation.problem = f"HTTP method {expected_method} is not declared in this file" - return validation - if not path_matches: - validation.result = "invalid-locator" - validation.problem = ( - f"route path {expected_path!r} is not declared for {expected_method}" - ) - return validation - if expected_function: - function_matches = [ - route - for route in path_matches - if route.details and route.details["function"] == expected_function - ] - if not function_matches: - validation.result = "invalid-locator" - validation.problem = ( - f"route exists but containing function is not {expected_function!r}" - ) - return validation - path_matches = function_matches - if len(path_matches) > 1: - validation.result = "ambiguous" - validation.problem = f"route locator matches {len(path_matches)} functions" - validation.manual_review_reason = validation.problem - validation.confidence = "low" - return validation - route = path_matches[0] - details = route.details or {} - if details.get("full_path") is None and expected_path != details.get("path"): - validation.result = "ambiguous" - validation.problem = "final router prefix cannot be resolved statically" - validation.manual_review_reason = validation.problem - validation.resolved = route - validation.generated_line_range = route.line_range - validation.suggested_locator = route.name - validation.suggested_kind = "python-route" - validation.confidence = "medium" - return validation - return apply_range_result(validation, route, cited) - - -def validate_javascript( - validation: Validation, - target: Path, - cited: tuple[int, int] | None, - parser: JavascriptParser, -) -> Validation: - if not parser.supported: - validation.result = "unsupported" - validation.problem = parser.reason - validation.manual_review_reason = parser.reason - validation.confidence = "low" - return validation - try: - symbols = parser.parse(target) - except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as exc: - validation.result = "unsupported" - validation.problem = f"JavaScript parser failed: {exc}" - validation.manual_review_reason = validation.problem - validation.confidence = "low" - return validation - locator = validation.locator or "" - located, exact = choose_symbol(symbols, locator) - if len(exact) > 1: - validation.result = "ambiguous" - validation.problem = f"locator matches {len(exact)} JavaScript declarations" - validation.manual_review_reason = validation.problem - validation.confidence = "low" - return validation - if not located: - validation.result = "invalid-locator" - validation.problem = f"JavaScript locator {locator!r} does not exist" - suggestion, confidence = suggest_nearby(symbols, locator, cited) - validation.confidence = confidence - if suggestion: - validation.suggested_kind = suggestion.kind - validation.suggested_locator = suggestion.name - validation.generated_line_range = suggestion.line_range - return validation - return apply_range_result(validation, located, cited) - - -def shell_functions(path: Path) -> list[Located]: - lines = path.read_text(encoding="utf-8").splitlines() - functions: list[Located] = [] - for index, line in enumerate(lines): - match = SHELL_FUNCTION_RE.match(line) - if not match: - continue - depth = 0 - end = index + 1 - for offset in range(index, len(lines)): - code = lines[offset].split("#", 1)[0] - depth += code.count("{") - code.count("}") - end = offset + 1 - if depth <= 0: - break - functions.append( - Located(match.group("name"), "shell-function", index + 1, end) - ) - return functions - - -def validate_shell( - validation: Validation, target: Path, cited: tuple[int, int] | None -) -> Validation: - try: - check = subprocess.run( - ["bash", "-n", str(target)], text=True, capture_output=True, check=False - ) - except OSError as exc: - validation.result = "unsupported" - validation.problem = f"bash unavailable: {exc}" - return validation - if check.returncode: - validation.result = "unsupported" - validation.problem = f"shell syntax check failed: {check.stderr.strip()}" - return validation - functions = shell_functions(target) - located, exact = choose_symbol(functions, validation.locator or "") - if len(exact) > 1: - validation.result = "ambiguous" - validation.problem = "shell function locator is duplicated" - validation.manual_review_reason = validation.problem - return validation - if not located: - validation.result = "invalid-locator" - validation.problem = f"shell function {validation.locator!r} does not exist" - return validation - return apply_range_result(validation, located, cited) - - -def validate_documentation( - validation: Validation, target: Path, cited: tuple[int, int] | None -) -> Validation: - locator = validation.locator or "" - if not locator: - validation.result = "invalid-locator" - validation.problem = "documentation-section requires a heading locator" - return validation - lines = target.read_text(encoding="utf-8").splitlines() - headings: list[Located] = [] - for number, line in enumerate(lines, 1): - match = MARKDOWN_HEADING_RE.match(line) - if not match: - continue - level = len(match.group("marks")) - end = len(lines) - for later, later_line in enumerate(lines[number:], number + 1): - later_match = MARKDOWN_HEADING_RE.match(later_line) - if later_match and len(later_match.group("marks")) <= level: - end = later - 1 - break - headings.append( - Located( - match.group("title").strip(), - "documentation-section", - number, - end, - ) - ) - located, exact = choose_symbol(headings, locator) - if len(exact) > 1: - validation.result = "ambiguous" - validation.problem = "documentation heading is duplicated" - validation.manual_review_reason = validation.problem - return validation - if not located: - validation.result = "invalid-locator" - validation.problem = f"documentation heading {locator!r} does not exist" - return validation - return apply_range_result(validation, located, cited) - - -def validate_evidence( - repo_root: Path, - feature_id: str, - index: int, - evidence: dict[str, Any], - javascript_parser: JavascriptParser, -) -> Validation: - target = safe_target(repo_root, evidence.get("path")) - kind = infer_kind(evidence, target) - locator = legacy_locator(evidence) - validation = Validation( - feature_id=feature_id, - evidence_index=index, - current_evidence=evidence, - inferred_kind=kind, - locator=locator, - result="ambiguous", - ) - if target is None or not target.is_file(): - validation.result = "invalid-path" - validation.problem = f"evidence path is unsafe or missing: {evidence.get('path')!r}" - validation.confidence = "high" - return validation - - cited = parse_range(evidence.get("line_range")) - explicit_range = evidence.get("line_range") - if explicit_range not in (None, "") and cited is None: - validation.result = "range-mismatch" - validation.problem = f"malformed line_range: {explicit_range!r}" - return validation - if cited is not None: - line_count = len( - target.read_text(encoding="utf-8", errors="replace").splitlines() - ) - if cited[0] > cited[1] or cited[1] > max(line_count, 1): - validation.result = "range-mismatch" - validation.problem = ( - f"line_range L{cited[0]}-L{cited[1]} is outside the " - f"{line_count}-line file" - ) - return validation - - if kind == "file": - if locator: - validation.result = "invalid-locator" - validation.problem = "file evidence must not include a locator/symbol" - validation.suggested_kind = "file" - validation.confidence = "high" - return validation - validation.result = "file-level-valid" - validation.suggested_kind = "file" - line_count = len(target.read_text(encoding="utf-8", errors="replace").splitlines()) - validation.generated_line_range = f"L1-L{max(line_count, 1)}" - return validation - - if not locator and kind not in {"python-module", "file"}: - validation.result = "invalid-locator" - validation.problem = f"{kind} evidence requires a locator" - return validation - - if kind in PYTHON_KINDS: - return validate_python(validation, target, cited) - if kind in JAVASCRIPT_KINDS: - return validate_javascript(validation, target, cited, javascript_parser) - if kind == "shell-function": - return validate_shell(validation, target, cited) - if kind == "documentation-section": - return validate_documentation(validation, target, cited) - if kind == "configuration-key": - validation.result = "ambiguous" - validation.problem = "configuration-key requires a format-aware parser" - validation.manual_review_reason = ( - "This validator does not guess configuration keys from text." - ) - validation.suggested_kind = ( - "file" if evidence.get("kind") is None else "manual-review" - ) - validation.confidence = "low" - return validation - - validation.result = "ambiguous" - validation.problem = f"no automatic resolver for inferred kind {kind!r}" - validation.manual_review_reason = ( - "Use explicit file evidence if the whole file is authoritative, " - "or manual-review with a precise reason." - ) - if evidence.get("kind") is None: - validation.suggested_kind = "file" - validation.suggested_locator = None - line_count = len(target.read_text(encoding="utf-8", errors="replace").splitlines()) - validation.generated_line_range = f"L1-L{max(line_count, 1)}" - validation.confidence = "medium" - return validation - - -def proposed_schema() -> dict[str, Any]: - return { - "required": ["path", "kind", "explanation"], - "optional": ["locator", "line_range"], - "properties": { - "path": "repository-relative file path", - "kind": sorted( - PYTHON_KINDS - | JAVASCRIPT_KINDS - | { - "shell-function", - "configuration-key", - "file", - "documentation-section", - "manual-review", - } - ), - "locator": ( - "parser-resolvable identity; omit for file evidence. " - "Routes use 'METHOD /path -> qualified.function'." - ), - "line_range": ( - "optional generated display metadata in Lx-Ly form; " - "never the primary identity" - ), - "explanation": "why this evidence supports the feature claim", - }, - "examples": [ - { - "path": "src/chat_handler.py", - "kind": "python-method", - "locator": "ChatHandler.preprocess_message", - "explanation": "Preprocesses attachments and URLs for chat requests.", - }, - { - "path": "Dockerfile", - "kind": "file", - "explanation": "Defines the container build.", - }, - ], - } - - -def assess_e2( - features: list[dict[str, Any]], validations: list[Validation] -) -> list[dict[str, Any]]: - """Record the audit's claim-relevance review for the ten frozen E2 records.""" - by_key = { - (item.feature_id, item.current_evidence.get("path"), item.locator): item - for item in validations - } - decisions = { - "CHAT-001": ( - "yes", - "Both cited tests exercise documented chat-stream behavior: emitted " - "stream metrics and non-destructive resend behavior.", - "retain", - ), - "MODEL-006": ( - "yes", - "The Node-backed test exercises the Copilot device-flow start/poll " - "contract and complete verification URI.", - "retain", - ), - "MODEL-007": ( - "yes", - "The Node-backed test exercises the ChatGPT subscription device-flow " - "contract and verification URI.", - "retain", - ), - "RESEARCH-003": ( - "partial", - "The test covers result ranking only, not SearXNG connectivity or " - "multi-provider dispatch in the feature claim.", - "demote", - ), - "DOCUMENT-002": ( - "no", - "The test covers removal of a PDF content marker, not PDF form " - "processing or high-fidelity rendering.", - "demote", - ), - "EMAIL-001": ( - "partial", - "The test covers health probing of account connections, not account " - "setup, SMTP behavior, or inbox polling.", - "demote", - ), - "SECURITY-002": ( - "yes", - "Relevant vault password-handling tests exist, but the cited test " - "function name is fabricated.", - "demote", - ), - "SECURITY-004": ( - "yes", - "Relevant prompt-injection tests exist, but both cited locator names " - "are fabricated umbrella names.", - "demote", - ), - "SECURITY-005": ( - "yes", - "Relevant URL and path confinement tests exist, but all three cited " - "locator names are fabricated umbrella names.", - "demote", - ), - "PLATFORM-009": ( - "manual-review", - "The shell diagnostic is executable evidence, but no automated test " - "function or test-suite evidence is cited and the script uses a " - "file-like fabricated symbol.", - "demote", - ), - } - assessments: list[dict[str, Any]] = [] - for feature in features: - if feature.get("evidence_maturity") != "E2": - continue - test_evidence = [ - evidence - for evidence in feature.get("evidence", []) - if str(evidence.get("path", "")).startswith("tests/") - ] - cited: list[dict[str, Any]] = [] - for evidence in test_evidence: - item = by_key.get( - (feature["id"], evidence.get("path"), legacy_locator(evidence)) - ) - cited.append( - { - "path": evidence.get("path"), - "locator": legacy_locator(evidence), - "exists": bool( - item - and item.result - not in {"invalid-path", "invalid-locator", "unsupported"} - ), - "validation_result": item.result if item else "not-validated", - } - ) - relevance, reason, decision = decisions[feature["id"]] - assessments.append( - { - "feature_id": feature["id"], - "feature_name": feature["name"], - "cited_test_evidence": cited, - "test_function_exists": ( - all(item["exists"] for item in cited) if cited else False - ), - "direct_support": relevance, - "support_reason": reason, - "suggested_decision": decision, - } - ) - return assessments - - -def build_report( - repo_root: Path, - catalog_path: Path, - features: list[dict[str, Any]], - validations: list[Validation], - javascript_parser: JavascriptParser, -) -> dict[str, Any]: - extensions = Counter() - kinds = Counter() - for item in validations: - extensions[extension_for(str(item.current_evidence.get("path", "")))] += 1 - kinds[item.inferred_kind] += 1 - results = Counter(item.result for item in validations) - affected = sorted( - { - item.feature_id - for item in validations - if item.result not in {"valid", "file-level-valid"} - } - ) - invalid_results = { - "invalid-path", - "invalid-locator", - "locator-outside-range", - "range-mismatch", - } - invalid_features = sorted( - { - item.feature_id - for item in validations - if item.result in invalid_results - } - ) - ambiguous = [ - item.to_dict() - for item in validations - if item.result in {"ambiguous", "unsupported"} - ] - return { - "audit": { - "repo_root": str(repo_root), - "catalog": str(catalog_path), - "feature_count": len(features), - "evidence_count": len(validations), - "javascript_parser": { - "supported": javascript_parser.supported, - "parser": javascript_parser.parser_name, - "reason": javascript_parser.reason, - }, - }, - "inventory": { - "total_evidence_items": len(validations), - "by_extension": dict(sorted(extensions.items())), - "by_inferred_evidence_type": dict(sorted(kinds.items())), - "features_with_invalid_or_unresolved_entries": affected, - "features_with_invalid_entries": invalid_features, - "invalid_feature_count": len(invalid_features), - "affected_feature_count": len(affected), - "ambiguous_or_unsupported_entries": ambiguous, - }, - "result_counts": {name: results.get(name, 0) for name in sorted(RESULTS)}, - "summary_counts": { - "valid": results["valid"] + results["file-level-valid"], - "invalid": sum( - results[name] - for name in ( - "invalid-path", - "invalid-locator", - "locator-outside-range", - "range-mismatch", - ) - ), - "ambiguous": results["ambiguous"], - "unsupported": results["unsupported"], - }, - "evidence_model": proposed_schema(), - "e2_assessment": assess_e2(features, validations), - "entries": [item.to_dict() for item in validations], - } - - -def build_repair_queue(validations: list[Validation]) -> dict[str, Any]: - grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) - for item in validations: - if item.result in {"valid", "file-level-valid"}: - continue - grouped[item.feature_id].append( - { - "evidence_index": item.evidence_index, - "current_evidence": item.current_evidence, - "detected_problem": item.problem or item.result, - "result": item.result, - "suggested_kind": item.suggested_kind, - "suggested_locator": item.suggested_locator, - "generated_line_range": item.generated_line_range, - "confidence": item.confidence, - "manual_review_reason": item.manual_review_reason, - } - ) - return { - "feature_count": len(grouped), - "defect_count": sum(len(items) for items in grouped.values()), - "features": [ - {"feature_id": feature_id, "defects": grouped[feature_id]} - for feature_id in sorted(grouped) - ], - } - - -def render_markdown(report: dict[str, Any]) -> str: - inventory = report["inventory"] - summary = report["summary_counts"] - lines = [ - "# Discovery evidence validation report", - "", - "This report is machine-generated. The catalog was not edited.", - "", - "## Summary", - "", - f"- Features: {report['audit']['feature_count']}", - f"- Evidence items: {report['audit']['evidence_count']}", - f"- Valid (including file-level): {summary['valid']}", - f"- Invalid: {summary['invalid']}", - f"- Ambiguous: {summary['ambiguous']}", - f"- Unsupported: {summary['unsupported']}", - f"- Affected features: {inventory['affected_feature_count']}", - "", - "## Evidence by extension", - "", - "| Extension | Items |", - "|---|---:|", - ] - lines.extend( - f"| `{extension}` | {count} |" - for extension, count in inventory["by_extension"].items() - ) - lines.extend( - [ - "", - "## Evidence by inferred kind", - "", - "| Kind | Items |", - "|---|---:|", - ] - ) - lines.extend( - f"| `{kind}` | {count} |" - for kind, count in inventory["by_inferred_evidence_type"].items() - ) - lines.extend( - [ - "", - "## Result categories", - "", - "| Result | Items |", - "|---|---:|", - ] - ) - lines.extend( - f"| `{result}` | {count} |" - for result, count in report["result_counts"].items() - ) - lines.extend( - [ - "", - "## JavaScript parser", - "", - report["audit"]["javascript_parser"]["reason"] + ".", - "", - "## Affected features", - "", - ", ".join( - f"`{feature_id}`" - for feature_id in inventory[ - "features_with_invalid_or_unresolved_entries" - ] - ) - or "None.", - "", - "## E2 relevance review", - "", - "| Feature | Cited test evidence | Exists | Direct support | Decision |", - "|---|---|---:|---|---|", - ] - ) - for item in report["e2_assessment"]: - citations = "<br>".join( - f"`{test['path']}:{test['locator']}`" - for test in item["cited_test_evidence"] - ) or "None" - lines.append( - f"| `{item['feature_id']}` | {citations} | " - f"{'yes' if item['test_function_exists'] else 'no'} | " - f"{item['direct_support']}: {item['support_reason']} | " - f"**{item['suggested_decision']}** |" - ) - lines.extend( - [ - "", - "### Numerical inconsistency resolved", - "", - "The six problematic cited E2 test records are the six nonexistent " - "locator names in SECURITY-002 (one), SECURITY-004 (two), and " - "SECURITY-005 (three). Seven feature demotions were suggested because " - "PLATFORM-009 is an additional feature-level maturity problem: it " - "cites no automated test function. The other three demotions are " - "claim-relevance findings for existing tests, so the figures measure " - "different things and should not be compared as the same denominator.", - "", - "## Proposed evidence schema", - "", - "Required: `path`, `kind`, and `explanation`. `locator` is required " - "for symbol/route/section evidence and omitted for `file`. " - "`line_range` is optional generated metadata.", - "", - "```json", - json.dumps(report["evidence_model"]["examples"], indent=2), - "```", - "", - "## Defects", - "", - "| Feature | Path | Locator | Result | Generated range | Problem |", - "|---|---|---|---|---|---|", - ] - ) - for item in report["entries"]: - if item["result"] in {"valid", "file-level-valid"}: - continue - evidence = item["current_evidence"] - problem = (item["problem"] or "").replace("|", "\\|") - lines.append( - f"| `{item['feature_id']}` | `{evidence.get('path', '')}` | " - f"`{item.get('locator') or ''}` | `{item['result']}` | " - f"`{item.get('generated_line_range') or ''}` | {problem} |" - ) - return "\n".join(lines) + "\n" - - -def write_outputs(output_dir: Path, report: dict[str, Any], queue: dict[str, Any]) -> None: - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "evidence-validation-report.json").write_text( - json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" - ) - (output_dir / "evidence-validation-report.md").write_text( - render_markdown(report), encoding="utf-8" - ) - (output_dir / "evidence-repair-queue.json").write_text( - json.dumps(queue, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" - ) - - -def run(argv: list[str] | None = None) -> int: - args = parse_args(argv) - repo_root = args.repo_root.resolve() - catalog_path = args.catalog.resolve() - features = load_catalog(catalog_path) - javascript_parser = JavascriptParser(repo_root) - validations = [ - validate_evidence( - repo_root, - str(feature.get("id", f"<feature:{feature_index}>")), - evidence_index, - evidence, - javascript_parser, - ) - for feature_index, feature in enumerate(features) - for evidence_index, evidence in enumerate(feature.get("evidence", [])) - if isinstance(evidence, dict) - ] - report = build_report( - repo_root, catalog_path, features, validations, javascript_parser - ) - queue = build_repair_queue(validations) - write_outputs(args.output_dir.resolve(), report, queue) - print(json.dumps(report["summary_counts"], sort_keys=True)) - return 1 if queue["defect_count"] else 0 - - -if __name__ == "__main__": - sys.exit(run()) diff --git a/specs/discovery/README.md b/specs/discovery/README.md new file mode 100644 index 000000000..faec5ed36 --- /dev/null +++ b/specs/discovery/README.md @@ -0,0 +1,30 @@ +# Discovery guide + +Discovery keeps a shared maintainer understanding of how the checked-in Odysseus code works without becoming a permanent audit system. It is a compact companion to the code for cross-cutting systems, not a feature certification or a substitute for normal testing. + +## Working model + +- Code is the ground truth for current behaviour. Trace current paths in source before recording a claim. +- These specs record current code-grounded behaviour and system boundaries only. They do not record accepted intent, design direction, refactor plans, or decision history. +- Keep this package in sync with code changes that alter a documented cross-cutting system, its authority boundary, or its canonical implementation location. +- Do not exhaustively revalidate existing functionality simply because it appears in a map. Investigate when there is a report, normal use visibly fails, a change affects the area, or a high-authority boundary needs review. +- Record a confirmed, actionable problem in the relevant project within the canonical Plane workspace. Keep high-level design, prioritization, ownership, and refactor discussion in Plane. Do not create parallel issue lists here. +- After implementation changes the code, update the relevant map to describe the resulting current state. +- Review safety-sensitive authority first: execution, data access, external tools, credentials, destructive operations, and unattended work. +- Cite stable source locations such as modules, routes, classes, and functions. Avoid fragile line ranges and generated evidence tables. + +## Package + +- [Current system map](system-map.md) describes the code-grounded subsystem boundaries. +- [Safety boundaries](safety-boundaries.md) records where review effort is most valuable. + +The existing [architecture runtime inventory](../architecture-runtime-inventory.md) remains useful structural context. It is explicitly a draft snapshot; re-check its measurements against the code before using them for an implementation decision. + +## How to use this package + +1. Start at the subsystem in the system map and confirm the relevant source. +2. For a defect, create or update the relevant project within the canonical Plane workspace with a reproducible report and ownership. +3. For a design, prioritization, ownership, or refactor question, use a Plane thread. +4. Make and validate the implementation through the normal engineering workflow, then update this package if the code changed a documented system boundary. + +This package intentionally has no generator, validator, maturity scale, feature database, or duplicate work tracker. diff --git a/specs/discovery/safety-boundaries.md b/specs/discovery/safety-boundaries.md new file mode 100644 index 000000000..69b966c1b --- /dev/null +++ b/specs/discovery/safety-boundaries.md @@ -0,0 +1,129 @@ +# Safety boundaries + +This document prioritizes review of broad authority. It records code-grounded safeguards and questions; it does not declare a vulnerability merely because a feature is powerful. No destructive test, external connection, or real credential was used for this map. + +## Shell and subprocess execution + +**Boundary.** Shell routes, agent bash and python tools, local model serving, and detached background jobs. + +**Available authority.** Commands run as the application process user and can create child processes. + +**User-controlled inputs.** Direct shell requests, model-produced tool arguments, scheduled-task prompts, and model-serving configuration. + +**Current safeguards.** Agent dispatch applies owner/admin checks and tool policy; process helpers use timeouts or bounded background-job lifecycle where implemented. + +**Confirmed risks or gaps.** Intentional authority with a confirmed gap: the agent shell starts in its workspace but is not sandboxed to it, and has no egress sandbox. This is documented in source and the threat model; it is not a newly demonstrated bypass. + +**Required verification.** Role-gate and disabled-tool outcomes, direct shell-route behaviour, and timeout, cancellation, and output handling for foreground and detached processes remain unverified. + +## Filesystem access and workspace confinement + +**Boundary.** Agent read, write, patch, listing, glob, and grep tools. + +**Available authority.** Read and modify files within active workspace confinement or fallback allowlisted roots. + +**User-controlled inputs.** Tool paths, patches, file contents, search patterns, and workspace selection passed into the tool dispatcher. + +**Current safeguards.** src/tool_execution.py resolves paths, blocks sensitive subpaths, applies allowlist containment, and tightens paths to the active workspace when one is bound. File tools use those resolvers. + +**Confirmed risks or gaps.** Intentional authority with safeguards. The file-tool policy does not sandbox the shell; treating a workspace as a whole-process containment boundary would be incorrect. + +**Required verification.** Traversal, symlink, sensitive-name, absolute-path, and workspace-switch behaviour remains unverified. + +## Agent-controlled tool dispatch + +**Boundary.** Model output becomes native or parsed tool calls and is dispatched by the agent loop. + +**Available authority.** The authority of every enabled tool, including privileged built-ins and external tools. + +**User-controlled inputs.** Chat content, attached/retrieved content that may influence the model, tool arguments, per-request tool selection, and policy toggles. + +**Current safeguards.** src/tool_security.py blocks protected tools for non-admin users and fails closed for malformed tool names; src/tool_policy.py supports disabled and guide-only policy; prompt-security helpers label untrusted context. + +**Confirmed risks or gaps.** Credible risk requiring verification: aliases, legacy text tools, native function calls, and MCP-qualified names must all reach the same policy outcome. The code has specific alias handling for email/MCP names, which makes this a sensitive compatibility seam. + +**Required verification.** The current policy outcomes for owner role, request mode, disabled state, native versus parsed invocation, qualified aliases, and external-content entry points remain unverified. + +## MCP and external tool servers + +**Boundary.** Configured MCP servers and their tools are exposed to the agent through the MCP manager and routes. + +**Available authority.** Depends on the server: external network access, local process access, messaging, or data mutation may be delegated outside the application. + +**User-controlled inputs.** Server configuration, remote OAuth completion, tool arguments, and model-selected MCP calls. + +**Current safeguards.** MCP routes are registered through routes/mcp_routes.py; MCP-qualified tools are denied to non-admin users by src/tool_security.py. OAuth state and token persistence are handled in src/mcp_oauth.py. + +**Confirmed risks or gaps.** Credible risk requiring verification: an MCP server authority is broader than the application can infer from its tool name. This map does not establish a trust or approval model for server installation and individual tool invocation. + +**Required verification.** Server onboarding, credential storage, server-origin trust, OAuth callback deployment, tool disablement, and invocation audit behaviour remain unverified. + +## Outbound network requests and URL validation + +**Boundary.** Search/content fetch, research, webhooks, skill import, provider endpoints, and other HTTP clients. + +**Available authority.** The application can make outbound requests from its network position. + +**User-controlled inputs.** Search/fetch URLs, imported skill URLs, webhook configuration, and some endpoint settings. + +**Current safeguards.** src/url_security.py validates untrusted public HTTP URLs and fails closed on unsuitable schemes or private addresses. services/search/content.py resolves and rejects non-public hosts, pins resolved addresses for fetches, caps bodies, and limits redirects. + +**Confirmed risks or gaps.** Intentional split: administrator-created model endpoints may target private providers, while untrusted URLs use public-address checks. That distinction is required for self-hosted deployments but needs explicit call-site review. + +**Required verification.** The URL-source classification for outbound clients and the current handling of redirects and DNS changes remain unverified. + +## Secrets, credentials, and vault sessions + +**Boundary.** Application-managed encrypted secrets, API keys, provider credentials, and Bitwarden/Vaultwarden CLI sessions. + +**Available authority.** Credentials unlock remote providers and connected personal services. + +**User-controlled inputs.** Administrative configuration, login/unlock requests, imported settings, and agent vault tool arguments. + +**Current safeguards.** src/secret_storage.py uses a locally stored Fernet key with restrictive permissions for supported database secrets. Vault routes require an administrator, avoid passing master passwords in command arguments, and set restrictive permissions on the vault-session file. + +**Confirmed risks or gaps.** Confirmed current boundary: vault session data is persisted through the vault path, not through src/secret_storage.py. This is an unresolved question about current security semantics, not a confirmed exposure. + +**Required verification.** Current encryption-at-rest, owner scope, rotation, lock/logout, backup/restore, and log/tool-result exposure behaviour remains unverified. + +## Authentication and privileged administration + +**Boundary.** Session authentication, API tokens, privileged routes, and internal tool loopback. + +**Available authority.** Administrative identity can access execution, settings, integrations, data deletion, and secrets. + +**User-controlled inputs.** Login/signup data, session cookies, API tokens, authentication configuration, and requests to privileged routes. + +**Current safeguards.** core/auth.py, core/middleware.py, and route-level checks establish identity and administrator gates. app.py warns when localhost bypass is configured; SECURITY.md documents deployment requirements. + +**Confirmed risks or gaps.** Intentional authority with safeguards. Security depends on deployments keeping authentication enabled and internal services private; this map does not audit reverse-proxy or environment configuration. + +**Required verification.** Setup, anonymous, non-admin, admin, token, and internal-loopback behaviour, including privileged-route gate consistency, remains unverified. + +## Deletion, wipe, backup, and restore + +**Boundary.** Administrative wipe, cleanup, backup import/export, and the backup restore command. + +**Available authority.** Delete or replace user data and credentials. + +**User-controlled inputs.** Administrative HTTP requests, cleanup choices, backup payloads, archive paths, and restore command options. + +**Current safeguards.** Administrative wipe routes use the administrative boundary. Cleanup exposes a preview route before mutation. The documented backup tool requires explicit restore confirmation, stages the old data directory, and validates archive members before extraction. + +**Confirmed risks or gaps.** Intentional destructive authority. Backup archives contain secrets by design, as documented in docs/backup-restore.md; this is an operator confidentiality responsibility, not a code defect established here. + +**Required verification.** Role-gate, confirmation, archive-rejection, staged-recovery, and owner-isolation behaviour remains unverified. No destructive runtime test was performed. + +## Background jobs and unattended task execution + +**Boundary.** Scheduled tasks, background-job monitor, startup tasks, and notification/delivery work that continue without an active browser request. + +**Available authority.** Scheduled agent work can obtain model access and, for eligible owners, shell and file tools; task output can interact with connected services. + +**User-controlled inputs.** Stored task prompt, schedule, model/crew selection, enabled-tool configuration, output target, and prior persisted state. + +**Current safeguards.** src/task_scheduler.py serializes execution, records task runs, associates work with an owner, and applies the agent owner-based tool gate. src/bg_jobs.py keeps bounded state and can terminate overlong subprocess jobs. + +**Confirmed risks or gaps.** Credible risk requiring verification: authority is inherited and exercised later, so changes to roles, task configuration, and disabled tools must be checked at execution time rather than assumed from task creation. + +**Required verification.** Creation, editing, role-change, scheduling, cancellation, restart-recovery, and execution behaviour remains unverified, including whether current policy is re-evaluated before privileged action. diff --git a/specs/discovery/system-map.md b/specs/discovery/system-map.md new file mode 100644 index 000000000..fcaf07e6a --- /dev/null +++ b/specs/discovery/system-map.md @@ -0,0 +1,123 @@ +# Current system map + +This document describes the checked-in code at the same commit as this document. It is a subsystem map, not an exhaustive feature catalog or runtime certification. Each section records only current implementation, canonical locations, confirmed current problems, and unresolved factual questions. + +## Startup and application composition + +**Current implementation.** app.py creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. src/app_initializer.py prepares application state; core/ provides persistence, authentication, middleware, sessions, and platform helpers. + +**Canonical code locations.** app.py; src/app_initializer.py; core/database.py; core/auth.py; core/middleware.py; routes/. + +**Confirmed problems.** None recorded by this mapping. + +**Open questions.** Which component currently owns startup and shutdown for each long-lived service? + +## Frontend shell and browser interaction + +**Current implementation.** static/index.html is served by the root and SPA deep-link routes in app.py; static/app.js, static/style.css, and static/js/ implement the client surface. + +**Canonical code locations.** static/index.html; static/app.js; static/js/; static/style.css; app.py deep-link handlers. + +**Confirmed problems.** The /backgrounds route calls serve_html_with_nonce for static/backgrounds.html, but that file is absent from static/. This is a confirmed broken prototype route, not evidence about the rest of the frontend. + +**Open questions.** Is /backgrounds currently an intentionally supported route or an obsolete prototype? + +## Chat, sessions, and streaming + +**Current implementation.** routes/chat_routes.py and routes/chat_helpers.py coordinate requests, session state, and SSE delivery. src/chat_handler.py, src/chat_processor.py, src/llm_core.py, and src/session_actions.py provide message preparation, provider interaction, and session operations. + +**Canonical code locations.** routes/chat_routes.py; routes/chat_helpers.py; routes/session_routes.py; src/chat_handler.py; src/chat_processor.py; src/llm_core.py; core/session_manager.py. + +**Confirmed problems.** src/agent_loop.py annotates _resolved_tool_event_name with Any but imports no Any and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline. + +**Open questions.** No end-to-end provider or browser streaming run was performed for this map. + +## Agents, tools, and execution + +**Current implementation.** src/agent_loop.py drives multi-round tool use. src/tool_execution.py dispatches calls and binds workspace context. src/agent_tools/ contains individual implementations; src/tool_security.py and src/tool_policy.py apply role and request policies. Long-running command work is represented by src/bg_jobs.py. + +**Canonical code locations.** src/agent_loop.py; src/tool_execution.py; src/agent_tools/; src/tool_security.py; src/tool_policy.py; src/tool_schemas.py; src/bg_jobs.py. + +**Confirmed problems.** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim. + +**Open questions.** Which native, legacy, and MCP-qualified invocation paths reach each policy gate? + +## Models, providers, and local serving + +**Current implementation.** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support. + +**Canonical code locations.** routes/model_routes.py; src/model_discovery.py; src/model_capabilities.py; src/endpoint_resolver.py; src/llm_core.py; routes/cookbook_routes.py; src/cookbook_serve_lifecycle.py; services/hwfit/. + +**Confirmed problems.** None recorded by this mapping. + +**Open questions.** Which endpoint inputs are administrator-created and permitted to use private provider addresses? + +## Search and research + +**Current implementation.** HTTP search routes use services/search/; research is exposed through routes/research/ and implemented in services/research/, src/deep_research.py, and related helpers. src/search/ remains an import-compatibility layer for callers not yet moved to services.search. + +**Canonical code locations.** routes/search_routes.py; services/search/; routes/research/research_routes.py; services/research/; src/deep_research.py; src/search/. + +**Confirmed problems.** The two search package paths are a confirmed compatibility arrangement, not two independent canonical implementations: several src/search modules forward to services.search. The remaining wrappers and compatibility imports can drift. + +**Open questions.** No live provider request was made; provider configuration and network access remain unverified. + +## Documents, retrieval, and personal knowledge + +**Current implementation.** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in src/pdf_runtime.py; form extraction and filling live separately in src/pdf_forms.py and src/pdf_form_doc.py. + +**Canonical code locations.** routes/document_routes.py; src/upload_handler.py; src/document_processor.py; src/document_actions.py; src/personal_docs.py; src/rag_manager.py; src/embeddings.py; src/pdf_runtime.py; src/pdf_forms.py; src/pdf_form_doc.py. + +**Confirmed problems.** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure. + +**Open questions.** Optional PDF dependencies and representative uploaded documents were not exercised. + +## Memory and skills + +**Current implementation.** Memory routes use services/memory/ and vector helpers. Skills are exposed through routes/skills_routes.py, stored and managed in services/memory/skills.py, and may be imported through services/memory/skill_importer.py. + +**Canonical code locations.** routes/memory/memory_routes.py; services/memory/; src/memory.py; src/memory_vector.py; routes/skills_routes.py; services/memory/skills.py; services/memory/skill_importer.py. + +**Confirmed problems.** None recorded by this mapping. + +**Open questions.** Which imported skill content can reach execution-capable paths, and which validation occurs before that point? + +## Personal services: email, calendar, contacts, notes, and tasks + +**Current implementation.** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler. + +**Canonical code locations.** routes/email_routes.py; routes/calendar_routes.py; routes/contacts/contacts_routes.py; routes/note/note_routes.py; routes/task_routes.py; routes/assistant_routes.py; src/caldav_sync.py; src/caldav_writeback.py; src/task_scheduler.py. + +**Confirmed problems.** None recorded by this mapping. + +**Open questions.** External account behaviour, writeback, and delivery require controlled credentials and are not runtime-validated here. + +## Media, speech, and image work + +**Current implementation.** Gallery and image routes coordinate media features. Service modules own speech and media integrations; src/generated_images.py and src/visual_report.py support artifact handling and presentation. + +**Canonical code locations.** routes/gallery/gallery_routes.py; routes/stt_routes.py; routes/tts_routes.py; src/generated_images.py; services/stt/; services/tts/; services/faces/; src/visual_report.py. + +**Confirmed problems.** None recorded by this mapping. + +**Open questions.** Hardware- and provider-dependent media workflows were not exercised. + +## Authentication, secrets, and privileged administration + +**Current implementation.** core/auth.py and core/middleware.py provide identity and request gates. src/secret_storage.py encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: routes/vault_routes.py and src/tools/vault.py invoke the Bitwarden CLI and persist its session data in the application data area. + +**Canonical code locations.** core/auth.py; core/middleware.py; routes/auth_routes.py; routes/api_token_routes.py; src/secret_storage.py; routes/vault_routes.py; src/tools/vault.py; routes/admin_wipe/admin_wipe_routes.py. + +**Confirmed problems.** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised. + +**Open questions.** What are the current confidentiality, ownership, rotation, and backup semantics for vault session data? + +## Persistence, background work, and operations + +**Current implementation.** SQLite models and persistence are centred in core/database.py; managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation. + +**Canonical code locations.** core/database.py; src/runtime_paths.py; src/task_scheduler.py; src/bg_jobs.py; src/bg_monitor.py; routes/backup_routes.py; routes/cleanup/cleanup_routes.py; routes/admin_wipe/admin_wipe_routes.py; scripts/odysseus-backup; docs/backup-restore.md. + +**Confirmed problems.** None recorded by this mapping. + +**Open questions.** What current behaviour applies to background execution, cancellation, retries, and authority inheritance? From 2ac72d596429f92e70ca8a78bf463ba893bcb8b6 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Sun, 26 Jul 2026 14:32:20 +0100 Subject: [PATCH 03/43] docs(discovery): improve map readability --- specs/discovery/README.md | 43 +++++----- specs/discovery/safety-boundaries.md | 123 +++++++++++++++------------ specs/discovery/system-map.md | 116 ++++++++++++++----------- 3 files changed, 157 insertions(+), 125 deletions(-) diff --git a/specs/discovery/README.md b/specs/discovery/README.md index faec5ed36..166ed3656 100644 --- a/specs/discovery/README.md +++ b/specs/discovery/README.md @@ -1,30 +1,33 @@ -# Discovery guide +# Odysseus discovery maps -Discovery keeps a shared maintainer understanding of how the checked-in Odysseus code works without becoming a permanent audit system. It is a compact companion to the code for cross-cutting systems, not a feature certification or a substitute for normal testing. +Compact, code-grounded maps of the cross-cutting systems in the checked-in Odysseus codebase. They are not a feature certification or substitute for normal testing, and do not become a permanent audit system. -## Working model +> [!IMPORTANT] +> Code is the source of truth for current behaviour. These specifications describe only the checked-in system and its current boundaries; keep them synchronized with relevant code changes. -- Code is the ground truth for current behaviour. Trace current paths in source before recording a claim. -- These specs record current code-grounded behaviour and system boundaries only. They do not record accepted intent, design direction, refactor plans, or decision history. -- Keep this package in sync with code changes that alter a documented cross-cutting system, its authority boundary, or its canonical implementation location. -- Do not exhaustively revalidate existing functionality simply because it appears in a map. Investigate when there is a report, normal use visibly fails, a change affects the area, or a high-authority boundary needs review. -- Record a confirmed, actionable problem in the relevant project within the canonical Plane workspace. Keep high-level design, prioritization, ownership, and refactor discussion in Plane. Do not create parallel issue lists here. -- After implementation changes the code, update the relevant map to describe the resulting current state. -- Review safety-sensitive authority first: execution, data access, external tools, credentials, destructive operations, and unattended work. -- Cite stable source locations such as modules, routes, classes, and functions. Avoid fragile line ranges and generated evidence tables. +## Explore the maps -## Package +| Document | Purpose | +|---|---| +| [Current system map](system-map.md) | Explains the current subsystem boundaries, implementation locations, confirmed problems, and factual open questions. | +| [Safety boundaries](safety-boundaries.md) | Maps broad authority, safeguards, confirmed risks or gaps, and unverified behaviour. | -- [Current system map](system-map.md) describes the code-grounded subsystem boundaries. -- [Safety boundaries](safety-boundaries.md) records where review effort is most valuable. +## Working rules -The existing [architecture runtime inventory](../architecture-runtime-inventory.md) remains useful structural context. It is explicitly a draft snapshot; re-check its measurements against the code before using them for an implementation decision. +- **Trace the code first.** Confirm the current path in source before recording a claim. +- **Keep specs current-state only.** Do not record intentions, design direction, refactor plans, or decision history here. +- **Synchronize with code.** Update this package when a code change alters a documented cross-cutting system, authority boundary, or canonical implementation location. +- **Investigate with cause.** Do not exhaustively revalidate existing functionality without a report, visible failure, relevant change, or high-authority review need. +- **Keep planning in Plane.** Record confirmed problems in the relevant project within the canonical Plane workspace; keep high-level design, prioritization, ownership, and refactor discussion there. +- **Review authority carefully.** Give execution, data access, external tools, credentials, destructive operations, and unattended work focused review. +- **Use stable locations.** Cite modules, routes, classes, and functions instead of fragile line ranges or generated evidence tables. -## How to use this package +## Working flow -1. Start at the subsystem in the system map and confirm the relevant source. +1. Start with the relevant map and trace the cited code. 2. For a defect, create or update the relevant project within the canonical Plane workspace with a reproducible report and ownership. -3. For a design, prioritization, ownership, or refactor question, use a Plane thread. -4. Make and validate the implementation through the normal engineering workflow, then update this package if the code changed a documented system boundary. +3. Use a Plane thread for design, prioritization, ownership, or refactor discussion. +4. After implementation changes the code, update the affected map to describe the resulting current state. -This package intentionally has no generator, validator, maturity scale, feature database, or duplicate work tracker. +> [!NOTE] +> This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](../architecture-runtime-inventory.md) remains useful structural context, but is an explicitly draft snapshot. diff --git a/specs/discovery/safety-boundaries.md b/specs/discovery/safety-boundaries.md index 69b966c1b..ee8b67d98 100644 --- a/specs/discovery/safety-boundaries.md +++ b/specs/discovery/safety-boundaries.md @@ -1,129 +1,142 @@ # Safety boundaries -This document prioritizes review of broad authority. It records code-grounded safeguards and questions; it does not declare a vulnerability merely because a feature is powerful. No destructive test, external connection, or real credential was used for this map. +> [!IMPORTANT] +> Broad authority does not by itself establish a vulnerability. This map records code-grounded safeguards, confirmed risks or gaps, and unverified behaviour. No destructive test, external connection, or real credential was used for this map. + +## Navigate the boundaries + +- [Shell and subprocess execution](#shell-and-subprocess-execution) +- [Filesystem access and workspace confinement](#filesystem-access-and-workspace-confinement) +- [Agent-controlled tool dispatch](#agent-controlled-tool-dispatch) +- [MCP and external tool servers](#mcp-and-external-tool-servers) +- [Outbound network requests and URL validation](#outbound-network-requests-and-url-validation) +- [Secrets, credentials, and vault sessions](#secrets-credentials-and-vault-sessions) +- [Authentication and privileged administration](#authentication-and-privileged-administration) +- [Deletion, wipe, backup, and restore](#deletion-wipe-backup-and-restore) +- [Background jobs and unattended task execution](#background-jobs-and-unattended-task-execution) ## Shell and subprocess execution -**Boundary.** Shell routes, agent bash and python tools, local model serving, and detached background jobs. +- **Boundary:** Shell routes, agent `bash` and `python` tools, local model serving, and detached background jobs. -**Available authority.** Commands run as the application process user and can create child processes. +- **Available authority:** Commands run as the application process user and can create child processes. -**User-controlled inputs.** Direct shell requests, model-produced tool arguments, scheduled-task prompts, and model-serving configuration. +- **User-controlled inputs:** Direct shell requests, model-produced tool arguments, scheduled-task prompts, and model-serving configuration. -**Current safeguards.** Agent dispatch applies owner/admin checks and tool policy; process helpers use timeouts or bounded background-job lifecycle where implemented. +- **Current safeguards:** Agent dispatch applies owner/admin checks and tool policy; process helpers use timeouts or bounded background-job lifecycle where implemented. -**Confirmed risks or gaps.** Intentional authority with a confirmed gap: the agent shell starts in its workspace but is not sandboxed to it, and has no egress sandbox. This is documented in source and the threat model; it is not a newly demonstrated bypass. +- **Confirmed risks or gaps:** Intentional authority with a confirmed gap: the agent shell starts in its workspace but is not sandboxed to it, and has no egress sandbox. This is documented in source and the threat model; it is not a newly demonstrated bypass. -**Required verification.** Role-gate and disabled-tool outcomes, direct shell-route behaviour, and timeout, cancellation, and output handling for foreground and detached processes remain unverified. +- **Unverified behaviour:** Role-gate and disabled-tool outcomes, direct shell-route behaviour, and timeout, cancellation, and output handling for foreground and detached processes remain unverified. ## Filesystem access and workspace confinement -**Boundary.** Agent read, write, patch, listing, glob, and grep tools. +- **Boundary:** Agent read, write, patch, listing, glob, and grep tools. -**Available authority.** Read and modify files within active workspace confinement or fallback allowlisted roots. +- **Available authority:** Read and modify files within active workspace confinement or fallback allowlisted roots. -**User-controlled inputs.** Tool paths, patches, file contents, search patterns, and workspace selection passed into the tool dispatcher. +- **User-controlled inputs:** Tool paths, patches, file contents, search patterns, and workspace selection passed into the tool dispatcher. -**Current safeguards.** src/tool_execution.py resolves paths, blocks sensitive subpaths, applies allowlist containment, and tightens paths to the active workspace when one is bound. File tools use those resolvers. +- **Current safeguards:** [`src/tool_execution.py`](../../src/tool_execution.py) resolves paths, blocks sensitive subpaths, applies allowlist containment, and tightens paths to the active workspace when one is bound. File tools use those resolvers. -**Confirmed risks or gaps.** Intentional authority with safeguards. The file-tool policy does not sandbox the shell; treating a workspace as a whole-process containment boundary would be incorrect. +- **Confirmed risks or gaps:** Intentional authority with safeguards. The file-tool policy does not sandbox the shell; treating a workspace as a whole-process containment boundary would be incorrect. -**Required verification.** Traversal, symlink, sensitive-name, absolute-path, and workspace-switch behaviour remains unverified. +- **Unverified behaviour:** Traversal, symlink, sensitive-name, absolute-path, and workspace-switch behaviour remains unverified. ## Agent-controlled tool dispatch -**Boundary.** Model output becomes native or parsed tool calls and is dispatched by the agent loop. +- **Boundary:** Model output becomes native or parsed tool calls and is dispatched by the agent loop. -**Available authority.** The authority of every enabled tool, including privileged built-ins and external tools. +- **Available authority:** The authority of every enabled tool, including privileged built-ins and external tools. -**User-controlled inputs.** Chat content, attached/retrieved content that may influence the model, tool arguments, per-request tool selection, and policy toggles. +- **User-controlled inputs:** Chat content, attached/retrieved content that may influence the model, tool arguments, per-request tool selection, and policy toggles. -**Current safeguards.** src/tool_security.py blocks protected tools for non-admin users and fails closed for malformed tool names; src/tool_policy.py supports disabled and guide-only policy; prompt-security helpers label untrusted context. +- **Current safeguards:** [`src/tool_security.py`](../../src/tool_security.py) blocks protected tools for non-admin users and fails closed for malformed tool names; [`src/tool_policy.py`](../../src/tool_policy.py) supports disabled and guide-only policy; prompt-security helpers label untrusted context. -**Confirmed risks or gaps.** Credible risk requiring verification: aliases, legacy text tools, native function calls, and MCP-qualified names must all reach the same policy outcome. The code has specific alias handling for email/MCP names, which makes this a sensitive compatibility seam. +- **Confirmed risks or gaps:** Credible risk requiring verification: aliases, legacy text tools, native function calls, and MCP-qualified names must all reach the same policy outcome. The code has specific alias handling for email/MCP names, which makes this a sensitive compatibility seam. -**Required verification.** The current policy outcomes for owner role, request mode, disabled state, native versus parsed invocation, qualified aliases, and external-content entry points remain unverified. +- **Unverified behaviour:** The current policy outcomes for owner role, request mode, disabled state, native versus parsed invocation, qualified aliases, and external-content entry points remain unverified. ## MCP and external tool servers -**Boundary.** Configured MCP servers and their tools are exposed to the agent through the MCP manager and routes. +- **Boundary:** Configured MCP servers and their tools are exposed to the agent through the MCP manager and routes. -**Available authority.** Depends on the server: external network access, local process access, messaging, or data mutation may be delegated outside the application. +- **Available authority:** Depends on the server: external network access, local process access, messaging, or data mutation may be delegated outside the application. -**User-controlled inputs.** Server configuration, remote OAuth completion, tool arguments, and model-selected MCP calls. +- **User-controlled inputs:** Server configuration, remote OAuth completion, tool arguments, and model-selected MCP calls. -**Current safeguards.** MCP routes are registered through routes/mcp_routes.py; MCP-qualified tools are denied to non-admin users by src/tool_security.py. OAuth state and token persistence are handled in src/mcp_oauth.py. +- **Current safeguards:** MCP routes are registered through [`routes/mcp_routes.py`](../../routes/mcp_routes.py); MCP-qualified tools are denied to non-admin users by [`src/tool_security.py`](../../src/tool_security.py). OAuth state and token persistence are handled in [`src/mcp_oauth.py`](../../src/mcp_oauth.py). -**Confirmed risks or gaps.** Credible risk requiring verification: an MCP server authority is broader than the application can infer from its tool name. This map does not establish a trust or approval model for server installation and individual tool invocation. +- **Confirmed risks or gaps:** Credible risk requiring verification: an MCP server authority is broader than the application can infer from its tool name. This map does not establish a trust or approval model for server installation and individual tool invocation. -**Required verification.** Server onboarding, credential storage, server-origin trust, OAuth callback deployment, tool disablement, and invocation audit behaviour remain unverified. +- **Unverified behaviour:** Server onboarding, credential storage, server-origin trust, OAuth callback deployment, tool disablement, and invocation audit behaviour remain unverified. ## Outbound network requests and URL validation -**Boundary.** Search/content fetch, research, webhooks, skill import, provider endpoints, and other HTTP clients. +- **Boundary:** Search/content fetch, research, webhooks, skill import, provider endpoints, and other HTTP clients. -**Available authority.** The application can make outbound requests from its network position. +- **Available authority:** The application can make outbound requests from its network position. -**User-controlled inputs.** Search/fetch URLs, imported skill URLs, webhook configuration, and some endpoint settings. +- **User-controlled inputs:** Search/fetch URLs, imported skill URLs, webhook configuration, and some endpoint settings. -**Current safeguards.** src/url_security.py validates untrusted public HTTP URLs and fails closed on unsuitable schemes or private addresses. services/search/content.py resolves and rejects non-public hosts, pins resolved addresses for fetches, caps bodies, and limits redirects. +- **Current safeguards:** [`src/url_security.py`](../../src/url_security.py) validates untrusted public HTTP URLs and fails closed on unsuitable schemes or private addresses. [`services/search/content.py`](../../services/search/content.py) resolves and rejects non-public hosts, pins resolved addresses for fetches, caps bodies, and limits redirects. -**Confirmed risks or gaps.** Intentional split: administrator-created model endpoints may target private providers, while untrusted URLs use public-address checks. That distinction is required for self-hosted deployments but needs explicit call-site review. +- **Confirmed risks or gaps:** Intentional split: administrator-created model endpoints may target private providers, while untrusted URLs use public-address checks. That distinction is required for self-hosted deployments but needs explicit call-site review. -**Required verification.** The URL-source classification for outbound clients and the current handling of redirects and DNS changes remain unverified. +- **Unverified behaviour:** The URL-source classification for outbound clients and the current handling of redirects and DNS changes remain unverified. ## Secrets, credentials, and vault sessions -**Boundary.** Application-managed encrypted secrets, API keys, provider credentials, and Bitwarden/Vaultwarden CLI sessions. +- **Boundary:** Application-managed encrypted secrets, API keys, provider credentials, and Bitwarden/Vaultwarden CLI sessions. -**Available authority.** Credentials unlock remote providers and connected personal services. +- **Available authority:** Credentials unlock remote providers and connected personal services. -**User-controlled inputs.** Administrative configuration, login/unlock requests, imported settings, and agent vault tool arguments. +- **User-controlled inputs:** Administrative configuration, login/unlock requests, imported settings, and agent vault tool arguments. -**Current safeguards.** src/secret_storage.py uses a locally stored Fernet key with restrictive permissions for supported database secrets. Vault routes require an administrator, avoid passing master passwords in command arguments, and set restrictive permissions on the vault-session file. +- **Current safeguards:** [`src/secret_storage.py`](../../src/secret_storage.py) uses a locally stored Fernet key with restrictive permissions for supported database secrets. Vault routes require an administrator, avoid passing master passwords in command arguments, and set restrictive permissions on the vault-session file. -**Confirmed risks or gaps.** Confirmed current boundary: vault session data is persisted through the vault path, not through src/secret_storage.py. This is an unresolved question about current security semantics, not a confirmed exposure. +- **Confirmed risks or gaps:** Confirmed current boundary: vault session data is persisted through the vault path, not through [`src/secret_storage.py`](../../src/secret_storage.py). This is an unresolved question about current security semantics, not a confirmed exposure. -**Required verification.** Current encryption-at-rest, owner scope, rotation, lock/logout, backup/restore, and log/tool-result exposure behaviour remains unverified. +- **Unverified behaviour:** Current encryption-at-rest, owner scope, rotation, lock/logout, backup/restore, and log/tool-result exposure behaviour remains unverified. ## Authentication and privileged administration -**Boundary.** Session authentication, API tokens, privileged routes, and internal tool loopback. +- **Boundary:** Session authentication, API tokens, privileged routes, and internal tool loopback. -**Available authority.** Administrative identity can access execution, settings, integrations, data deletion, and secrets. +- **Available authority:** Administrative identity can access execution, settings, integrations, data deletion, and secrets. -**User-controlled inputs.** Login/signup data, session cookies, API tokens, authentication configuration, and requests to privileged routes. +- **User-controlled inputs:** Login/signup data, session cookies, API tokens, authentication configuration, and requests to privileged routes. -**Current safeguards.** core/auth.py, core/middleware.py, and route-level checks establish identity and administrator gates. app.py warns when localhost bypass is configured; SECURITY.md documents deployment requirements. +- **Current safeguards:** [`core/auth.py`](../../core/auth.py), [`core/middleware.py`](../../core/middleware.py), and route-level checks establish identity and administrator gates. [`app.py`](../../app.py) warns when localhost bypass is configured; [`SECURITY.md`](../../SECURITY.md) documents deployment requirements. -**Confirmed risks or gaps.** Intentional authority with safeguards. Security depends on deployments keeping authentication enabled and internal services private; this map does not audit reverse-proxy or environment configuration. +- **Confirmed risks or gaps:** Intentional authority with safeguards. Security depends on deployments keeping authentication enabled and internal services private; this map does not audit reverse-proxy or environment configuration. -**Required verification.** Setup, anonymous, non-admin, admin, token, and internal-loopback behaviour, including privileged-route gate consistency, remains unverified. +- **Unverified behaviour:** Setup, anonymous, non-admin, admin, token, and internal-loopback behaviour, including privileged-route gate consistency, remains unverified. ## Deletion, wipe, backup, and restore -**Boundary.** Administrative wipe, cleanup, backup import/export, and the backup restore command. +- **Boundary:** Administrative wipe, cleanup, backup import/export, and the backup restore command. -**Available authority.** Delete or replace user data and credentials. +- **Available authority:** Delete or replace user data and credentials. -**User-controlled inputs.** Administrative HTTP requests, cleanup choices, backup payloads, archive paths, and restore command options. +- **User-controlled inputs:** Administrative HTTP requests, cleanup choices, backup payloads, archive paths, and restore command options. -**Current safeguards.** Administrative wipe routes use the administrative boundary. Cleanup exposes a preview route before mutation. The documented backup tool requires explicit restore confirmation, stages the old data directory, and validates archive members before extraction. +- **Current safeguards:** Administrative wipe routes use the administrative boundary. Cleanup exposes a preview route before mutation. The documented backup tool requires explicit restore confirmation, stages the old data directory, and validates archive members before extraction. -**Confirmed risks or gaps.** Intentional destructive authority. Backup archives contain secrets by design, as documented in docs/backup-restore.md; this is an operator confidentiality responsibility, not a code defect established here. +- **Confirmed risks or gaps:** Intentional destructive authority. Backup archives contain secrets by design, as documented in [`docs/backup-restore.md`](../../docs/backup-restore.md); this is an operator confidentiality responsibility, not a code defect established here. -**Required verification.** Role-gate, confirmation, archive-rejection, staged-recovery, and owner-isolation behaviour remains unverified. No destructive runtime test was performed. +- **Unverified behaviour:** Role-gate, confirmation, archive-rejection, staged-recovery, and owner-isolation behaviour remains unverified. No destructive runtime test was performed. ## Background jobs and unattended task execution -**Boundary.** Scheduled tasks, background-job monitor, startup tasks, and notification/delivery work that continue without an active browser request. +- **Boundary:** Scheduled tasks, background-job monitor, startup tasks, and notification/delivery work that continue without an active browser request. -**Available authority.** Scheduled agent work can obtain model access and, for eligible owners, shell and file tools; task output can interact with connected services. +- **Available authority:** Scheduled agent work can obtain model access and, for eligible owners, shell and file tools; task output can interact with connected services. -**User-controlled inputs.** Stored task prompt, schedule, model/crew selection, enabled-tool configuration, output target, and prior persisted state. +- **User-controlled inputs:** Stored task prompt, schedule, model/crew selection, enabled-tool configuration, output target, and prior persisted state. -**Current safeguards.** src/task_scheduler.py serializes execution, records task runs, associates work with an owner, and applies the agent owner-based tool gate. src/bg_jobs.py keeps bounded state and can terminate overlong subprocess jobs. +- **Current safeguards:** [`src/task_scheduler.py`](../../src/task_scheduler.py) serializes execution, records task runs, associates work with an owner, and applies the agent owner-based tool gate. [`src/bg_jobs.py`](../../src/bg_jobs.py) keeps bounded state and can terminate overlong subprocess jobs. -**Confirmed risks or gaps.** Credible risk requiring verification: authority is inherited and exercised later, so changes to roles, task configuration, and disabled tools must be checked at execution time rather than assumed from task creation. +- **Confirmed risks or gaps:** Credible risk requiring verification: authority is inherited and exercised later, so changes to roles, task configuration, and disabled tools must be checked at execution time rather than assumed from task creation. -**Required verification.** Creation, editing, role-change, scheduling, cancellation, restart-recovery, and execution behaviour remains unverified, including whether current policy is re-evaluated before privileged action. +- **Unverified behaviour:** Creation, editing, role-change, scheduling, cancellation, restart-recovery, and execution behaviour remains unverified, including whether current policy is re-evaluated before privileged action. diff --git a/specs/discovery/system-map.md b/specs/discovery/system-map.md index fcaf07e6a..2f9f50eaa 100644 --- a/specs/discovery/system-map.md +++ b/specs/discovery/system-map.md @@ -1,123 +1,139 @@ # Current system map -This document describes the checked-in code at the same commit as this document. It is a subsystem map, not an exhaustive feature catalog or runtime certification. Each section records only current implementation, canonical locations, confirmed current problems, and unresolved factual questions. +> [!NOTE] +> This map describes the checked-in code at the same commit as this document. It is a subsystem map, not an exhaustive feature catalog or runtime certification. Each section records only current implementation, canonical locations, confirmed current problems, and unresolved factual questions. + +## Navigate the system + +- [Startup and application composition](#startup-and-application-composition) +- [Frontend shell and browser interaction](#frontend-shell-and-browser-interaction) +- [Chat, sessions, and streaming](#chat-sessions-and-streaming) +- [Agents, tools, and execution](#agents-tools-and-execution) +- [Models, providers, and local serving](#models-providers-and-local-serving) +- [Search and research](#search-and-research) +- [Documents, retrieval, and personal knowledge](#documents-retrieval-and-personal-knowledge) +- [Memory and skills](#memory-and-skills) +- [Email, calendar, contacts, notes, and tasks](#email-calendar-contacts-notes-and-tasks) +- [Media, speech, and image work](#media-speech-and-image-work) +- [Authentication, secrets, and privileged administration](#authentication-secrets-and-privileged-administration) +- [Persistence, background work, and operations](#persistence-background-work-and-operations) ## Startup and application composition -**Current implementation.** app.py creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. src/app_initializer.py prepares application state; core/ provides persistence, authentication, middleware, sessions, and platform helpers. +- **How it works:** [`app.py`](../../app.py) creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. [`src/app_initializer.py`](../../src/app_initializer.py) prepares application state; [`core/`](../../core/) provides persistence, authentication, middleware, sessions, and platform helpers. -**Canonical code locations.** app.py; src/app_initializer.py; core/database.py; core/auth.py; core/middleware.py; routes/. +- **Canonical code:** [`app.py`](../../app.py); [`src/app_initializer.py`](../../src/app_initializer.py); [`core/database.py`](../../core/database.py); [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/`](../../routes/). -**Confirmed problems.** None recorded by this mapping. +- **Known problems:** None recorded by this mapping. -**Open questions.** Which component currently owns startup and shutdown for each long-lived service? +- **Open question:** Which component currently owns startup and shutdown for each long-lived service? ## Frontend shell and browser interaction -**Current implementation.** static/index.html is served by the root and SPA deep-link routes in app.py; static/app.js, static/style.css, and static/js/ implement the client surface. +- **How it works:** [`static/index.html`](../../static/index.html) is served by the root and SPA deep-link routes in [`app.py`](../../app.py); [`static/app.js`](../../static/app.js), [`static/style.css`](../../static/style.css), and [`static/js/`](../../static/js/) implement the client surface. -**Canonical code locations.** static/index.html; static/app.js; static/js/; static/style.css; app.py deep-link handlers. +- **Canonical code:** [`static/index.html`](../../static/index.html); [`static/app.js`](../../static/app.js); [`static/js/`](../../static/js/); [`static/style.css`](../../static/style.css); [`app.py`](../../app.py) deep-link handlers. -**Confirmed problems.** The /backgrounds route calls serve_html_with_nonce for static/backgrounds.html, but that file is absent from static/. This is a confirmed broken prototype route, not evidence about the rest of the frontend. +- **Known problems:** The `/backgrounds` route in [`app.py`](../../app.py) calls `serve_html_with_nonce` for `static/backgrounds.html`, but that file is absent from [`static/`](../../static/). This is a confirmed broken prototype route, not evidence about the rest of the frontend. -**Open questions.** Is /backgrounds currently an intentionally supported route or an obsolete prototype? +- **Open question:** Is `/backgrounds` currently an intentionally supported route or an obsolete prototype? ## Chat, sessions, and streaming -**Current implementation.** routes/chat_routes.py and routes/chat_helpers.py coordinate requests, session state, and SSE delivery. src/chat_handler.py, src/chat_processor.py, src/llm_core.py, and src/session_actions.py provide message preparation, provider interaction, and session operations. +- **How it works:** [`routes/chat_routes.py`](../../routes/chat_routes.py) and [`routes/chat_helpers.py`](../../routes/chat_helpers.py) coordinate requests, session state, and SSE delivery. [`src/chat_handler.py`](../../src/chat_handler.py), [`src/chat_processor.py`](../../src/chat_processor.py), [`src/llm_core.py`](../../src/llm_core.py), and [`src/session_actions.py`](../../src/session_actions.py) provide message preparation, provider interaction, and session operations. -**Canonical code locations.** routes/chat_routes.py; routes/chat_helpers.py; routes/session_routes.py; src/chat_handler.py; src/chat_processor.py; src/llm_core.py; core/session_manager.py. +- **Canonical code:** [`routes/chat_routes.py`](../../routes/chat_routes.py); [`routes/chat_helpers.py`](../../routes/chat_helpers.py); [`routes/session_routes.py`](../../routes/session_routes.py); [`src/chat_handler.py`](../../src/chat_handler.py); [`src/chat_processor.py`](../../src/chat_processor.py); [`src/llm_core.py`](../../src/llm_core.py); [`core/session_manager.py`](../../core/session_manager.py). -**Confirmed problems.** src/agent_loop.py annotates _resolved_tool_event_name with Any but imports no Any and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline. +- **Known problems:** [`src/agent_loop.py`](../../src/agent_loop.py) annotates `_resolved_tool_event_name` with `Any` but imports no `Any` and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline. -**Open questions.** No end-to-end provider or browser streaming run was performed for this map. +- **Open question:** No end-to-end provider or browser streaming run was performed for this map. ## Agents, tools, and execution -**Current implementation.** src/agent_loop.py drives multi-round tool use. src/tool_execution.py dispatches calls and binds workspace context. src/agent_tools/ contains individual implementations; src/tool_security.py and src/tool_policy.py apply role and request policies. Long-running command work is represented by src/bg_jobs.py. +- **How it works:** [`src/agent_loop.py`](../../src/agent_loop.py) drives multi-round tool use. [`src/tool_execution.py`](../../src/tool_execution.py) dispatches calls and binds workspace context. [`src/agent_tools/`](../../src/agent_tools/) contains individual implementations; [`src/tool_security.py`](../../src/tool_security.py) and [`src/tool_policy.py`](../../src/tool_policy.py) apply role and request policies. Long-running command work is represented by [`src/bg_jobs.py`](../../src/bg_jobs.py). -**Canonical code locations.** src/agent_loop.py; src/tool_execution.py; src/agent_tools/; src/tool_security.py; src/tool_policy.py; src/tool_schemas.py; src/bg_jobs.py. +- **Canonical code:** [`src/agent_loop.py`](../../src/agent_loop.py); [`src/tool_execution.py`](../../src/tool_execution.py); [`src/agent_tools/`](../../src/agent_tools/); [`src/tool_security.py`](../../src/tool_security.py); [`src/tool_policy.py`](../../src/tool_policy.py); [`src/tool_schemas.py`](../../src/tool_schemas.py); [`src/bg_jobs.py`](../../src/bg_jobs.py). -**Confirmed problems.** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim. +- **Known problems:** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim. -**Open questions.** Which native, legacy, and MCP-qualified invocation paths reach each policy gate? +- **Open question:** Which native, legacy, and MCP-qualified invocation paths reach each policy gate? ## Models, providers, and local serving -**Current implementation.** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support. +- **How it works:** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support. -**Canonical code locations.** routes/model_routes.py; src/model_discovery.py; src/model_capabilities.py; src/endpoint_resolver.py; src/llm_core.py; routes/cookbook_routes.py; src/cookbook_serve_lifecycle.py; services/hwfit/. +- **Canonical code:** [`routes/model_routes.py`](../../routes/model_routes.py); [`src/model_discovery.py`](../../src/model_discovery.py); [`src/model_capabilities.py`](../../src/model_capabilities.py); [`src/endpoint_resolver.py`](../../src/endpoint_resolver.py); [`src/llm_core.py`](../../src/llm_core.py); [`routes/cookbook_routes.py`](../../routes/cookbook_routes.py); [`src/cookbook_serve_lifecycle.py`](../../src/cookbook_serve_lifecycle.py); [`services/hwfit/`](../../services/hwfit/). -**Confirmed problems.** None recorded by this mapping. +- **Known problems:** None recorded by this mapping. -**Open questions.** Which endpoint inputs are administrator-created and permitted to use private provider addresses? +- **Open question:** Which endpoint inputs are administrator-created and permitted to use private provider addresses? ## Search and research -**Current implementation.** HTTP search routes use services/search/; research is exposed through routes/research/ and implemented in services/research/, src/deep_research.py, and related helpers. src/search/ remains an import-compatibility layer for callers not yet moved to services.search. +- **How it works:** HTTP search routes use [`services/search/`](../../services/search/); research is exposed through [`routes/research/`](../../routes/research/) and implemented in [`services/research/`](../../services/research/), [`src/deep_research.py`](../../src/deep_research.py), and related helpers. [`src/search/`](../../src/search/) remains an import-compatibility layer for callers not yet moved to `services.search`. -**Canonical code locations.** routes/search_routes.py; services/search/; routes/research/research_routes.py; services/research/; src/deep_research.py; src/search/. +- **Canonical code:** [`routes/search_routes.py`](../../routes/search_routes.py); [`services/search/`](../../services/search/); [`routes/research/research_routes.py`](../../routes/research/research_routes.py); [`services/research/`](../../services/research/); [`src/deep_research.py`](../../src/deep_research.py); [`src/search/`](../../src/search/). -**Confirmed problems.** The two search package paths are a confirmed compatibility arrangement, not two independent canonical implementations: several src/search modules forward to services.search. The remaining wrappers and compatibility imports can drift. +- **Known problems:** The two search package paths are a confirmed compatibility arrangement, not two independent canonical implementations: several [`src/search/`](../../src/search/) modules forward to [`services/search/`](../../services/search/). The remaining wrappers and compatibility imports can drift. -**Open questions.** No live provider request was made; provider configuration and network access remain unverified. +- **Open question:** No live provider request was made; provider configuration and network access remain unverified. ## Documents, retrieval, and personal knowledge -**Current implementation.** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in src/pdf_runtime.py; form extraction and filling live separately in src/pdf_forms.py and src/pdf_form_doc.py. +- **How it works:** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in [`src/pdf_runtime.py`](../../src/pdf_runtime.py); form extraction and filling live separately in [`src/pdf_forms.py`](../../src/pdf_forms.py) and [`src/pdf_form_doc.py`](../../src/pdf_form_doc.py). -**Canonical code locations.** routes/document_routes.py; src/upload_handler.py; src/document_processor.py; src/document_actions.py; src/personal_docs.py; src/rag_manager.py; src/embeddings.py; src/pdf_runtime.py; src/pdf_forms.py; src/pdf_form_doc.py. +- **Canonical code:** [`routes/document_routes.py`](../../routes/document_routes.py); [`src/upload_handler.py`](../../src/upload_handler.py); [`src/document_processor.py`](../../src/document_processor.py); [`src/document_actions.py`](../../src/document_actions.py); [`src/personal_docs.py`](../../src/personal_docs.py); [`src/rag_manager.py`](../../src/rag_manager.py); [`src/embeddings.py`](../../src/embeddings.py); [`src/pdf_runtime.py`](../../src/pdf_runtime.py); [`src/pdf_forms.py`](../../src/pdf_forms.py); [`src/pdf_form_doc.py`](../../src/pdf_form_doc.py). -**Confirmed problems.** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure. +- **Known problems:** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure. -**Open questions.** Optional PDF dependencies and representative uploaded documents were not exercised. +- **Open question:** Optional PDF dependencies and representative uploaded documents were not exercised. ## Memory and skills -**Current implementation.** Memory routes use services/memory/ and vector helpers. Skills are exposed through routes/skills_routes.py, stored and managed in services/memory/skills.py, and may be imported through services/memory/skill_importer.py. +- **How it works:** Memory routes use [`services/memory/`](../../services/memory/) and vector helpers. Skills are exposed through [`routes/skills_routes.py`](../../routes/skills_routes.py), stored and managed in [`services/memory/skills.py`](../../services/memory/skills.py), and may be imported through [`services/memory/skill_importer.py`](../../services/memory/skill_importer.py). -**Canonical code locations.** routes/memory/memory_routes.py; services/memory/; src/memory.py; src/memory_vector.py; routes/skills_routes.py; services/memory/skills.py; services/memory/skill_importer.py. +- **Canonical code:** [`routes/memory/memory_routes.py`](../../routes/memory/memory_routes.py); [`services/memory/`](../../services/memory/); [`src/memory.py`](../../src/memory.py); [`src/memory_vector.py`](../../src/memory_vector.py); [`routes/skills_routes.py`](../../routes/skills_routes.py); [`services/memory/skills.py`](../../services/memory/skills.py); [`services/memory/skill_importer.py`](../../services/memory/skill_importer.py). -**Confirmed problems.** None recorded by this mapping. +- **Known problems:** None recorded by this mapping. -**Open questions.** Which imported skill content can reach execution-capable paths, and which validation occurs before that point? +- **Open question:** Which imported skill content can reach execution-capable paths, and which validation occurs before that point? -## Personal services: email, calendar, contacts, notes, and tasks +## Email, calendar, contacts, notes, and tasks -**Current implementation.** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler. +- **How it works:** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler. -**Canonical code locations.** routes/email_routes.py; routes/calendar_routes.py; routes/contacts/contacts_routes.py; routes/note/note_routes.py; routes/task_routes.py; routes/assistant_routes.py; src/caldav_sync.py; src/caldav_writeback.py; src/task_scheduler.py. +- **Canonical code:** [`routes/email_routes.py`](../../routes/email_routes.py); [`routes/calendar_routes.py`](../../routes/calendar_routes.py); [`routes/contacts/contacts_routes.py`](../../routes/contacts/contacts_routes.py); [`routes/note/note_routes.py`](../../routes/note/note_routes.py); [`routes/task_routes.py`](../../routes/task_routes.py); [`routes/assistant_routes.py`](../../routes/assistant_routes.py); [`src/caldav_sync.py`](../../src/caldav_sync.py); [`src/caldav_writeback.py`](../../src/caldav_writeback.py); [`src/task_scheduler.py`](../../src/task_scheduler.py). -**Confirmed problems.** None recorded by this mapping. +- **Known problems:** None recorded by this mapping. -**Open questions.** External account behaviour, writeback, and delivery require controlled credentials and are not runtime-validated here. +- **Open question:** External account behaviour, writeback, and delivery require controlled credentials and are not runtime-validated here. ## Media, speech, and image work -**Current implementation.** Gallery and image routes coordinate media features. Service modules own speech and media integrations; src/generated_images.py and src/visual_report.py support artifact handling and presentation. +- **How it works:** Gallery and image routes coordinate media features. Service modules own speech and media integrations; [`src/generated_images.py`](../../src/generated_images.py) and [`src/visual_report.py`](../../src/visual_report.py) support artifact handling and presentation. -**Canonical code locations.** routes/gallery/gallery_routes.py; routes/stt_routes.py; routes/tts_routes.py; src/generated_images.py; services/stt/; services/tts/; services/faces/; src/visual_report.py. +- **Canonical code:** [`routes/gallery/gallery_routes.py`](../../routes/gallery/gallery_routes.py); [`routes/stt_routes.py`](../../routes/stt_routes.py); [`routes/tts_routes.py`](../../routes/tts_routes.py); [`src/generated_images.py`](../../src/generated_images.py); [`services/stt/`](../../services/stt/); [`services/tts/`](../../services/tts/); [`services/faces/`](../../services/faces/); [`src/visual_report.py`](../../src/visual_report.py). -**Confirmed problems.** None recorded by this mapping. +- **Known problems:** None recorded by this mapping. -**Open questions.** Hardware- and provider-dependent media workflows were not exercised. +- **Open question:** Hardware- and provider-dependent media workflows were not exercised. ## Authentication, secrets, and privileged administration -**Current implementation.** core/auth.py and core/middleware.py provide identity and request gates. src/secret_storage.py encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: routes/vault_routes.py and src/tools/vault.py invoke the Bitwarden CLI and persist its session data in the application data area. +- **How it works:** [`core/auth.py`](../../core/auth.py) and [`core/middleware.py`](../../core/middleware.py) provide identity and request gates. [`src/secret_storage.py`](../../src/secret_storage.py) encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: [`routes/vault_routes.py`](../../routes/vault_routes.py) and [`src/tools/vault.py`](../../src/tools/vault.py) invoke the Bitwarden CLI and persist its session data in the application data area. -**Canonical code locations.** core/auth.py; core/middleware.py; routes/auth_routes.py; routes/api_token_routes.py; src/secret_storage.py; routes/vault_routes.py; src/tools/vault.py; routes/admin_wipe/admin_wipe_routes.py. +- **Canonical code:** [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/auth_routes.py`](../../routes/auth_routes.py); [`routes/api_token_routes.py`](../../routes/api_token_routes.py); [`src/secret_storage.py`](../../src/secret_storage.py); [`routes/vault_routes.py`](../../routes/vault_routes.py); [`src/tools/vault.py`](../../src/tools/vault.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py). -**Confirmed problems.** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised. +- **Known problems:** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised. -**Open questions.** What are the current confidentiality, ownership, rotation, and backup semantics for vault session data? +- **Open question:** What are the current confidentiality, ownership, rotation, and backup semantics for vault session data? ## Persistence, background work, and operations -**Current implementation.** SQLite models and persistence are centred in core/database.py; managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation. +- **How it works:** SQLite models and persistence are centred in [`core/database.py`](../../core/database.py); managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation. -**Canonical code locations.** core/database.py; src/runtime_paths.py; src/task_scheduler.py; src/bg_jobs.py; src/bg_monitor.py; routes/backup_routes.py; routes/cleanup/cleanup_routes.py; routes/admin_wipe/admin_wipe_routes.py; scripts/odysseus-backup; docs/backup-restore.md. +- **Canonical code:** [`core/database.py`](../../core/database.py); [`src/runtime_paths.py`](../../src/runtime_paths.py); [`src/task_scheduler.py`](../../src/task_scheduler.py); [`src/bg_jobs.py`](../../src/bg_jobs.py); [`src/bg_monitor.py`](../../src/bg_monitor.py); [`routes/backup_routes.py`](../../routes/backup_routes.py); [`routes/cleanup/cleanup_routes.py`](../../routes/cleanup/cleanup_routes.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py); [`scripts/odysseus-backup`](../../scripts/odysseus-backup); [`docs/backup-restore.md`](../../docs/backup-restore.md). -**Confirmed problems.** None recorded by this mapping. +- **Known problems:** None recorded by this mapping. -**Open questions.** What current behaviour applies to background execution, cancellation, retries, and authority inheritance? +- **Open question:** What current behaviour applies to background execution, cancellation, retries, and authority inheritance? From 69041b5f50b845231417bd79a2d5afccd2d9bad3 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Sun, 26 Jul 2026 19:13:13 +0100 Subject: [PATCH 04/43] docs: correct confirmed stale guidance --- THREAT_MODEL.md | 10 +++++----- docs/setup.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 48665a61d..ec10cd105 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -68,14 +68,14 @@ External content that reaches the LLM is treated as untrusted via `src/prompt_se - `X-Content-Type-Options: nosniff` and `Referrer-Policy: no-referrer` everywhere. - **CSP:** nonce-based `script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net`. `style-src 'unsafe-inline'` is intentionally kept — `static/index.html` ships inline `<style>` blocks and JS modules set `style=""` attributes at runtime. Inline styles do not execute script so the risk is visual-only. Removing this requires templating the HTML files and auditing all JS-set style attributes. +## Token-Supplied Model Endpoints + +Direct `/api/v1/chat` requests with a token-supplied `base_url` must use a public HTTP(S) endpoint. This restriction applies only to untrusted direct values; administrator-configured endpoints may intentionally use local or LAN URLs for private model providers. + ## Known Gaps These are open, acknowledged, and contributor help is welcome: 1. **No shell/filesystem sandbox.** The agent `bash` and `read_file`/`write_file` tools run as the app process user with no network egress filtering or filesystem confinement. A successful prompt-injection reaching a shell-enabled admin session can make outbound requests to internal services. See #1058 for the sandbox proposal. -2. **SSRF via `/api/v1/chat` `base_url` parameter.** A chat-scoped API token can supply an arbitrary `base_url`; the server forwards the LLM request to that host without validating the scheme or address. PR #1039 fixes this. - -3. **`src/search/` partial consolidation.** `src.search.core` and `src.search.providers` correctly alias `services.search` via `sys.modules` replacement. `analytics`, `cache`, `content`, `query`, and `ranking` are still independent copies that can drift. The SSRF regression tests in `tests/test_webhook_ssrf_resilience.py` test `src.webhook_manager` directly (separate from search), so the safety net there is intact. See #1058. - -4. **Token scopes are coarse.** There is no way to grant a session a subset of the owning user's privileges. Companion/mobile tokens carry either `chat` or `admin` scope with no per-capability granularity. +2. **Token scopes are coarse.** There is no way to grant a session a subset of the owning user's privileges. Companion/mobile tokens carry either `chat` or `admin` scope with no per-capability granularity. diff --git a/docs/setup.md b/docs/setup.md index 53a6fb28c..61381461b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -441,7 +441,7 @@ uv pip sync requirements.lock # reproduce it exactly la ### Outlook / Office 365 email Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox -passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the +passwords will fail. See [email-outlook.md](email-outlook.md) for the current limitation and the planned integration direction. ## Security Notes From ab6f3fa70b7f9dc2863ed199ed710c34582cdf13 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Sun, 26 Jul 2026 19:57:49 +0100 Subject: [PATCH 05/43] docs(discovery): clarify non-canonical evidence role --- specs/discovery/README.md | 23 ++++++++++++----------- specs/discovery/safety-boundaries.md | 2 +- specs/discovery/system-map.md | 28 ++++++++++++++-------------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/specs/discovery/README.md b/specs/discovery/README.md index 166ed3656..4343f9ec7 100644 --- a/specs/discovery/README.md +++ b/specs/discovery/README.md @@ -1,33 +1,34 @@ # Odysseus discovery maps -Compact, code-grounded maps of the cross-cutting systems in the checked-in Odysseus codebase. They are not a feature certification or substitute for normal testing, and do not become a permanent audit system. +Compact, code-grounded discovery maps of cross-cutting systems in the checked-in Odysseus codebase. They preserve investigation context and open factual questions; they are not canonical subsystem specifications, a feature certification, or a substitute for normal testing. > [!IMPORTANT] -> Code is the source of truth for current behaviour. These specifications describe only the checked-in system and its current boundaries; keep them synchronized with relevant code changes. +> Checked-in code is the source of truth for current behaviour. Mature subsystem specifications, where they exist, are the canonical documentation of accepted subsystem behaviour. Check code, tests, and configuration before reconciling a discovery finding. Discovery remains non-canonical. ## Explore the maps | Document | Purpose | |---|---| -| [Current system map](system-map.md) | Explains the current subsystem boundaries, implementation locations, confirmed problems, and factual open questions. | -| [Safety boundaries](safety-boundaries.md) | Maps broad authority, safeguards, confirmed risks or gaps, and unverified behaviour. | +| [Current system map](system-map.md) | Records evidence locations, confirmed local observations, and factual open questions about subsystem boundaries. | +| [Safety boundaries](safety-boundaries.md) | Records evidence about broad authority, safeguards, confirmed risks or gaps, and unverified behaviour. | ## Working rules - **Trace the code first.** Confirm the current path in source before recording a claim. -- **Keep specs current-state only.** Do not record intentions, design direction, refactor plans, or decision history here. -- **Synchronize with code.** Update this package when a code change alters a documented cross-cutting system, authority boundary, or canonical implementation location. +- **Promote selectively.** When an owning mature specification exists, add a fact only when it is verified, useful, and not already represented there. +- **Record missing ownership.** When no owning specification exists, retain the verified finding in discovery and record missing documentation ownership as a follow-up. +- **Retain uncertainty here.** Keep unresolved questions and useful investigation context in discovery rather than treating them as canonical truth. +- **Keep specifications current-state only.** Do not record intentions, design direction, refactor plans, decision history, priority, ownership, or sequencing here. - **Investigate with cause.** Do not exhaustively revalidate existing functionality without a report, visible failure, relevant change, or high-authority review need. -- **Keep planning in Plane.** Record confirmed problems in the relevant project within the canonical Plane workspace; keep high-level design, prioritization, ownership, and refactor discussion there. - **Review authority carefully.** Give execution, data access, external tools, credentials, destructive operations, and unattended work focused review. - **Use stable locations.** Cite modules, routes, classes, and functions instead of fragile line ranges or generated evidence tables. -## Working flow +## Reconciliation flow 1. Start with the relevant map and trace the cited code. -2. For a defect, create or update the relevant project within the canonical Plane workspace with a reproducible report and ownership. -3. Use a Plane thread for design, prioritization, ownership, or refactor discussion. -4. After implementation changes the code, update the affected map to describe the resulting current state. +2. Classify the finding against current source evidence and an owning mature specification where one exists. +3. Promote only verified, useful facts that are missing from an existing owning specification. +4. When no owning specification exists, retain the verified finding here and record missing documentation ownership as a follow-up; otherwise retain unresolved context here and correct stale wording. > [!NOTE] > This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](../architecture-runtime-inventory.md) remains useful structural context, but is an explicitly draft snapshot. diff --git a/specs/discovery/safety-boundaries.md b/specs/discovery/safety-boundaries.md index ee8b67d98..4340aa8d5 100644 --- a/specs/discovery/safety-boundaries.md +++ b/specs/discovery/safety-boundaries.md @@ -1,7 +1,7 @@ # Safety boundaries > [!IMPORTANT] -> Broad authority does not by itself establish a vulnerability. This map records code-grounded safeguards, confirmed risks or gaps, and unverified behaviour. No destructive test, external connection, or real credential was used for this map. +> This non-canonical discovery map records code-grounded safeguards, confirmed risks or gaps, and unverified behaviour. Broad authority does not by itself establish a vulnerability. Verify the cited source before relying on a finding. No destructive test, external connection, or real credential was used for this map. ## Navigate the boundaries diff --git a/specs/discovery/system-map.md b/specs/discovery/system-map.md index 2f9f50eaa..ce65e2b02 100644 --- a/specs/discovery/system-map.md +++ b/specs/discovery/system-map.md @@ -1,7 +1,7 @@ # Current system map > [!NOTE] -> This map describes the checked-in code at the same commit as this document. It is a subsystem map, not an exhaustive feature catalog or runtime certification. Each section records only current implementation, canonical locations, confirmed current problems, and unresolved factual questions. +> This non-canonical discovery map is an evidence guide, not an exhaustive feature catalog or runtime certification. Verify the cited source before relying on a finding. Each section records local implementation observations, evidence locations, confirmed current problems, and unresolved factual questions. ## Navigate the system @@ -22,7 +22,7 @@ - **How it works:** [`app.py`](../../app.py) creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. [`src/app_initializer.py`](../../src/app_initializer.py) prepares application state; [`core/`](../../core/) provides persistence, authentication, middleware, sessions, and platform helpers. -- **Canonical code:** [`app.py`](../../app.py); [`src/app_initializer.py`](../../src/app_initializer.py); [`core/database.py`](../../core/database.py); [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/`](../../routes/). +- **Evidence locations:** [`app.py`](../../app.py); [`src/app_initializer.py`](../../src/app_initializer.py); [`core/database.py`](../../core/database.py); [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/`](../../routes/). - **Known problems:** None recorded by this mapping. @@ -32,7 +32,7 @@ - **How it works:** [`static/index.html`](../../static/index.html) is served by the root and SPA deep-link routes in [`app.py`](../../app.py); [`static/app.js`](../../static/app.js), [`static/style.css`](../../static/style.css), and [`static/js/`](../../static/js/) implement the client surface. -- **Canonical code:** [`static/index.html`](../../static/index.html); [`static/app.js`](../../static/app.js); [`static/js/`](../../static/js/); [`static/style.css`](../../static/style.css); [`app.py`](../../app.py) deep-link handlers. +- **Evidence locations:** [`static/index.html`](../../static/index.html); [`static/app.js`](../../static/app.js); [`static/js/`](../../static/js/); [`static/style.css`](../../static/style.css); [`app.py`](../../app.py) deep-link handlers. - **Known problems:** The `/backgrounds` route in [`app.py`](../../app.py) calls `serve_html_with_nonce` for `static/backgrounds.html`, but that file is absent from [`static/`](../../static/). This is a confirmed broken prototype route, not evidence about the rest of the frontend. @@ -42,7 +42,7 @@ - **How it works:** [`routes/chat_routes.py`](../../routes/chat_routes.py) and [`routes/chat_helpers.py`](../../routes/chat_helpers.py) coordinate requests, session state, and SSE delivery. [`src/chat_handler.py`](../../src/chat_handler.py), [`src/chat_processor.py`](../../src/chat_processor.py), [`src/llm_core.py`](../../src/llm_core.py), and [`src/session_actions.py`](../../src/session_actions.py) provide message preparation, provider interaction, and session operations. -- **Canonical code:** [`routes/chat_routes.py`](../../routes/chat_routes.py); [`routes/chat_helpers.py`](../../routes/chat_helpers.py); [`routes/session_routes.py`](../../routes/session_routes.py); [`src/chat_handler.py`](../../src/chat_handler.py); [`src/chat_processor.py`](../../src/chat_processor.py); [`src/llm_core.py`](../../src/llm_core.py); [`core/session_manager.py`](../../core/session_manager.py). +- **Evidence locations:** [`routes/chat_routes.py`](../../routes/chat_routes.py); [`routes/chat_helpers.py`](../../routes/chat_helpers.py); [`routes/session_routes.py`](../../routes/session_routes.py); [`src/chat_handler.py`](../../src/chat_handler.py); [`src/chat_processor.py`](../../src/chat_processor.py); [`src/llm_core.py`](../../src/llm_core.py); [`core/session_manager.py`](../../core/session_manager.py). - **Known problems:** [`src/agent_loop.py`](../../src/agent_loop.py) annotates `_resolved_tool_event_name` with `Any` but imports no `Any` and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline. @@ -52,7 +52,7 @@ - **How it works:** [`src/agent_loop.py`](../../src/agent_loop.py) drives multi-round tool use. [`src/tool_execution.py`](../../src/tool_execution.py) dispatches calls and binds workspace context. [`src/agent_tools/`](../../src/agent_tools/) contains individual implementations; [`src/tool_security.py`](../../src/tool_security.py) and [`src/tool_policy.py`](../../src/tool_policy.py) apply role and request policies. Long-running command work is represented by [`src/bg_jobs.py`](../../src/bg_jobs.py). -- **Canonical code:** [`src/agent_loop.py`](../../src/agent_loop.py); [`src/tool_execution.py`](../../src/tool_execution.py); [`src/agent_tools/`](../../src/agent_tools/); [`src/tool_security.py`](../../src/tool_security.py); [`src/tool_policy.py`](../../src/tool_policy.py); [`src/tool_schemas.py`](../../src/tool_schemas.py); [`src/bg_jobs.py`](../../src/bg_jobs.py). +- **Evidence locations:** [`src/agent_loop.py`](../../src/agent_loop.py); [`src/tool_execution.py`](../../src/tool_execution.py); [`src/agent_tools/`](../../src/agent_tools/); [`src/tool_security.py`](../../src/tool_security.py); [`src/tool_policy.py`](../../src/tool_policy.py); [`src/tool_schemas.py`](../../src/tool_schemas.py); [`src/bg_jobs.py`](../../src/bg_jobs.py). - **Known problems:** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim. @@ -62,7 +62,7 @@ - **How it works:** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support. -- **Canonical code:** [`routes/model_routes.py`](../../routes/model_routes.py); [`src/model_discovery.py`](../../src/model_discovery.py); [`src/model_capabilities.py`](../../src/model_capabilities.py); [`src/endpoint_resolver.py`](../../src/endpoint_resolver.py); [`src/llm_core.py`](../../src/llm_core.py); [`routes/cookbook_routes.py`](../../routes/cookbook_routes.py); [`src/cookbook_serve_lifecycle.py`](../../src/cookbook_serve_lifecycle.py); [`services/hwfit/`](../../services/hwfit/). +- **Evidence locations:** [`routes/model_routes.py`](../../routes/model_routes.py); [`src/model_discovery.py`](../../src/model_discovery.py); [`src/model_capabilities.py`](../../src/model_capabilities.py); [`src/endpoint_resolver.py`](../../src/endpoint_resolver.py); [`src/llm_core.py`](../../src/llm_core.py); [`routes/cookbook_routes.py`](../../routes/cookbook_routes.py); [`src/cookbook_serve_lifecycle.py`](../../src/cookbook_serve_lifecycle.py); [`services/hwfit/`](../../services/hwfit/). - **Known problems:** None recorded by this mapping. @@ -72,9 +72,9 @@ - **How it works:** HTTP search routes use [`services/search/`](../../services/search/); research is exposed through [`routes/research/`](../../routes/research/) and implemented in [`services/research/`](../../services/research/), [`src/deep_research.py`](../../src/deep_research.py), and related helpers. [`src/search/`](../../src/search/) remains an import-compatibility layer for callers not yet moved to `services.search`. -- **Canonical code:** [`routes/search_routes.py`](../../routes/search_routes.py); [`services/search/`](../../services/search/); [`routes/research/research_routes.py`](../../routes/research/research_routes.py); [`services/research/`](../../services/research/); [`src/deep_research.py`](../../src/deep_research.py); [`src/search/`](../../src/search/). +- **Evidence locations:** [`routes/search_routes.py`](../../routes/search_routes.py); [`services/search/`](../../services/search/); [`routes/research/research_routes.py`](../../routes/research/research_routes.py); [`services/research/`](../../services/research/); [`src/deep_research.py`](../../src/deep_research.py); [`src/search/`](../../src/search/). -- **Known problems:** The two search package paths are a confirmed compatibility arrangement, not two independent canonical implementations: several [`src/search/`](../../src/search/) modules forward to [`services/search/`](../../services/search/). The remaining wrappers and compatibility imports can drift. +- **Known problems:** None recorded by this mapping. - **Open question:** No live provider request was made; provider configuration and network access remain unverified. @@ -82,7 +82,7 @@ - **How it works:** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in [`src/pdf_runtime.py`](../../src/pdf_runtime.py); form extraction and filling live separately in [`src/pdf_forms.py`](../../src/pdf_forms.py) and [`src/pdf_form_doc.py`](../../src/pdf_form_doc.py). -- **Canonical code:** [`routes/document_routes.py`](../../routes/document_routes.py); [`src/upload_handler.py`](../../src/upload_handler.py); [`src/document_processor.py`](../../src/document_processor.py); [`src/document_actions.py`](../../src/document_actions.py); [`src/personal_docs.py`](../../src/personal_docs.py); [`src/rag_manager.py`](../../src/rag_manager.py); [`src/embeddings.py`](../../src/embeddings.py); [`src/pdf_runtime.py`](../../src/pdf_runtime.py); [`src/pdf_forms.py`](../../src/pdf_forms.py); [`src/pdf_form_doc.py`](../../src/pdf_form_doc.py). +- **Evidence locations:** [`routes/document_routes.py`](../../routes/document_routes.py); [`src/upload_handler.py`](../../src/upload_handler.py); [`src/document_processor.py`](../../src/document_processor.py); [`src/document_actions.py`](../../src/document_actions.py); [`src/personal_docs.py`](../../src/personal_docs.py); [`src/rag_manager.py`](../../src/rag_manager.py); [`src/embeddings.py`](../../src/embeddings.py); [`src/pdf_runtime.py`](../../src/pdf_runtime.py); [`src/pdf_forms.py`](../../src/pdf_forms.py); [`src/pdf_form_doc.py`](../../src/pdf_form_doc.py). - **Known problems:** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure. @@ -92,7 +92,7 @@ - **How it works:** Memory routes use [`services/memory/`](../../services/memory/) and vector helpers. Skills are exposed through [`routes/skills_routes.py`](../../routes/skills_routes.py), stored and managed in [`services/memory/skills.py`](../../services/memory/skills.py), and may be imported through [`services/memory/skill_importer.py`](../../services/memory/skill_importer.py). -- **Canonical code:** [`routes/memory/memory_routes.py`](../../routes/memory/memory_routes.py); [`services/memory/`](../../services/memory/); [`src/memory.py`](../../src/memory.py); [`src/memory_vector.py`](../../src/memory_vector.py); [`routes/skills_routes.py`](../../routes/skills_routes.py); [`services/memory/skills.py`](../../services/memory/skills.py); [`services/memory/skill_importer.py`](../../services/memory/skill_importer.py). +- **Evidence locations:** [`routes/memory/memory_routes.py`](../../routes/memory/memory_routes.py); [`services/memory/`](../../services/memory/); [`src/memory.py`](../../src/memory.py); [`src/memory_vector.py`](../../src/memory_vector.py); [`routes/skills_routes.py`](../../routes/skills_routes.py); [`services/memory/skills.py`](../../services/memory/skills.py); [`services/memory/skill_importer.py`](../../services/memory/skill_importer.py). - **Known problems:** None recorded by this mapping. @@ -102,7 +102,7 @@ - **How it works:** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler. -- **Canonical code:** [`routes/email_routes.py`](../../routes/email_routes.py); [`routes/calendar_routes.py`](../../routes/calendar_routes.py); [`routes/contacts/contacts_routes.py`](../../routes/contacts/contacts_routes.py); [`routes/note/note_routes.py`](../../routes/note/note_routes.py); [`routes/task_routes.py`](../../routes/task_routes.py); [`routes/assistant_routes.py`](../../routes/assistant_routes.py); [`src/caldav_sync.py`](../../src/caldav_sync.py); [`src/caldav_writeback.py`](../../src/caldav_writeback.py); [`src/task_scheduler.py`](../../src/task_scheduler.py). +- **Evidence locations:** [`routes/email_routes.py`](../../routes/email_routes.py); [`routes/calendar_routes.py`](../../routes/calendar_routes.py); [`routes/contacts/contacts_routes.py`](../../routes/contacts/contacts_routes.py); [`routes/note/note_routes.py`](../../routes/note/note_routes.py); [`routes/task_routes.py`](../../routes/task_routes.py); [`routes/assistant_routes.py`](../../routes/assistant_routes.py); [`src/caldav_sync.py`](../../src/caldav_sync.py); [`src/caldav_writeback.py`](../../src/caldav_writeback.py); [`src/task_scheduler.py`](../../src/task_scheduler.py). - **Known problems:** None recorded by this mapping. @@ -112,7 +112,7 @@ - **How it works:** Gallery and image routes coordinate media features. Service modules own speech and media integrations; [`src/generated_images.py`](../../src/generated_images.py) and [`src/visual_report.py`](../../src/visual_report.py) support artifact handling and presentation. -- **Canonical code:** [`routes/gallery/gallery_routes.py`](../../routes/gallery/gallery_routes.py); [`routes/stt_routes.py`](../../routes/stt_routes.py); [`routes/tts_routes.py`](../../routes/tts_routes.py); [`src/generated_images.py`](../../src/generated_images.py); [`services/stt/`](../../services/stt/); [`services/tts/`](../../services/tts/); [`services/faces/`](../../services/faces/); [`src/visual_report.py`](../../src/visual_report.py). +- **Evidence locations:** [`routes/gallery/gallery_routes.py`](../../routes/gallery/gallery_routes.py); [`routes/stt_routes.py`](../../routes/stt_routes.py); [`routes/tts_routes.py`](../../routes/tts_routes.py); [`src/generated_images.py`](../../src/generated_images.py); [`services/stt/`](../../services/stt/); [`services/tts/`](../../services/tts/); [`services/faces/`](../../services/faces/); [`src/visual_report.py`](../../src/visual_report.py). - **Known problems:** None recorded by this mapping. @@ -122,7 +122,7 @@ - **How it works:** [`core/auth.py`](../../core/auth.py) and [`core/middleware.py`](../../core/middleware.py) provide identity and request gates. [`src/secret_storage.py`](../../src/secret_storage.py) encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: [`routes/vault_routes.py`](../../routes/vault_routes.py) and [`src/tools/vault.py`](../../src/tools/vault.py) invoke the Bitwarden CLI and persist its session data in the application data area. -- **Canonical code:** [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/auth_routes.py`](../../routes/auth_routes.py); [`routes/api_token_routes.py`](../../routes/api_token_routes.py); [`src/secret_storage.py`](../../src/secret_storage.py); [`routes/vault_routes.py`](../../routes/vault_routes.py); [`src/tools/vault.py`](../../src/tools/vault.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py). +- **Evidence locations:** [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/auth_routes.py`](../../routes/auth_routes.py); [`routes/api_token_routes.py`](../../routes/api_token_routes.py); [`src/secret_storage.py`](../../src/secret_storage.py); [`routes/vault_routes.py`](../../routes/vault_routes.py); [`src/tools/vault.py`](../../src/tools/vault.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py). - **Known problems:** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised. @@ -132,7 +132,7 @@ - **How it works:** SQLite models and persistence are centred in [`core/database.py`](../../core/database.py); managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation. -- **Canonical code:** [`core/database.py`](../../core/database.py); [`src/runtime_paths.py`](../../src/runtime_paths.py); [`src/task_scheduler.py`](../../src/task_scheduler.py); [`src/bg_jobs.py`](../../src/bg_jobs.py); [`src/bg_monitor.py`](../../src/bg_monitor.py); [`routes/backup_routes.py`](../../routes/backup_routes.py); [`routes/cleanup/cleanup_routes.py`](../../routes/cleanup/cleanup_routes.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py); [`scripts/odysseus-backup`](../../scripts/odysseus-backup); [`docs/backup-restore.md`](../../docs/backup-restore.md). +- **Evidence locations:** [`core/database.py`](../../core/database.py); [`src/runtime_paths.py`](../../src/runtime_paths.py); [`src/task_scheduler.py`](../../src/task_scheduler.py); [`src/bg_jobs.py`](../../src/bg_jobs.py); [`src/bg_monitor.py`](../../src/bg_monitor.py); [`routes/backup_routes.py`](../../routes/backup_routes.py); [`routes/cleanup/cleanup_routes.py`](../../routes/cleanup/cleanup_routes.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py); [`scripts/odysseus-backup`](../../scripts/odysseus-backup); [`docs/backup-restore.md`](../../docs/backup-restore.md). - **Known problems:** None recorded by this mapping. From 7fe3bc64829a184cc43fd99de88b0a420c4743ac Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Sun, 26 Jul 2026 20:11:10 +0100 Subject: [PATCH 06/43] docs(discovery): move discovery to repository root --- {specs/discovery => discovery}/README.md | 2 +- .../safety-boundaries.md | 18 +-- discovery/system-map.md | 139 ++++++++++++++++++ specs/discovery/system-map.md | 139 ------------------ 4 files changed, 149 insertions(+), 149 deletions(-) rename {specs/discovery => discovery}/README.md (94%) rename {specs/discovery => discovery}/safety-boundaries.md (79%) create mode 100644 discovery/system-map.md delete mode 100644 specs/discovery/system-map.md diff --git a/specs/discovery/README.md b/discovery/README.md similarity index 94% rename from specs/discovery/README.md rename to discovery/README.md index 4343f9ec7..f4c205837 100644 --- a/specs/discovery/README.md +++ b/discovery/README.md @@ -31,4 +31,4 @@ Compact, code-grounded discovery maps of cross-cutting systems in the checked-in 4. When no owning specification exists, retain the verified finding here and record missing documentation ownership as a follow-up; otherwise retain unresolved context here and correct stale wording. > [!NOTE] -> This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](../architecture-runtime-inventory.md) remains useful structural context, but is an explicitly draft snapshot. +> This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](../specs/architecture-runtime-inventory.md) remains useful structural context, but is an explicitly draft snapshot. diff --git a/specs/discovery/safety-boundaries.md b/discovery/safety-boundaries.md similarity index 79% rename from specs/discovery/safety-boundaries.md rename to discovery/safety-boundaries.md index 4340aa8d5..fa3ab7f3b 100644 --- a/specs/discovery/safety-boundaries.md +++ b/discovery/safety-boundaries.md @@ -37,7 +37,7 @@ - **User-controlled inputs:** Tool paths, patches, file contents, search patterns, and workspace selection passed into the tool dispatcher. -- **Current safeguards:** [`src/tool_execution.py`](../../src/tool_execution.py) resolves paths, blocks sensitive subpaths, applies allowlist containment, and tightens paths to the active workspace when one is bound. File tools use those resolvers. +- **Current safeguards:** [`src/tool_execution.py`](../src/tool_execution.py) resolves paths, blocks sensitive subpaths, applies allowlist containment, and tightens paths to the active workspace when one is bound. File tools use those resolvers. - **Confirmed risks or gaps:** Intentional authority with safeguards. The file-tool policy does not sandbox the shell; treating a workspace as a whole-process containment boundary would be incorrect. @@ -51,7 +51,7 @@ - **User-controlled inputs:** Chat content, attached/retrieved content that may influence the model, tool arguments, per-request tool selection, and policy toggles. -- **Current safeguards:** [`src/tool_security.py`](../../src/tool_security.py) blocks protected tools for non-admin users and fails closed for malformed tool names; [`src/tool_policy.py`](../../src/tool_policy.py) supports disabled and guide-only policy; prompt-security helpers label untrusted context. +- **Current safeguards:** [`src/tool_security.py`](../src/tool_security.py) blocks protected tools for non-admin users and fails closed for malformed tool names; [`src/tool_policy.py`](../src/tool_policy.py) supports disabled and guide-only policy; prompt-security helpers label untrusted context. - **Confirmed risks or gaps:** Credible risk requiring verification: aliases, legacy text tools, native function calls, and MCP-qualified names must all reach the same policy outcome. The code has specific alias handling for email/MCP names, which makes this a sensitive compatibility seam. @@ -65,7 +65,7 @@ - **User-controlled inputs:** Server configuration, remote OAuth completion, tool arguments, and model-selected MCP calls. -- **Current safeguards:** MCP routes are registered through [`routes/mcp_routes.py`](../../routes/mcp_routes.py); MCP-qualified tools are denied to non-admin users by [`src/tool_security.py`](../../src/tool_security.py). OAuth state and token persistence are handled in [`src/mcp_oauth.py`](../../src/mcp_oauth.py). +- **Current safeguards:** MCP routes are registered through [`routes/mcp_routes.py`](../routes/mcp_routes.py); MCP-qualified tools are denied to non-admin users by [`src/tool_security.py`](../src/tool_security.py). OAuth state and token persistence are handled in [`src/mcp_oauth.py`](../src/mcp_oauth.py). - **Confirmed risks or gaps:** Credible risk requiring verification: an MCP server authority is broader than the application can infer from its tool name. This map does not establish a trust or approval model for server installation and individual tool invocation. @@ -79,7 +79,7 @@ - **User-controlled inputs:** Search/fetch URLs, imported skill URLs, webhook configuration, and some endpoint settings. -- **Current safeguards:** [`src/url_security.py`](../../src/url_security.py) validates untrusted public HTTP URLs and fails closed on unsuitable schemes or private addresses. [`services/search/content.py`](../../services/search/content.py) resolves and rejects non-public hosts, pins resolved addresses for fetches, caps bodies, and limits redirects. +- **Current safeguards:** [`src/url_security.py`](../src/url_security.py) validates untrusted public HTTP URLs and fails closed on unsuitable schemes or private addresses. [`services/search/content.py`](../services/search/content.py) resolves and rejects non-public hosts, pins resolved addresses for fetches, caps bodies, and limits redirects. - **Confirmed risks or gaps:** Intentional split: administrator-created model endpoints may target private providers, while untrusted URLs use public-address checks. That distinction is required for self-hosted deployments but needs explicit call-site review. @@ -93,9 +93,9 @@ - **User-controlled inputs:** Administrative configuration, login/unlock requests, imported settings, and agent vault tool arguments. -- **Current safeguards:** [`src/secret_storage.py`](../../src/secret_storage.py) uses a locally stored Fernet key with restrictive permissions for supported database secrets. Vault routes require an administrator, avoid passing master passwords in command arguments, and set restrictive permissions on the vault-session file. +- **Current safeguards:** [`src/secret_storage.py`](../src/secret_storage.py) uses a locally stored Fernet key with restrictive permissions for supported database secrets. Vault routes require an administrator, avoid passing master passwords in command arguments, and set restrictive permissions on the vault-session file. -- **Confirmed risks or gaps:** Confirmed current boundary: vault session data is persisted through the vault path, not through [`src/secret_storage.py`](../../src/secret_storage.py). This is an unresolved question about current security semantics, not a confirmed exposure. +- **Confirmed risks or gaps:** Confirmed current boundary: vault session data is persisted through the vault path, not through [`src/secret_storage.py`](../src/secret_storage.py). This is an unresolved question about current security semantics, not a confirmed exposure. - **Unverified behaviour:** Current encryption-at-rest, owner scope, rotation, lock/logout, backup/restore, and log/tool-result exposure behaviour remains unverified. @@ -107,7 +107,7 @@ - **User-controlled inputs:** Login/signup data, session cookies, API tokens, authentication configuration, and requests to privileged routes. -- **Current safeguards:** [`core/auth.py`](../../core/auth.py), [`core/middleware.py`](../../core/middleware.py), and route-level checks establish identity and administrator gates. [`app.py`](../../app.py) warns when localhost bypass is configured; [`SECURITY.md`](../../SECURITY.md) documents deployment requirements. +- **Current safeguards:** [`core/auth.py`](../core/auth.py), [`core/middleware.py`](../core/middleware.py), and route-level checks establish identity and administrator gates. [`app.py`](../app.py) warns when localhost bypass is configured; [`SECURITY.md`](../SECURITY.md) documents deployment requirements. - **Confirmed risks or gaps:** Intentional authority with safeguards. Security depends on deployments keeping authentication enabled and internal services private; this map does not audit reverse-proxy or environment configuration. @@ -123,7 +123,7 @@ - **Current safeguards:** Administrative wipe routes use the administrative boundary. Cleanup exposes a preview route before mutation. The documented backup tool requires explicit restore confirmation, stages the old data directory, and validates archive members before extraction. -- **Confirmed risks or gaps:** Intentional destructive authority. Backup archives contain secrets by design, as documented in [`docs/backup-restore.md`](../../docs/backup-restore.md); this is an operator confidentiality responsibility, not a code defect established here. +- **Confirmed risks or gaps:** Intentional destructive authority. Backup archives contain secrets by design, as documented in [`docs/backup-restore.md`](../docs/backup-restore.md); this is an operator confidentiality responsibility, not a code defect established here. - **Unverified behaviour:** Role-gate, confirmation, archive-rejection, staged-recovery, and owner-isolation behaviour remains unverified. No destructive runtime test was performed. @@ -135,7 +135,7 @@ - **User-controlled inputs:** Stored task prompt, schedule, model/crew selection, enabled-tool configuration, output target, and prior persisted state. -- **Current safeguards:** [`src/task_scheduler.py`](../../src/task_scheduler.py) serializes execution, records task runs, associates work with an owner, and applies the agent owner-based tool gate. [`src/bg_jobs.py`](../../src/bg_jobs.py) keeps bounded state and can terminate overlong subprocess jobs. +- **Current safeguards:** [`src/task_scheduler.py`](../src/task_scheduler.py) serializes execution, records task runs, associates work with an owner, and applies the agent owner-based tool gate. [`src/bg_jobs.py`](../src/bg_jobs.py) keeps bounded state and can terminate overlong subprocess jobs. - **Confirmed risks or gaps:** Credible risk requiring verification: authority is inherited and exercised later, so changes to roles, task configuration, and disabled tools must be checked at execution time rather than assumed from task creation. diff --git a/discovery/system-map.md b/discovery/system-map.md new file mode 100644 index 000000000..938af76f3 --- /dev/null +++ b/discovery/system-map.md @@ -0,0 +1,139 @@ +# Current system map + +> [!NOTE] +> This non-canonical discovery map is an evidence guide, not an exhaustive feature catalog or runtime certification. Verify the cited source before relying on a finding. Each section records local implementation observations, evidence locations, confirmed current problems, and unresolved factual questions. + +## Navigate the system + +- [Startup and application composition](#startup-and-application-composition) +- [Frontend shell and browser interaction](#frontend-shell-and-browser-interaction) +- [Chat, sessions, and streaming](#chat-sessions-and-streaming) +- [Agents, tools, and execution](#agents-tools-and-execution) +- [Models, providers, and local serving](#models-providers-and-local-serving) +- [Search and research](#search-and-research) +- [Documents, retrieval, and personal knowledge](#documents-retrieval-and-personal-knowledge) +- [Memory and skills](#memory-and-skills) +- [Email, calendar, contacts, notes, and tasks](#email-calendar-contacts-notes-and-tasks) +- [Media, speech, and image work](#media-speech-and-image-work) +- [Authentication, secrets, and privileged administration](#authentication-secrets-and-privileged-administration) +- [Persistence, background work, and operations](#persistence-background-work-and-operations) + +## Startup and application composition + +- **How it works:** [`app.py`](../app.py) creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. [`src/app_initializer.py`](../src/app_initializer.py) prepares application state; [`core/`](../core/) provides persistence, authentication, middleware, sessions, and platform helpers. + +- **Evidence locations:** [`app.py`](../app.py); [`src/app_initializer.py`](../src/app_initializer.py); [`core/database.py`](../core/database.py); [`core/auth.py`](../core/auth.py); [`core/middleware.py`](../core/middleware.py); [`routes/`](../routes/). + +- **Known problems:** None recorded by this mapping. + +- **Open question:** Which component currently owns startup and shutdown for each long-lived service? + +## Frontend shell and browser interaction + +- **How it works:** [`static/index.html`](../static/index.html) is served by the root and SPA deep-link routes in [`app.py`](../app.py); [`static/app.js`](../static/app.js), [`static/style.css`](../static/style.css), and [`static/js/`](../static/js/) implement the client surface. + +- **Evidence locations:** [`static/index.html`](../static/index.html); [`static/app.js`](../static/app.js); [`static/js/`](../static/js/); [`static/style.css`](../static/style.css); [`app.py`](../app.py) deep-link handlers. + +- **Known problems:** The `/backgrounds` route in [`app.py`](../app.py) calls `serve_html_with_nonce` for `static/backgrounds.html`, but that file is absent from [`static/`](../static/). This is a confirmed broken prototype route, not evidence about the rest of the frontend. + +- **Open question:** Is `/backgrounds` currently an intentionally supported route or an obsolete prototype? + +## Chat, sessions, and streaming + +- **How it works:** [`routes/chat_routes.py`](../routes/chat_routes.py) and [`routes/chat_helpers.py`](../routes/chat_helpers.py) coordinate requests, session state, and SSE delivery. [`src/chat_handler.py`](../src/chat_handler.py), [`src/chat_processor.py`](../src/chat_processor.py), [`src/llm_core.py`](../src/llm_core.py), and [`src/session_actions.py`](../src/session_actions.py) provide message preparation, provider interaction, and session operations. + +- **Evidence locations:** [`routes/chat_routes.py`](../routes/chat_routes.py); [`routes/chat_helpers.py`](../routes/chat_helpers.py); [`routes/session_routes.py`](../routes/session_routes.py); [`src/chat_handler.py`](../src/chat_handler.py); [`src/chat_processor.py`](../src/chat_processor.py); [`src/llm_core.py`](../src/llm_core.py); [`core/session_manager.py`](../core/session_manager.py). + +- **Known problems:** [`src/agent_loop.py`](../src/agent_loop.py) annotates `_resolved_tool_event_name` with `Any` but imports no `Any` and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline. + +- **Open question:** No end-to-end provider or browser streaming run was performed for this map. + +## Agents, tools, and execution + +- **How it works:** [`src/agent_loop.py`](../src/agent_loop.py) drives multi-round tool use. [`src/tool_execution.py`](../src/tool_execution.py) dispatches calls and binds workspace context. [`src/agent_tools/`](../src/agent_tools/) contains individual implementations; [`src/tool_security.py`](../src/tool_security.py) and [`src/tool_policy.py`](../src/tool_policy.py) apply role and request policies. Long-running command work is represented by [`src/bg_jobs.py`](../src/bg_jobs.py). + +- **Evidence locations:** [`src/agent_loop.py`](../src/agent_loop.py); [`src/tool_execution.py`](../src/tool_execution.py); [`src/agent_tools/`](../src/agent_tools/); [`src/tool_security.py`](../src/tool_security.py); [`src/tool_policy.py`](../src/tool_policy.py); [`src/tool_schemas.py`](../src/tool_schemas.py); [`src/bg_jobs.py`](../src/bg_jobs.py). + +- **Known problems:** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim. + +- **Open question:** Which native, legacy, and MCP-qualified invocation paths reach each policy gate? + +## Models, providers, and local serving + +- **How it works:** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support. + +- **Evidence locations:** [`routes/model_routes.py`](../routes/model_routes.py); [`src/model_discovery.py`](../src/model_discovery.py); [`src/model_capabilities.py`](../src/model_capabilities.py); [`src/endpoint_resolver.py`](../src/endpoint_resolver.py); [`src/llm_core.py`](../src/llm_core.py); [`routes/cookbook_routes.py`](../routes/cookbook_routes.py); [`src/cookbook_serve_lifecycle.py`](../src/cookbook_serve_lifecycle.py); [`services/hwfit/`](../services/hwfit/). + +- **Known problems:** None recorded by this mapping. + +- **Open question:** Which endpoint inputs are administrator-created and permitted to use private provider addresses? + +## Search and research + +- **How it works:** HTTP search routes use [`services/search/`](../services/search/); research is exposed through [`routes/research/`](../routes/research/) and implemented in [`services/research/`](../services/research/), [`src/deep_research.py`](../src/deep_research.py), and related helpers. [`src/search/`](../src/search/) remains an import-compatibility layer for callers not yet moved to `services.search`. + +- **Evidence locations:** [`routes/search_routes.py`](../routes/search_routes.py); [`services/search/`](../services/search/); [`routes/research/research_routes.py`](../routes/research/research_routes.py); [`services/research/`](../services/research/); [`src/deep_research.py`](../src/deep_research.py); [`src/search/`](../src/search/). + +- **Known problems:** None recorded by this mapping. + +- **Open question:** No live provider request was made; provider configuration and network access remain unverified. + +## Documents, retrieval, and personal knowledge + +- **How it works:** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in [`src/pdf_runtime.py`](../src/pdf_runtime.py); form extraction and filling live separately in [`src/pdf_forms.py`](../src/pdf_forms.py) and [`src/pdf_form_doc.py`](../src/pdf_form_doc.py). + +- **Evidence locations:** [`routes/document_routes.py`](../routes/document_routes.py); [`src/upload_handler.py`](../src/upload_handler.py); [`src/document_processor.py`](../src/document_processor.py); [`src/document_actions.py`](../src/document_actions.py); [`src/personal_docs.py`](../src/personal_docs.py); [`src/rag_manager.py`](../src/rag_manager.py); [`src/embeddings.py`](../src/embeddings.py); [`src/pdf_runtime.py`](../src/pdf_runtime.py); [`src/pdf_forms.py`](../src/pdf_forms.py); [`src/pdf_form_doc.py`](../src/pdf_form_doc.py). + +- **Known problems:** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure. + +- **Open question:** Optional PDF dependencies and representative uploaded documents were not exercised. + +## Memory and skills + +- **How it works:** Memory routes use [`services/memory/`](../services/memory/) and vector helpers. Skills are exposed through [`routes/skills_routes.py`](../routes/skills_routes.py), stored and managed in [`services/memory/skills.py`](../services/memory/skills.py), and may be imported through [`services/memory/skill_importer.py`](../services/memory/skill_importer.py). + +- **Evidence locations:** [`routes/memory/memory_routes.py`](../routes/memory/memory_routes.py); [`services/memory/`](../services/memory/); [`src/memory.py`](../src/memory.py); [`src/memory_vector.py`](../src/memory_vector.py); [`routes/skills_routes.py`](../routes/skills_routes.py); [`services/memory/skills.py`](../services/memory/skills.py); [`services/memory/skill_importer.py`](../services/memory/skill_importer.py). + +- **Known problems:** None recorded by this mapping. + +- **Open question:** Which imported skill content can reach execution-capable paths, and which validation occurs before that point? + +## Email, calendar, contacts, notes, and tasks + +- **How it works:** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler. + +- **Evidence locations:** [`routes/email_routes.py`](../routes/email_routes.py); [`routes/calendar_routes.py`](../routes/calendar_routes.py); [`routes/contacts/contacts_routes.py`](../routes/contacts/contacts_routes.py); [`routes/note/note_routes.py`](../routes/note/note_routes.py); [`routes/task_routes.py`](../routes/task_routes.py); [`routes/assistant_routes.py`](../routes/assistant_routes.py); [`src/caldav_sync.py`](../src/caldav_sync.py); [`src/caldav_writeback.py`](../src/caldav_writeback.py); [`src/task_scheduler.py`](../src/task_scheduler.py). + +- **Known problems:** None recorded by this mapping. + +- **Open question:** External account behaviour, writeback, and delivery require controlled credentials and are not runtime-validated here. + +## Media, speech, and image work + +- **How it works:** Gallery and image routes coordinate media features. Service modules own speech and media integrations; [`src/generated_images.py`](../src/generated_images.py) and [`src/visual_report.py`](../src/visual_report.py) support artifact handling and presentation. + +- **Evidence locations:** [`routes/gallery/gallery_routes.py`](../routes/gallery/gallery_routes.py); [`routes/stt_routes.py`](../routes/stt_routes.py); [`routes/tts_routes.py`](../routes/tts_routes.py); [`src/generated_images.py`](../src/generated_images.py); [`services/stt/`](../services/stt/); [`services/tts/`](../services/tts/); [`services/faces/`](../services/faces/); [`src/visual_report.py`](../src/visual_report.py). + +- **Known problems:** None recorded by this mapping. + +- **Open question:** Hardware- and provider-dependent media workflows were not exercised. + +## Authentication, secrets, and privileged administration + +- **How it works:** [`core/auth.py`](../core/auth.py) and [`core/middleware.py`](../core/middleware.py) provide identity and request gates. [`src/secret_storage.py`](../src/secret_storage.py) encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: [`routes/vault_routes.py`](../routes/vault_routes.py) and [`src/tools/vault.py`](../src/tools/vault.py) invoke the Bitwarden CLI and persist its session data in the application data area. + +- **Evidence locations:** [`core/auth.py`](../core/auth.py); [`core/middleware.py`](../core/middleware.py); [`routes/auth_routes.py`](../routes/auth_routes.py); [`routes/api_token_routes.py`](../routes/api_token_routes.py); [`src/secret_storage.py`](../src/secret_storage.py); [`routes/vault_routes.py`](../routes/vault_routes.py); [`src/tools/vault.py`](../src/tools/vault.py); [`routes/admin_wipe/admin_wipe_routes.py`](../routes/admin_wipe/admin_wipe_routes.py). + +- **Known problems:** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised. + +- **Open question:** What are the current confidentiality, ownership, rotation, and backup semantics for vault session data? + +## Persistence, background work, and operations + +- **How it works:** SQLite models and persistence are centred in [`core/database.py`](../core/database.py); managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation. + +- **Evidence locations:** [`core/database.py`](../core/database.py); [`src/runtime_paths.py`](../src/runtime_paths.py); [`src/task_scheduler.py`](../src/task_scheduler.py); [`src/bg_jobs.py`](../src/bg_jobs.py); [`src/bg_monitor.py`](../src/bg_monitor.py); [`routes/backup_routes.py`](../routes/backup_routes.py); [`routes/cleanup/cleanup_routes.py`](../routes/cleanup/cleanup_routes.py); [`routes/admin_wipe/admin_wipe_routes.py`](../routes/admin_wipe/admin_wipe_routes.py); [`scripts/odysseus-backup`](../scripts/odysseus-backup); [`docs/backup-restore.md`](../docs/backup-restore.md). + +- **Known problems:** None recorded by this mapping. + +- **Open question:** What current behaviour applies to background execution, cancellation, retries, and authority inheritance? diff --git a/specs/discovery/system-map.md b/specs/discovery/system-map.md deleted file mode 100644 index ce65e2b02..000000000 --- a/specs/discovery/system-map.md +++ /dev/null @@ -1,139 +0,0 @@ -# Current system map - -> [!NOTE] -> This non-canonical discovery map is an evidence guide, not an exhaustive feature catalog or runtime certification. Verify the cited source before relying on a finding. Each section records local implementation observations, evidence locations, confirmed current problems, and unresolved factual questions. - -## Navigate the system - -- [Startup and application composition](#startup-and-application-composition) -- [Frontend shell and browser interaction](#frontend-shell-and-browser-interaction) -- [Chat, sessions, and streaming](#chat-sessions-and-streaming) -- [Agents, tools, and execution](#agents-tools-and-execution) -- [Models, providers, and local serving](#models-providers-and-local-serving) -- [Search and research](#search-and-research) -- [Documents, retrieval, and personal knowledge](#documents-retrieval-and-personal-knowledge) -- [Memory and skills](#memory-and-skills) -- [Email, calendar, contacts, notes, and tasks](#email-calendar-contacts-notes-and-tasks) -- [Media, speech, and image work](#media-speech-and-image-work) -- [Authentication, secrets, and privileged administration](#authentication-secrets-and-privileged-administration) -- [Persistence, background work, and operations](#persistence-background-work-and-operations) - -## Startup and application composition - -- **How it works:** [`app.py`](../../app.py) creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. [`src/app_initializer.py`](../../src/app_initializer.py) prepares application state; [`core/`](../../core/) provides persistence, authentication, middleware, sessions, and platform helpers. - -- **Evidence locations:** [`app.py`](../../app.py); [`src/app_initializer.py`](../../src/app_initializer.py); [`core/database.py`](../../core/database.py); [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/`](../../routes/). - -- **Known problems:** None recorded by this mapping. - -- **Open question:** Which component currently owns startup and shutdown for each long-lived service? - -## Frontend shell and browser interaction - -- **How it works:** [`static/index.html`](../../static/index.html) is served by the root and SPA deep-link routes in [`app.py`](../../app.py); [`static/app.js`](../../static/app.js), [`static/style.css`](../../static/style.css), and [`static/js/`](../../static/js/) implement the client surface. - -- **Evidence locations:** [`static/index.html`](../../static/index.html); [`static/app.js`](../../static/app.js); [`static/js/`](../../static/js/); [`static/style.css`](../../static/style.css); [`app.py`](../../app.py) deep-link handlers. - -- **Known problems:** The `/backgrounds` route in [`app.py`](../../app.py) calls `serve_html_with_nonce` for `static/backgrounds.html`, but that file is absent from [`static/`](../../static/). This is a confirmed broken prototype route, not evidence about the rest of the frontend. - -- **Open question:** Is `/backgrounds` currently an intentionally supported route or an obsolete prototype? - -## Chat, sessions, and streaming - -- **How it works:** [`routes/chat_routes.py`](../../routes/chat_routes.py) and [`routes/chat_helpers.py`](../../routes/chat_helpers.py) coordinate requests, session state, and SSE delivery. [`src/chat_handler.py`](../../src/chat_handler.py), [`src/chat_processor.py`](../../src/chat_processor.py), [`src/llm_core.py`](../../src/llm_core.py), and [`src/session_actions.py`](../../src/session_actions.py) provide message preparation, provider interaction, and session operations. - -- **Evidence locations:** [`routes/chat_routes.py`](../../routes/chat_routes.py); [`routes/chat_helpers.py`](../../routes/chat_helpers.py); [`routes/session_routes.py`](../../routes/session_routes.py); [`src/chat_handler.py`](../../src/chat_handler.py); [`src/chat_processor.py`](../../src/chat_processor.py); [`src/llm_core.py`](../../src/llm_core.py); [`core/session_manager.py`](../../core/session_manager.py). - -- **Known problems:** [`src/agent_loop.py`](../../src/agent_loop.py) annotates `_resolved_tool_event_name` with `Any` but imports no `Any` and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline. - -- **Open question:** No end-to-end provider or browser streaming run was performed for this map. - -## Agents, tools, and execution - -- **How it works:** [`src/agent_loop.py`](../../src/agent_loop.py) drives multi-round tool use. [`src/tool_execution.py`](../../src/tool_execution.py) dispatches calls and binds workspace context. [`src/agent_tools/`](../../src/agent_tools/) contains individual implementations; [`src/tool_security.py`](../../src/tool_security.py) and [`src/tool_policy.py`](../../src/tool_policy.py) apply role and request policies. Long-running command work is represented by [`src/bg_jobs.py`](../../src/bg_jobs.py). - -- **Evidence locations:** [`src/agent_loop.py`](../../src/agent_loop.py); [`src/tool_execution.py`](../../src/tool_execution.py); [`src/agent_tools/`](../../src/agent_tools/); [`src/tool_security.py`](../../src/tool_security.py); [`src/tool_policy.py`](../../src/tool_policy.py); [`src/tool_schemas.py`](../../src/tool_schemas.py); [`src/bg_jobs.py`](../../src/bg_jobs.py). - -- **Known problems:** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim. - -- **Open question:** Which native, legacy, and MCP-qualified invocation paths reach each policy gate? - -## Models, providers, and local serving - -- **How it works:** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support. - -- **Evidence locations:** [`routes/model_routes.py`](../../routes/model_routes.py); [`src/model_discovery.py`](../../src/model_discovery.py); [`src/model_capabilities.py`](../../src/model_capabilities.py); [`src/endpoint_resolver.py`](../../src/endpoint_resolver.py); [`src/llm_core.py`](../../src/llm_core.py); [`routes/cookbook_routes.py`](../../routes/cookbook_routes.py); [`src/cookbook_serve_lifecycle.py`](../../src/cookbook_serve_lifecycle.py); [`services/hwfit/`](../../services/hwfit/). - -- **Known problems:** None recorded by this mapping. - -- **Open question:** Which endpoint inputs are administrator-created and permitted to use private provider addresses? - -## Search and research - -- **How it works:** HTTP search routes use [`services/search/`](../../services/search/); research is exposed through [`routes/research/`](../../routes/research/) and implemented in [`services/research/`](../../services/research/), [`src/deep_research.py`](../../src/deep_research.py), and related helpers. [`src/search/`](../../src/search/) remains an import-compatibility layer for callers not yet moved to `services.search`. - -- **Evidence locations:** [`routes/search_routes.py`](../../routes/search_routes.py); [`services/search/`](../../services/search/); [`routes/research/research_routes.py`](../../routes/research/research_routes.py); [`services/research/`](../../services/research/); [`src/deep_research.py`](../../src/deep_research.py); [`src/search/`](../../src/search/). - -- **Known problems:** None recorded by this mapping. - -- **Open question:** No live provider request was made; provider configuration and network access remain unverified. - -## Documents, retrieval, and personal knowledge - -- **How it works:** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in [`src/pdf_runtime.py`](../../src/pdf_runtime.py); form extraction and filling live separately in [`src/pdf_forms.py`](../../src/pdf_forms.py) and [`src/pdf_form_doc.py`](../../src/pdf_form_doc.py). - -- **Evidence locations:** [`routes/document_routes.py`](../../routes/document_routes.py); [`src/upload_handler.py`](../../src/upload_handler.py); [`src/document_processor.py`](../../src/document_processor.py); [`src/document_actions.py`](../../src/document_actions.py); [`src/personal_docs.py`](../../src/personal_docs.py); [`src/rag_manager.py`](../../src/rag_manager.py); [`src/embeddings.py`](../../src/embeddings.py); [`src/pdf_runtime.py`](../../src/pdf_runtime.py); [`src/pdf_forms.py`](../../src/pdf_forms.py); [`src/pdf_form_doc.py`](../../src/pdf_form_doc.py). - -- **Known problems:** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure. - -- **Open question:** Optional PDF dependencies and representative uploaded documents were not exercised. - -## Memory and skills - -- **How it works:** Memory routes use [`services/memory/`](../../services/memory/) and vector helpers. Skills are exposed through [`routes/skills_routes.py`](../../routes/skills_routes.py), stored and managed in [`services/memory/skills.py`](../../services/memory/skills.py), and may be imported through [`services/memory/skill_importer.py`](../../services/memory/skill_importer.py). - -- **Evidence locations:** [`routes/memory/memory_routes.py`](../../routes/memory/memory_routes.py); [`services/memory/`](../../services/memory/); [`src/memory.py`](../../src/memory.py); [`src/memory_vector.py`](../../src/memory_vector.py); [`routes/skills_routes.py`](../../routes/skills_routes.py); [`services/memory/skills.py`](../../services/memory/skills.py); [`services/memory/skill_importer.py`](../../services/memory/skill_importer.py). - -- **Known problems:** None recorded by this mapping. - -- **Open question:** Which imported skill content can reach execution-capable paths, and which validation occurs before that point? - -## Email, calendar, contacts, notes, and tasks - -- **How it works:** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler. - -- **Evidence locations:** [`routes/email_routes.py`](../../routes/email_routes.py); [`routes/calendar_routes.py`](../../routes/calendar_routes.py); [`routes/contacts/contacts_routes.py`](../../routes/contacts/contacts_routes.py); [`routes/note/note_routes.py`](../../routes/note/note_routes.py); [`routes/task_routes.py`](../../routes/task_routes.py); [`routes/assistant_routes.py`](../../routes/assistant_routes.py); [`src/caldav_sync.py`](../../src/caldav_sync.py); [`src/caldav_writeback.py`](../../src/caldav_writeback.py); [`src/task_scheduler.py`](../../src/task_scheduler.py). - -- **Known problems:** None recorded by this mapping. - -- **Open question:** External account behaviour, writeback, and delivery require controlled credentials and are not runtime-validated here. - -## Media, speech, and image work - -- **How it works:** Gallery and image routes coordinate media features. Service modules own speech and media integrations; [`src/generated_images.py`](../../src/generated_images.py) and [`src/visual_report.py`](../../src/visual_report.py) support artifact handling and presentation. - -- **Evidence locations:** [`routes/gallery/gallery_routes.py`](../../routes/gallery/gallery_routes.py); [`routes/stt_routes.py`](../../routes/stt_routes.py); [`routes/tts_routes.py`](../../routes/tts_routes.py); [`src/generated_images.py`](../../src/generated_images.py); [`services/stt/`](../../services/stt/); [`services/tts/`](../../services/tts/); [`services/faces/`](../../services/faces/); [`src/visual_report.py`](../../src/visual_report.py). - -- **Known problems:** None recorded by this mapping. - -- **Open question:** Hardware- and provider-dependent media workflows were not exercised. - -## Authentication, secrets, and privileged administration - -- **How it works:** [`core/auth.py`](../../core/auth.py) and [`core/middleware.py`](../../core/middleware.py) provide identity and request gates. [`src/secret_storage.py`](../../src/secret_storage.py) encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: [`routes/vault_routes.py`](../../routes/vault_routes.py) and [`src/tools/vault.py`](../../src/tools/vault.py) invoke the Bitwarden CLI and persist its session data in the application data area. - -- **Evidence locations:** [`core/auth.py`](../../core/auth.py); [`core/middleware.py`](../../core/middleware.py); [`routes/auth_routes.py`](../../routes/auth_routes.py); [`routes/api_token_routes.py`](../../routes/api_token_routes.py); [`src/secret_storage.py`](../../src/secret_storage.py); [`routes/vault_routes.py`](../../routes/vault_routes.py); [`src/tools/vault.py`](../../src/tools/vault.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py). - -- **Known problems:** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised. - -- **Open question:** What are the current confidentiality, ownership, rotation, and backup semantics for vault session data? - -## Persistence, background work, and operations - -- **How it works:** SQLite models and persistence are centred in [`core/database.py`](../../core/database.py); managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation. - -- **Evidence locations:** [`core/database.py`](../../core/database.py); [`src/runtime_paths.py`](../../src/runtime_paths.py); [`src/task_scheduler.py`](../../src/task_scheduler.py); [`src/bg_jobs.py`](../../src/bg_jobs.py); [`src/bg_monitor.py`](../../src/bg_monitor.py); [`routes/backup_routes.py`](../../routes/backup_routes.py); [`routes/cleanup/cleanup_routes.py`](../../routes/cleanup/cleanup_routes.py); [`routes/admin_wipe/admin_wipe_routes.py`](../../routes/admin_wipe/admin_wipe_routes.py); [`scripts/odysseus-backup`](../../scripts/odysseus-backup); [`docs/backup-restore.md`](../../docs/backup-restore.md). - -- **Known problems:** None recorded by this mapping. - -- **Open question:** What current behaviour applies to background execution, cancellation, retries, and authority inheritance? From c762efe1c97ae9df8e2f729e0488b9a400876d24 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Mon, 27 Jul 2026 16:45:48 +0100 Subject: [PATCH 07/43] docs: replace detailed roadmap with directional overview --- .github/ISSUE_TEMPLATE/feature_request.yml | 4 +- README.md | 2 +- ROADMAP.md | 118 +++++++----------- tests/test_provider_endpoints_url_building.py | 6 +- 4 files changed, 49 insertions(+), 81 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 4ee603ee9..ae8d18f3f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -8,8 +8,8 @@ body: value: | **Before submitting:** search [open issues](https://github.com/odysseus-dev/odysseus/issues) and [discussions](https://github.com/odysseus-dev/odysseus/discussions) first. - Feature requests that duplicate [ROADMAP.md](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md) - or an existing open issue will be closed as duplicates. + The [roadmap](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md) is directional rather than a complete backlog. + Feature requests that duplicate an existing issue or accepted proposal may be closed as duplicates. If your idea needs community input before it becomes a concrete proposal, start a [discussion](https://github.com/odysseus-dev/odysseus/discussions/categories/ideas) instead. diff --git a/README.md b/README.md index 705ec6b68..b8d2017d7 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/in ## Contributing -Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, docs, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md). +Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, documentation, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md), the [public roadmap](ROADMAP.md), and the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues). ## Security diff --git a/ROADMAP.md b/ROADMAP.md index f5c47ae18..bcf0c155c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,87 +1,55 @@ -# Roadmap / Help Wanted +# Roadmap -Odysseus is on a voyage, but not home yet. It works great for me (lol), but this ship is moving fast and feedback/help would be appreciated! (I don't know what I'm doing, help). +This document provides a high-level view of the areas Odysseus is currently improving. -If you see weird CSS, strange layout behavior, or a suspiciously murky corner of -the codebase, you are probably right to stay away. +It is directional rather than exhaustive. Priorities may change as the project evolves, defects are discovered, and maintainers learn more from implementation work and user feedback. -## High Priority +For current implementation work, see the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues). Accepted behaviour should be documented in the repository alongside the code. -- SQUASH BUGS -- Fresh install smoke tests on Linux, macOS, and Windows. Docker, native Python, - and WSL all need coverage. +## Current priorities -- Integration audit: do integrations even work? Confirm what works, what needs setup docs, and what should be removed or hidden. -- Cookbook reliability on other computers. This is probably the area most likely to need work across different machines, GPUs, drivers, shells, and Python environments. -- Cookbook SGLang support across platforms. Make sure SGLang setup/serve works - predictably on Linux, Windows/WSL, macOS where possible, Docker, and common - NVIDIA/AMD hardware paths. -- Deep Research model presets by hardware. Recommend approved model/parameter - profiles for small, medium, and large local setups so people with different - hardware can use Deep Research without guessing. Surface this either in Deep - Research settings or as a Cookbook scan/dropdown suggestion. -- Cookbook model scan/download ranking. Prioritize newer architectures and - better hardware-fit models instead of scoring everything almost the same. - Ranking should account for architecture age, quant format, VRAM/RAM fit, - backend support, vision/mmproj requirements, and likely serve reliability. -- Cookbook error feedback and logging. Failed downloads, dependency installs, - preflights, and serve jobs should show the actual command/output/error in the - UI, with copyable logs and clear next steps instead of just "crashed". -- Agent prompt/context bloat. Agent mode is too heavy for smaller local models: - tool schemas, skills, memory, documents, and instructions can eat the context - before the user request really starts. We need slimmer prompts, better tool - selection, smaller default tool sets, and clearer guidance for models with - 4k/8k/16k context windows. -- Local model speculative decoding support. For Odysseus-tuned local models, - plan to ship or recommend a small same-tokenizer draft model when the serving - backend supports it. Early vLLM testing showed a generic `Qwen3-0.6B` draft - beside `Qwen3-8B` can materially reduce wall time, while an unsupported - DSpark conversion performed poorly. Treat this as a supported draft-model lane - first; keep MTP-specific packaging as future work only when the architecture - and runtime support are real. Judge this by time-to-success, tool correctness, - grammar, and unchanged target output, not tokens/sec alone. -- Skill/tool prompt-injection audit. User-editable skills, notes, documents, - fetched pages, and memories should be treated as untrusted data. Keep testing - whether models follow malicious instructions from those surfaces. -- Better degraded-state reporting for ChromaDB, SearXNG, email, ntfy, and provider probes. -- Email performance audit. Fetching, searching, opening, deleting, and sending - email can feel slow, especially over IMAP/SMTP providers with high latency. - Need someone who knows mail performance to profile the current flow, identify - whether the bottleneck is IMAP folder select/fetch, cache invalidation, - attachment/body loading, SMTP handshakes, or frontend refresh behavior, then - propose safer caching/prefetch/batching without breaking multi-account state. -- Provider setup/probing audit for Anthropic, Gemini, Groq, xAI, OpenRouter, OpenAI, and DeepSeek. +### Reliability and setup -## Refactor Targets -- CSS cleanup. `static/style.css` basically Calypso's island atm. -- Tour core helper. The onboarding tours have too much copy-pasted scaffolding; promote a shared `tour-core.js` helper before adding more tours. -- Modal/window positioning cleanup. Some window controls have improved, but the - underlying popup/dropdown/fixed-position behavior is still too fragile. -- Mobile media override discoverability. A lot of "CSS did not move" bugs are mobile `@media` overrides of the same selector; comments or linting around desktop/mobile paired rules would help. -- Dead code pass for old routes, stale feature flags, and unused UI states. +- Improve fresh-install and smoke-test coverage across supported environments. +- Make provider setup, probing, and failure states more predictable. +- Improve Cookbook reliability across hardware, operating systems, drivers, shells, and serving backends. +- Improve degraded-state reporting and recovery guidance when optional services are unavailable. -## Frontend +### Local model workflows -- Expand the Editor for quicker, more robust everyday use. Better file/document - handling, smoother window behavior, clearer save/export flows, stronger image - editing affordances, and fewer brittle edge cases. -- Better AI integration for Notes and Todos. Notes should be easier for the - agent to read, update, summarize, and turn into actions. Todos should be - assignable to an agent from the UI, possibly through a button, task action, - or dedicated skill/tool flow. -- Mobile gallery/editor polish. Easier to launch/download inpaint model or any missing pieces. -- Accessibility pass: keyboard navigation, focus states, contrast, reduced motion. -- Improve empty states and error messages on fresh installs. -- Tighten first-run setup, hints, and tours so they do not repeat or fight each other. -- Vendor CDN assets eventually for a more fully self-hosted/offline mode. +- Improve hardware-aware model recommendations and compatibility guidance. +- Evaluate serving optimizations, including speculative decoding, through reproducible benchmarks. +- Improve installation, preflight checks, logging, and error reporting for local model serving. +- Reduce prompt and context overhead for smaller local models. -## Backend +### Safety and resilience -- More tests around endpoint probing and provider setup. -- Better task scheduler defaults and visibility. -- Backup/restore guide and helper flow for `data/`. -- Security hardening around admin-only tools and clear docs for their risk. +- Continue hardening tool execution, filesystem access, credentials, networking, and destructive operations. +- Treat content from documents, notes, memories, skills, and fetched pages as potentially untrusted. +- Improve security-focused regression coverage and operational guidance. +- Review integrations that expand access to sensitive data or privileged operations. -## Not The Focus Right Now +### Product usability -I prob shouldnt add more themes. +- Improve first-run setup, onboarding, hints, and tours. +- Improve accessibility, keyboard navigation, focus behaviour, contrast, and reduced-motion support. +- Improve empty states, error messages, and recovery paths. +- Strengthen Notes, Todos, Editor, mobile, and everyday workspace flows. + +### Architecture and maintainability + +- Reduce duplication and technical debt through focused, reviewable refactors. +- Improve subsystem documentation as behaviour and architecture become stable. +- Remove stale code, obsolete feature flags, and unsupported integrations. +- Keep implementation decisions grounded in current code and verified behaviour. + +## Tracking work + +Concrete implementation tasks, defects, proposals, and technical investigations are tracked in: + +- [GitHub Issues](https://github.com/odysseus-dev/odysseus/issues) +- [Contributing Guide](CONTRIBUTING.md) + +Maintainers may use additional private coordination tools for ownership, planning, and unresolved decisions. + +This roadmap is not a complete backlog or a guarantee that a particular item will be delivered. diff --git a/tests/test_provider_endpoints_url_building.py b/tests/test_provider_endpoints_url_building.py index 56d129e1c..7ddbe2ba8 100644 --- a/tests/test_provider_endpoints_url_building.py +++ b/tests/test_provider_endpoints_url_building.py @@ -1,8 +1,8 @@ """Provider endpoint URL-building tests. -Covers ``build_chat_url`` and ``build_models_url`` for every provider named in -ROADMAP.md: Anthropic, Gemini, Groq, xAI, OpenRouter, OpenAI, DeepSeek, Ollama -(local + cloud). +Covers ``build_chat_url`` and ``build_models_url`` for the provider families +exercised by this module: Anthropic, Gemini, Groq, xAI, OpenRouter, OpenAI, +DeepSeek, and Ollama (local and cloud). """ import pytest From 8670aed9adb6f5a1677a4406fcc669aa2ea5d6f3 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Tue, 28 Jul 2026 07:59:37 +0100 Subject: [PATCH 08/43] docs: classify architecture inventory and refresh guidance --- discovery/README.md | 2 +- discovery/architecture-runtime-inventory.md | 331 +++++++++++++ docs/ARCHITECTURE.md | 87 ++++ docs/STYLE.md | 72 +++ docs/pr-blocker-audit.md | 4 +- specs/architecture-runtime-inventory.md | 443 ++---------------- tests/LAYOUT_INVENTORY.md | 300 +++++------- tests/OVERSIZED_TEST_SPLIT_PLAN.md | 325 +++++++------ .../tools/build_oversized_test_split_plan.py | 8 +- 9 files changed, 832 insertions(+), 740 deletions(-) create mode 100644 discovery/architecture-runtime-inventory.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/STYLE.md diff --git a/discovery/README.md b/discovery/README.md index f4c205837..234726851 100644 --- a/discovery/README.md +++ b/discovery/README.md @@ -31,4 +31,4 @@ Compact, code-grounded discovery maps of cross-cutting systems in the checked-in 4. When no owning specification exists, retain the verified finding here and record missing documentation ownership as a follow-up; otherwise retain unresolved context here and correct stale wording. > [!NOTE] -> This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](../specs/architecture-runtime-inventory.md) remains useful structural context, but is an explicitly draft snapshot. +> This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](./architecture-runtime-inventory.md) preserves dated structural metrics, investigation context, and historical planning as an explicitly non-canonical snapshot. diff --git a/discovery/architecture-runtime-inventory.md b/discovery/architecture-runtime-inventory.md new file mode 100644 index 000000000..22a56d9bf --- /dev/null +++ b/discovery/architecture-runtime-inventory.md @@ -0,0 +1,331 @@ +# Architecture runtime inventory + +> [!WARNING] +> This document is a dated structural snapshot, not a canonical runtime specification. +> Counts, paths, and implementation details may drift as the repository changes. +> Verify implementation-sensitive claims against the current code and tests. + +- **Branch:** `discovery` +- **Commit:** `c762efe1c97a` +- **Generated:** `2026-07-28T05:38:14+01:00` +- **Historical context:** readability and refactor planning in [#4071](https://github.com/odysseus-dev/odysseus/issues/4071) and [#4082](https://github.com/odysseus-dev/odysseus/issues/4082) + +## Disposition + +> [!NOTE] +> Reviewed for documentation classification. Stable runtime structure and +> subsystem ownership have been transferred to the proposed canonical +> destination, [`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md), for +> maintainer review. +> +> This document retains dated metrics, rankings, investigation context, +> refactor-sensitive observations, and historical planning. Those contents are +> non-canonical and belong under `discovery/`. + +| Content | Authority and destination | +|---|---| +| Stable runtime structure | Proposed canonical destination: `docs/ARCHITECTURE.md` | +| Stable subsystem boundaries | Proposed canonical destination: `docs/ARCHITECTURE.md` | +| Frontend module organization | `static/js/MODULE_SUMMARY.md` | +| Counts, line totals, and rankings | This non-canonical inventory | +| Investigation context and open questions | `discovery/` | +| Refactor options and prioritization | Issues, Plane, or non-canonical discovery material | + +The transfer preserves the stable facts without promoting generated metrics or +historical prioritization into canonical documentation. + +## Purpose + +This inventory provides a reviewable map of the current repository structure, +large runtime modules, major subsystem boundaries, and refactor-sensitive areas. + +It does not: + +- define accepted subsystem behaviour; +- certify runtime correctness; +- prescribe a committed refactor sequence; +- replace focused specifications, tests, or source review. + +For cross-cutting implementation evidence, see the [discovery maps](./README.md). + +## Top-level runtime structure + +| Area | Role | +|---|---| +| `app.py` | FastAPI application composition and entry point | +| `launcher.py` | Application launch support | +| `setup.py` | Native setup workflow | +| `core/` | Authentication, middleware, persistence, sessions, and platform primitives | +| `routes/` | HTTP and API route handlers | +| `src/` | Application services, orchestration, tools, providers, and runtime helpers | +| `services/` | Domain-oriented service packages | +| `mcp_servers/` | Built-in MCP server implementations | +| `scripts/` | CLI tools, diagnostics, maintenance, and migration helpers | +| `static/` | No-build browser frontend and bundled assets | +| `tests/` | Automated test suite and supporting test infrastructure | + +## Directory snapshot + +| Directory | Tracked files | Tracked Python files | Direct subdirectories | +|---|---:|---:|---| +| `src/` | 143 | 143 | `agent_tools/`, `model_capability_readers/`, `search/`, `tools/` | +| `routes/` | 73 | 73 | `admin_wipe/`, `cleanup/`, `compare/`, `contacts/`, `gallery/`, `history/`, `memory/`, `note/`, `research/` | +| `core/` | 11 | 11 | None | +| `services/` | 42 | 40 | `docs/`, `faces/`, `hwfit/`, `memory/`, `research/`, `search/`, `shell/`, `stt/`, `tts/`, `youtube/` | +| `mcp_servers/` | 5 | 5 | None | +| `scripts/` | 44 | 17 | `_completion/`, `_lib/`, `demo_email/` | +| `static/js/` | 154 | 0 | `calendar/`, `color/`, `compare/`, `editor/`, `emailLibrary/`, `markdown/`, `model/`, `research/`, `util/` | +| `tests/` | 768 | 758 | `cli/`, `helpers/`, `streaming/`, `tools/` | + +> [!NOTE] +> Counts in this table use `git ls-files`, so generated caches, virtual +> environments, and other untracked local files are excluded. + +## Largest backend modules + +Large files are review signals, not proof that a module should be split. +Coupling, ownership, import compatibility, tests, and runtime authority matter more +than line count alone. + +| Rank | File | Lines | Classes | Top-level functions | Review signal | +|---:|---|---:|---:|---:|---| +| 1 | `routes/email_routes.py` | 6032 | 1 | 58 | High | +| 2 | `src/agent_loop.py` | 5248 | 0 | 63 | High | +| 3 | `routes/cookbook_routes.py` | 4545 | 0 | 16 | High | +| 4 | `mcp_servers/email_server.py` | 2920 | 0 | 77 | High | +| 5 | `src/llm_core.py` | 2895 | 3 | 85 | High | +| 6 | `src/builtin_actions.py` | 2845 | 2 | 27 | High | +| 7 | `routes/model_routes.py` | 2743 | 0 | 65 | High | +| 8 | `src/task_scheduler.py` | 2627 | 1 | 8 | Medium | +| 9 | `core/database.py` | 2562 | 28 | 67 | High | +| 10 | `routes/gallery/gallery_routes.py` | 2325 | 0 | 16 | Medium | +| 11 | `routes/chat_routes.py` | 2063 | 0 | 18 | Medium | +| 12 | `routes/shell_routes.py` | 1971 | 1 | 21 | Medium | +| 13 | `src/visual_report.py` | 1933 | 0 | 11 | Medium | +| 14 | `routes/email_helpers.py` | 1888 | 3 | 48 | Medium | +| 15 | `routes/document_routes.py` | 1810 | 0 | 5 | Medium | +| 16 | `src/tools/cookbook.py` | 1705 | 0 | 34 | Medium | +| 17 | `routes/calendar_routes.py` | 1667 | 2 | 19 | Medium | +| 18 | `routes/skills_routes.py` | 1662 | 3 | 19 | Medium | +| 19 | `src/tool_schemas.py` | 1595 | 0 | 3 | Medium | +| 20 | `routes/email_pollers.py` | 1551 | 0 | 23 | Medium | + +The largest current backend concentrations include: + +- email routing and helper logic; +- agent-loop orchestration; +- Cookbook lifecycle and serving logic; +- provider and model routing; +- task scheduling; +- shared database models and persistence helpers. + +These areas require focused ownership and compatibility analysis before structural +changes are attempted. + +## Largest frontend modules + +| Rank | File | Lines | +|---:|---|---:| +| 1 | `static/style.css` | 41132 | +| 2 | `static/js/document.js` | 11200 | +| 3 | `static/js/emailLibrary.js` | 8505 | +| 4 | `static/js/slashCommands.js` | 6520 | +| 5 | `static/js/chat.js` | 6001 | +| 6 | `static/js/settings.js` | 5819 | +| 7 | `static/js/notes.js` | 5365 | +| 8 | `static/app.js` | 4681 | +| 9 | `static/js/cookbookRunning.js` | 4433 | +| 10 | `static/js/galleryEditor.js` | 4386 | +| 11 | `static/js/cookbookServe.js` | 4305 | +| 12 | `static/js/calendar.js` | 3722 | +| 13 | `static/js/cookbook.js` | 3677 | +| 14 | `static/js/sessions.js` | 3665 | +| 15 | `static/js/documentLibrary.js` | 3422 | +| 16 | `static/js/tasks.js` | 3187 | +| 17 | `static/js/admin.js` | 3144 | +| 18 | `static/js/gallery.js` | 2958 | +| 19 | `static/js/cookbook-hwfit.js` | 2826 | +| 20 | `static/js/chatRenderer.js` | 2808 | + +The browser frontend remains a no-build ES-module application. Its current source +tree is authoritative; the maintained structural summary is available in +[`static/js/MODULE_SUMMARY.md`](../static/js/MODULE_SUMMARY.md). + +CSS modularization remains tracked separately in +[#2617](https://github.com/odysseus-dev/odysseus/issues/2617). + +## Major subsystem boundaries + +| Subsystem | Primary implementation locations | +|---|---| +| Application startup | `app.py`, `src/app_initializer.py`, `core/` | +| Authentication and sessions | `core/auth.py`, `core/middleware.py`, `core/session_manager.py`, `routes/auth_routes.py` | +| Chat and streaming | `routes/chat_routes.py`, `routes/chat_helpers.py`, `src/chat_handler.py`, `src/chat_processor.py`, `src/llm_core.py` | +| Agents and tools | `src/agent_loop.py`, `src/tool_execution.py`, `src/agent_tools/`, `src/tools/`, `src/tool_policy.py`, `src/tool_security.py` | +| Models and providers | `routes/model_routes.py`, `src/model_discovery.py`, `src/model_capabilities.py`, `src/endpoint_resolver.py`, `src/llm_core.py` | +| Cookbook and hardware fit | `routes/cookbook_routes.py`, `routes/cookbook_helpers.py`, `src/cookbook_serve_lifecycle.py`, `services/hwfit/` | +| Search and research | `routes/search_routes.py`, `services/search/`, `routes/research/`, `services/research/`, `src/deep_research.py` | +| Documents and retrieval | `routes/document_routes.py`, `src/document_processor.py`, `src/personal_docs.py`, `src/rag_manager.py`, `src/pdf_runtime.py` | +| Memory and skills | `routes/memory/`, `services/memory/`, `routes/skills_routes.py` | +| Email | `routes/email_routes.py`, `routes/email_helpers.py`, `routes/email_pollers.py`, `mcp_servers/email_server.py` | +| Calendar, contacts, notes, and tasks | `routes/calendar_routes.py`, `routes/contacts/`, `routes/note/`, `routes/task_routes.py`, `src/task_scheduler.py` | +| Media and speech | `routes/gallery/`, `routes/stt_routes.py`, `routes/tts_routes.py`, `services/stt/`, `services/tts/` | +| Persistence and operations | `core/database.py`, `src/runtime_paths.py`, `src/bg_jobs.py`, `routes/backup_routes.py`, `routes/cleanup/` | + +For a broader evidence map, see +[`system-map.md`](./system-map.md). + +## Refactor-sensitive areas + +### Shared persistence + +`core/database.py` is a central dependency containing models and shared persistence +helpers. Changes can affect routes, services, background work, tests, migrations, +and import compatibility. + +A split should not begin from file size alone. It requires: + +- an importer inventory; +- model and helper ownership decisions; +- migration compatibility checks; +- stable re-export or migration strategy; +- focused and full-suite validation. + +### Agent orchestration + +`src/agent_loop.py` coordinates model interaction, tool selection, policy decisions, +multi-round execution, and background behaviour. Extraction work must preserve tool +event semantics, policy enforcement, cancellation, and test patch points. + +Historical agent-loop modularization discussion is tracked in +[#3266](https://github.com/odysseus-dev/odysseus/issues/3266). + +### Tool implementation boundaries + +Tool implementation is no longer represented by one proposed future package alone. +Current responsibilities are distributed across: + +- `src/tool_execution.py`; +- `src/tool_schemas.py`; +- `src/tool_index.py`; +- `src/tool_policy.py`; +- `src/tool_security.py`; +- `src/agent_tools/`; +- `src/tools/`; +- remaining compatibility surfaces such as `src/tool_implementations.py`. + +Historical tool modularization work is tracked in +[#3629](https://github.com/odysseus-dev/odysseus/issues/3629). + +### Route ownership + +`routes/` now contains both flat modules and domain packages. Existing package +boundaries should be extended only through focused changes. Broad mechanical route +movement would affect registration, imports, tests, monkeypatch targets, and +compatibility paths. + +### Frontend concentration + +The no-build frontend contains several large JavaScript modules and one central CSS +file. Refactors should preserve module load order, global compatibility exports, +DOM contracts, deep-link handling, and browser behaviour. + +## Non-implemented architecture options + +> [!NOTE] +> The paths below are historical or possible design directions. They do not describe +> the current repository and are not approved implementation plans. + +Earlier planning discussed: + +- renaming `app.py` to `main.py`; +- moving agent orchestration into a new `src/agent/` package; +- introducing broad `src/domain/`, `src/infra/`, `src/api/`, or `src/pkg/` layers; +- moving all routes into domain subpackages; +- splitting database models into a new infrastructure hierarchy. + +These options should be reconsidered against the current tree rather than copied +forward as assumed targets. + +## Refactor guardrails + +- Keep structural changes behaviour-preserving. +- Change one ownership boundary at a time. +- Do not mix file movement with unrelated feature work. +- Preserve existing import and monkeypatch paths where compatibility is required. +- Identify focused tests before modifying high-authority modules. +- Validate startup, imports, and affected runtime paths. +- Avoid repository-wide package reorganizations without maintainer agreement. +- Treat generated metrics as snapshots, not architectural decisions. + +## Reproduce the snapshot + +Run these commands from the repository root. + +```bash +# Tracked directory totals +for dir in src routes core services mcp_servers scripts static/js tests; do + files="$(git ls-files "$dir" | wc -l)" + python_files="$(git ls-files "$dir" '*.py' | wc -l)" + + printf '%-14s tracked=%-5s python=%-5s\n' \ + "$dir" \ + "$files" \ + "$python_files" +done + +# Largest tracked backend files +git ls-files \ + 'app.py' \ + 'launcher.py' \ + 'setup.py' \ + 'core/*.py' \ + 'core/**/*.py' \ + 'routes/*.py' \ + 'routes/**/*.py' \ + 'services/*.py' \ + 'services/**/*.py' \ + 'src/*.py' \ + 'src/**/*.py' \ + 'mcp_servers/*.py' \ + 'scripts/*.py' \ + 'scripts/**/*.py' | +xargs wc -l | +sort -nr | +head -31 + +# Largest tracked frontend source files +git ls-files \ + 'static/*.js' \ + 'static/*.css' \ + 'static/*.html' \ + 'static/**/*.js' \ + 'static/**/*.css' \ + 'static/**/*.html' | +grep -vE '\.min\.js$' | +xargs wc -l | +sort -nr | +head -31 +``` + +## Validation for architecture changes + +Use the smallest relevant checks first, then expand according to risk: + +```bash +python3 -m compileall -q app.py core routes services src +venv/bin/python -m pytest tests/<focused-test-file>.py -q +venv/bin/python -m pytest -q +``` + +Startup, browser, Docker, and integration checks may also be required depending on +the affected boundary. + +## Related documentation + +- [Documentation style](../docs/STYLE.md) +- [Discovery maps](../discovery/README.md) +- [Current system map](../discovery/system-map.md) +- [Safety boundaries](../discovery/safety-boundaries.md) +- [Frontend module summary](../static/js/MODULE_SUMMARY.md) +- [Testing standard](../tests/TESTING_STANDARD.md) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 000000000..d00077d7e --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,87 @@ +# Architecture + +> [!NOTE] +> This document is the proposed canonical destination for stable high-level +> architecture facts. It remains subject to maintainer review. Source code, +> tests, and configuration remain authoritative for implementation-sensitive +> behaviour. + +## Purpose + +This document identifies the stable runtime boundaries and primary ownership +locations used to navigate and extend Odysseus. + +It intentionally excludes generated metrics, file-size rankings, refactor +priorities, unresolved investigation findings, and proposed package layouts. + +## Runtime structure + +| Area | Responsibility | +|---|---| +| `app.py` | FastAPI application composition and primary application entry point | +| `launcher.py` | Application launch support | +| `setup.py` | Native setup workflow | +| `core/` | Authentication, middleware, persistence, sessions, and platform primitives | +| `routes/` | HTTP and API route handlers | +| `src/` | Application orchestration, tools, providers, and runtime helpers | +| `services/` | Domain-oriented service implementations | +| `mcp_servers/` | Built-in MCP server implementations | +| `scripts/` | CLI tools, diagnostics, maintenance, and migration helpers | +| `static/` | No-build browser frontend and bundled assets | +| `tests/` | Automated tests and supporting test infrastructure | + +## Subsystem boundaries + +| Subsystem | Primary implementation locations | +|---|---| +| Application startup | `app.py`, `src/app_initializer.py`, `core/` | +| Authentication and sessions | `core/auth.py`, `core/middleware.py`, `core/session_manager.py`, `routes/auth_routes.py` | +| Chat and streaming | `routes/chat_routes.py`, `routes/chat_helpers.py`, `src/chat_handler.py`, `src/chat_processor.py`, `src/llm_core.py` | +| Agents and tools | `src/agent_loop.py`, `src/tool_execution.py`, `src/agent_tools/`, `src/tools/`, `src/tool_policy.py`, `src/tool_security.py` | +| Models and providers | `routes/model_routes.py`, `src/model_discovery.py`, `src/model_capabilities.py`, `src/endpoint_resolver.py`, `src/llm_core.py` | +| Cookbook and hardware fit | `routes/cookbook_routes.py`, `routes/cookbook_helpers.py`, `src/cookbook_serve_lifecycle.py`, `services/hwfit/` | +| Search and research | `routes/search_routes.py`, `services/search/`, `routes/research/`, `services/research/`, `src/deep_research.py` | +| Documents and retrieval | `routes/document_routes.py`, `src/document_processor.py`, `src/personal_docs.py`, `src/rag_manager.py`, `src/pdf_runtime.py` | +| Memory and skills | `routes/memory/`, `services/memory/`, `routes/skills_routes.py` | +| Email | `routes/email_routes.py`, `routes/email_helpers.py`, `routes/email_pollers.py`, `mcp_servers/email_server.py` | +| Calendar, contacts, notes, and tasks | `routes/calendar_routes.py`, `routes/contacts/`, `routes/note/`, `routes/task_routes.py`, `src/task_scheduler.py` | +| Media and speech | `routes/gallery/`, `routes/stt_routes.py`, `routes/tts_routes.py`, `services/stt/`, `services/tts/` | +| Persistence and operations | `core/database.py`, `src/runtime_paths.py`, `src/bg_jobs.py`, `routes/backup_routes.py`, `routes/cleanup/` | + +## Architectural constraints + +- Preserve established import and compatibility paths unless a focused change + explicitly migrates them. +- Keep HTTP concerns in route modules and reusable domain behaviour in runtime + or service modules. +- Treat shared persistence, agent orchestration, tool execution, and application + startup as high-authority boundaries. +- Change one ownership boundary at a time. +- Do not mix structural movement with unrelated feature behaviour. +- Validate affected imports, startup paths, compatibility surfaces, and tests. + +## Frontend + +The browser frontend is a no-build ES-module application under `static/`. + +Its maintained module-level structure is documented in +[`static/js/MODULE_SUMMARY.md`](../static/js/MODULE_SUMMARY.md). + +## Investigation and snapshots + +Non-canonical investigation material is maintained under [`discovery/`](../discovery/). + +The following documents may contain dated observations, metrics, unresolved +questions, or historical planning and must not be treated as specifications: + +- [`discovery/system-map.md`](../discovery/system-map.md) +- [`discovery/architecture-runtime-inventory.md`](../discovery/architecture-runtime-inventory.md) + +## Documentation authority + +- Code, tests, and configuration define implemented behaviour. +- Mature specifications define accepted subsystem behaviour where they exist. +- Following maintainer acceptance, this document will define the high-level + architecture map. +- Discovery documents preserve evidence and uncertainty but remain + non-canonical. diff --git a/docs/STYLE.md b/docs/STYLE.md new file mode 100644 index 000000000..23ea6d2c5 --- /dev/null +++ b/docs/STYLE.md @@ -0,0 +1,72 @@ +# Documentation style + +This guide defines the shared structure and writing conventions for Odysseus documentation. + +## Principles + +- Write for a clear audience and purpose. +- State whether a document is canonical, informational, a snapshot, or planning material. +- Prefer current behaviour over historical explanation. +- Link to source files, tests, issues, or other documentation when useful. +- Separate verified behaviour from assumptions, open questions, and future work. +- Keep headings descriptive and consistent. +- Use Markdown callouts where status or risk must be visible. +- Do not use emojis. + +## Document status + +Use a status callout near the top when the document is not normal canonical guidance. + +### Canonical documentation + +> [!IMPORTANT] +> This document describes accepted current behaviour. Verify implementation-sensitive details against the current code and tests. + +### Discovery material + +> [!NOTE] +> This is non-canonical discovery material. It records code-grounded observations and open questions. + +### Snapshot or inventory + +> [!WARNING] +> This document is a dated snapshot. Counts, paths, and implementation details may drift as the codebase changes. + +### Planning material + +> [!NOTE] +> This document records planning context. It does not define current runtime behaviour or guarantee future implementation. + +## Recommended structure + +Use the following sections where relevant: + +1. Title +2. Purpose or status callout +3. Scope +4. Current behaviour or guidance +5. Safety, limitations, or known gaps +6. Validation or evidence +7. Related documentation + +Not every document needs every section. + +## Writing style + +- Use concise sentences. +- Prefer direct language. +- Avoid jokes, filler, and informal warnings. +- Avoid repeating the same guidance across several files. +- Link to the owning document instead of duplicating large sections. +- Use lists for procedures, requirements, and comparisons. +- Use tables only when they improve scanning. +- Use fenced code blocks with an appropriate language identifier. +- Use relative repository links for internal files. + +## Authority + +The current code, tests, and configuration are the source of truth for implemented behaviour. + +Canonical documentation describes accepted behaviour and supported workflows. + +Discovery, inventory, and planning documents must identify themselves explicitly and must not silently become behavioural specifications. diff --git a/docs/pr-blocker-audit.md b/docs/pr-blocker-audit.md index b56f28cb3..75d94bc87 100644 --- a/docs/pr-blocker-audit.md +++ b/docs/pr-blocker-audit.md @@ -181,8 +181,8 @@ Dirty, blocked, conflicting, and unknown merge states are shown as risk/caution ## Validation ```bash -python3 -m py_compile scripts/pr_blocker_audit.py tests/test_pr_blocker_audit.py -python3 -m pytest tests/test_pr_blocker_audit.py -q +venv/bin/python -m py_compile scripts/pr_blocker_audit.py tests/test_pr_blocker_audit.py +venv/bin/python -m pytest tests/test_pr_blocker_audit.py -q python3 scripts/pr_blocker_audit.py --help git diff --check ``` diff --git a/specs/architecture-runtime-inventory.md b/specs/architecture-runtime-inventory.md index 5c8e4bc21..4ec156b15 100644 --- a/specs/architecture-runtime-inventory.md +++ b/specs/architecture-runtime-inventory.md @@ -1,412 +1,31 @@ -# Architecture Runtime Inventory - -> **Purpose**: Phase 0 planning baseline for codebase readability improvements (#4071). -> **Parent issue**: [#4082](https://github.com/odysseus-dev/odysseus/issues/4082) -> **Last updated**: dev@b58af42 | 2026-06-16 -> **Status**: Draft — to be reviewed before follow-up slices open. -> **Snapshot basis**: Importer / file / import-line counts are refreshed to `dev@b58af42` (2026-06-16) and are recomputable via the commands in §3.4. **Line counts** in §2.1 / §2.2 are a snapshot from an earlier baseline and drift as `dev` moves — recompute any of them with `wc -l <file>`. This inventory tracks structure and risk, not live metrics. - -This document maps the current runtime module structure, identifies high-risk boundaries, and recommends safe first refactor slices. It does **not** move files, change imports, or alter runtime behavior. - ---- - -## 1. Current Structure Overview - -### 1.1 Top-Level Layout - -``` -odysseus/ -├── app.py # FastAPI app entrypoint (1,145 lines) -├── conf/ # Configuration (config.py, settings.py, settings_scrub.py) -├── src/ # 95 flat .py files + 2 subdirectories -│ ├── agent_tools/ # Tool helpers: document, filesystem, subprocess, web -│ └── search/ # Search subsystem -├── routes/ # 54 flat .py files — HTTP route handlers -├── core/ # 10 files — database models, auth, middleware, session -├── mcp_servers/ # 5 files — MCP server implementations -├── scripts/ # CLI tools and one-shot scripts -├── static/ # Frontend HTML/CSS/JS -├── tests/ # 583 test files (~54,800 lines) -└── services/ # (exists as needed) -``` - -### 1.2 Directory Flatness Metric - -| Directory | Flat `.py` Files | Subdirectories | Concern | -|-----------|-----------------|----------------|---------| -| `src/` | **95** | 2 (`agent_tools/`, `search/`) | No domain grouping; 95 files in one directory | -| `routes/` | **54** | 0 | All route handlers in one flat directory | -| `core/` | 10 | 0 | Manageable, but `database.py` is oversized | - ---- - -## 2. Largest Runtime Modules - -### 2.1 Python Backend - -| Rank | File | Lines | Classes | Functions | Risk | -|------|------|-------|---------|-----------|------| -| 1 | `src/tool_implementations.py` | **4,032** | 0 | ~48 | **HIGH** | -| 2 | `routes/email_routes.py` | **3,245** | — | — | **MEDIUM** | -| 3 | `routes/cookbook_routes.py` | **2,969** | — | — | **MEDIUM** | -| 4 | `src/agent_loop.py` | **2,961** | 0 | ~24 | **HIGH** | -| 5 | `src/task_scheduler.py` | **2,330** | — | 5 | MEDIUM | -| 6 | `routes/model_routes.py` | **2,266** | — | — | MEDIUM | -| 7 | `core/database.py` | **2,265** | 28 | ~59 helpers | **HIGH** | -| 8 | `src/builtin_actions.py` | **2,262** | 2 | ~24 | MEDIUM | -| 9 | `src/llm_core.py` | **2,164** | — | — | MEDIUM | -| 10 | `mcp_servers/email_server.py` | 2,197 | — | — | LOW (separate process) | -| 11 | `src/visual_report.py` | 1,918 | — | — | LOW | -| 12 | `routes/gallery_routes.py` | 1,896 | — | — | LOW | -| 13 | `src/ai_interaction.py` | 1,846 | — | — | MEDIUM | -| 14 | `routes/document_routes.py` | 1,717 | — | — | LOW | -| 15 | `routes/skills_routes.py` | 1,648 | — | — | LOW | - -**Heuristic**: Files > 2,000 lines with 20+ public symbols and many importers are the highest-risk splits. Files 1,000–2,000 lines are medium-risk if tightly coupled. - -### 2.2 Frontend - -| File | Lines | Concern | -|------|-------|---------| -| `static/style.css` | **36,653** | Entire app CSS in one file (tracked separately in #2617) | -| `static/js/document.js` | **9,776** | Single JS file for document functionality | -| `static/js/slashCommands.js` | 6,498 | | -| `static/js/settings.js` | 5,266 | | -| `static/js/emailLibrary.js` | 5,217 | | -| `static/js/notes.js` | 5,124 | | -| `static/js/chat.js` | 4,985 | | -| `static/app.js` | 4,090 | | - -**Note**: Frontend modularization is tracked separately in #2617 (CSS) and is not the focus of this Phase 0 inventory. Frontend is listed here for completeness but follow-up slices should target Python backend boundaries first. - ---- - -## 3. Import Dependency Graph - -### 3.1 Who Depends on `core/database.py` - -**102 files** import from `core.database` — this is the most depended-upon module: - -- All route handlers (`routes/*.py`) -- Most `src/*.py` files -- `core/session_manager.py`, `core/auth.py` -- Multiple test files - -**Implication**: Any split of `core/database.py` is the highest-risk refactor. It should be tackled **last**, never first. - -### 3.2 Who Depends on `src/tool_implementations.py` - -**17 files** import from `src.tool_implementations`: -- `src/agent_loop.py`, `src/builtin_actions.py`, `src/tool_index.py` -- `src/task_scheduler.py`, `src/tool_policy.py` -- Various tests - -### 3.3 Who Depends on `src/agent_loop.py` - -**22 files** import from `src.agent_loop`: - -- `src/tool_policy.py`, `src/teacher_escalation.py`, `src/bg_monitor.py` -- `src/task_scheduler.py` -- Multiple test files - -### 3.4 Cross-Layer Import Violations - -**`src/` importing from `routes/`** (backwards dependency — domain logic depending on HTTP layer): - -``` -src/tool_implementations.py ──→ routes/calendar_routes.py -src/tool_implementations.py ──→ routes/cookbook_helpers.py -src/tool_implementations.py ──→ routes/email_helpers.py -src/tool_implementations.py ──→ routes/email_pollers.py -src/tool_implementations.py ──→ routes/email_routes.py -src/tool_implementations.py ──→ routes/model_routes.py -src/tool_implementations.py ──→ routes/note_routes.py -src/tool_implementations.py ──→ routes/prefs_routes.py -``` - -> These are **runtime imports** (inside function bodies, not at module top), which mitigates circular import risk but indicates fuzzy layer boundaries. Function-level inline imports from the HTTP layer into business logic are a code smell. - -**Import counts (top-level)**: -| Direction | Count | Notes | -|-----------|-------|-------| -| `routes/` → `src/` | **374** | Expected: HTTP handlers call domain logic | -| `routes/` → `core/` | **126** | Expected: handlers access DB models | -| `src/` → `routes/` | **31** | **Unexpected**: domain logic reaching into HTTP layer (direct grep of import lines referencing `routes/`) | -| `src/` → `core/` | **106** | Acceptable but could be reduced with a data-access layer | - -> **How the metrics in this document are computed** — recompute against current `dev` before treating any count as authoritative (the tree drifts; these numbers are a snapshot, not a live value): -> - `src/` flat `.py` files: `find src -maxdepth 1 -name '*.py' | wc -l` -> - `tests/` test files: `find tests -name 'test_*.py' | wc -l` -> - `core.database` importers: `grep -rlE '(from|import) +core\.database' --include='*.py' . | grep -v core/database.py | wc -l` -> - `src.agent_loop` importers: `grep -rlE '(from|import) +src\.agent_loop' --include='*.py' . | grep -v src/agent_loop.py | wc -l` -> - Cross-layer import lines: `grep -rhE '(from|import) +<pkg>' --include='*.py' <dir>/ | wc -l` (e.g. `(from|import) +routes` over `src/`) - ---- - -## 4. Route Ownership Map - -Routes can be grouped into logical feature domains. Current flat structure obscures these boundaries: - -| Domain | Route Files | Total Lines | Review Complexity | -|--------|-------------|-------------|-------------------| -| **Email** | `email_routes.py`, `email_helpers.py`, `email_pollers.py` | 5,936 | HIGH — most complex domain | -| **Chat / Agent** | `chat_routes.py`, `chat_helpers.py`, `shell_routes.py`, `codex_routes.py`, `skills_routes.py` | 6,365 | HIGH — core interaction surface | -| **Cookbook** | `cookbook_routes.py`, `cookbook_helpers.py`, `cookbook_output.py` | 4,110 | MEDIUM | -| **Model / LLM** | `model_routes.py`, `assistant_routes.py`, `copilot_routes.py` | 2,764 | MEDIUM | -| **Calendar / Contacts** | `calendar_routes.py`, `contacts_routes.py` | 2,336 | MEDIUM | -| **Documents** | `document_routes.py`, `document_helpers.py` | 1,954 | LOW | -| **Auth** | `auth_routes.py`, `api_token_routes.py`, `device_flow.py` | 1,171 | LOW | -| **Tasks** | `task_routes.py` (standalone) | 1,157 | LOW | -| **Session** | `session_routes.py` (standalone) | 1,287 | LOW | -| **Gallery** | `gallery_routes.py`, `gallery_helpers.py` | 1,896 | LOW | -| **Memory** | `memory_routes.py` | — | LOW | -| **Research** | `research_routes.py` | — | LOW | -| **MCP** | `mcp_routes.py` | — | LOW | -| **Notes** | `note_routes.py` | — | LOW | -| **Other** | `prefs_routes.py`, `upload_routes.py`, `vault_routes.py`, `webhook_routes.py`, `workspace_routes.py`, `search_routes.py`, `history_routes.py`, `hwfit_routes.py`, `preset_routes.py`, `signature_routes.py`, `backup_routes.py`, `cleanup_routes.py`, `diagnostics_routes.py`, `embedding_routes.py`, `emoji_routes.py`, `font_routes.py`, `stt_routes.py`, `tts_routes.py`, `compare_routes.py`, `personal_routes.py`, `editor_draft_routes.py`, `admin_wipe_routes.py`, `chatgpt_subscription_routes.py` | 2,000+ | LOW individual, HIGH cumulative | - ---- - -## 5. Tool Registry & Implementation Boundaries - -### 5.1 Current Tool Architecture - -| Component | File | Lines | Role | -|-----------|------|-------|------| -| Tool schemas | `src/tool_schemas.py` | 1,392 | JSON Schema tool definitions (Duck-TypedDict) | -| Tool index | `src/tool_index.py` | 542 | RAG-based tool retrieval from ChromaDB | -| Tool implementations | `src/tool_implementations.py` | 4,032 | 33 `do_*` functions — all tool execution logic | -| Tool security | `src/tool_security.py` | — | Owner-scoped tool blocking | -| Tool policy | `src/tool_policy.py` | — | Guide-only directive, plan-mode disabled tools | -| Tool utils | `src/tool_utils.py` | — | Shared tool helpers | - -### 5.2 Tool Implementation Categories - -The 33 `do_*` functions in `tool_implementations.py` fall into natural domain groups — the basis for slice 1's split in §6.2: - -| Category | `do_*` functions | Count | -|----------|------------------|-------| -| **System / config** | `do_manage_skills`, `do_manage_tasks`, `do_manage_endpoints`, `do_manage_mcp`, `do_manage_webhooks`, `do_manage_tokens`, `do_manage_settings`, `do_api_call`, `do_app_api` | 9 | -| **Cookbook / model serving** | `do_download_model`, `do_serve_model`, `do_list_served_models`, `do_stop_served_model`, `do_tail_serve_output`, `do_list_downloads`, `do_cancel_download`, `do_search_hf_models`, `do_adopt_served_model`, `do_list_cookbook_servers`, `do_list_serve_presets`, `do_serve_preset`, `do_list_cached_models` | 13 | -| **Notes** | `do_manage_notes` | 1 | -| **Calendar** | `do_manage_calendar` | 1 | -| **Search** | `do_search_chats` | 1 | -| **Research** | `do_manage_research`, `do_trigger_research` | 2 | -| **Contacts** | `do_resolve_contact`, `do_manage_contact` | 2 | -| **Vault** | `do_vault_search`, `do_vault_get`, `do_vault_unlock` | 3 | -| **Image** | `do_edit_image` | 1 | -| | **Total** | **33** | - -> Low-level tools (filesystem, subprocess, web fetch, document parsing) live in `src/agent_tools/`, **not** in `tool_implementations.py` — out of scope for this split. - ---- - -## 6. Risk Assessment & Candidate Slice Ranking - -> **Candidate proposals, not a committed plan.** The rankings, package shapes (e.g. `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/`), split ordering, and route-grouping strategy below are **options for maintainer discussion**. Per #4082/#4071, slice ownership and order are settled by maintainers before any follow-up PR. §1–§3 above are the factual current-state inventory. - -### 6.1 Risk Scale - -| Level | Criteria | -|-------|----------| -| **LOW** | File has ≤3 importers AND ≤500 lines, OR is a pure refactor with clear boundaries | -| **MEDIUM** | File has 4–15 importers OR 500–1,500 lines | -| **HIGH** | File has 16+ importers OR >2,000 lines, OR has cross-layer import violations | - -### 6.2 Ranked Split Candidates - -| Priority | Target | Risk | Rationale | -|----------|--------|------|-----------| -| **1** | `src/tool_implementations.py` → `src/tools/*.py` | **MEDIUM** | 4,032 lines → ~10 files by tool category. Already has natural boundaries. 17 importers, tracked in #3629. Use `__init__.py` shim to keep existing imports working. | -| **2** | `routes/` → domain subdirectories (one domain per PR) | **MEDIUM** | 54 flat files. Done **one domain at a time** (e.g. a standalone PR for the email domain, then chat, …), not a broad reorganization — route modules carry helper imports, registration assumptions, and test import paths. | -| **3** | `src/agent_loop.py` → `src/agent/loop.py` + submodules | **MEDIUM-HIGH** | 2,961 lines, 24 functions. Can extract prompt building, classification, verification, and runaway detection. Tracked in #3266. | -| **4** | `src/` → `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/` | **MEDIUM** | Structural reorganization. Split flat `src/` into layered packages. Must come after routes and tools are stable. | -| **5** | `routes/email_*.py` consolidation | **LOW** | Already grouped by filename prefix. Low-risk cleanup within the email domain. | -| **6** | `core/database.py` → `src/infra/database/models/*.py` | **HIGH** | 28 classes, 102 importers. Highest-risk split. Must be **last** in any sequence. Requires careful import shim strategy. | -| **7** | Frontend CSS modularization | **MEDIUM** | 36,653 lines. Tracked in #2617. Separate timeline from backend work. | -| **8** | Frontend JS modularization | **MEDIUM** | 9,776 lines in `document.js`. Introduce ES modules at minimum. | - -### 6.3 Candidate First 3 Behavior-Preserving Slices - -**Slice 1: Split `tool_implementations.py`** (Lowest-risk high-impact) - -- Create `src/tools/` package with one file per tool category -- Add `src/tools/__init__.py` re-exporting all symbols with current names -- Update 17 importers to use new paths (can be deferred via shim) -- Validation: `python -m pytest tests/ -x -q` + manual smoke test of tool execution -- Reference: #3629 - -**Slice 2: Group `routes/` by domain** (one domain per PR, not a broad sweep) - -Route modules carry helper imports, router registration assumptions, and test import paths, so this must be done **one domain at a time** rather than as a single reorganization PR. Example sequence (each its own PR): - -- PR 2a: move the **email** domain (`email_routes.py`, `email_helpers.py`, `email_pollers.py`) → `routes/email/` + shim -- PR 2b: move the **chat/agent** domain → `routes/chat/` + shim -- PR 2c: move the **cookbook** domain → `routes/cookbook/` + shim -- …and so on per domain from §4 - -Each PR: add `__init__.py` re-exporting old names, update `app.py` router imports, validation `python app.py` starts clean. **No behavior change** — pure file reorganization. - -**Slice 3: Extract `agent_loop.py` submodules** (Improve reviewability) - -- Move prompt assembly → `src/agent/prompt.py` -- Move request classification → `src/agent/classifier.py` -- Move sub-agent verification → `src/agent/verifier.py` -- Move runaway detection → `src/agent/runaway.py` -- Move context management → `src/agent/context.py` -- Keep `src/agent/loop.py` as the main orchestration module -- Validation: `python -m pytest tests/test_agent_loop.py tests/test_loop_breaker_runaway.py -v` - ---- - -## 7. Safety Guardrails for Follow-Up Work - -Per maintainer guidance in #4082 and #4071: - -- [ ] **One domain/slice per PR** — never mix multiple reorganizations -- [ ] **No behavior changes** mixed with file moves — pure reorganization only -- [ ] **Keep compatibility shims** — `__init__.py` re-exports for all existing import paths -- [ ] **Add or identify focused tests** before risky splits -- [ ] **Do not start with `core/database.py`** or broad route movement unless this inventory shows a safe boundary -- [ ] **Prefer small, reviewable slices** over large restructures -- [ ] **No packaging/runtime/tooling migration** mixed into file moves -- [ ] **No frontend framework migration** inside this stabilization lane -- [ ] **Validate with `python -m compileall`** — every PR must pass CI checks -- [ ] **Validate with `pytest`** — run the full test suite before opening each PR - ---- - -## 8. Validation Commands - -Each follow-up PR should be verifiable with these commands before submission: - -```bash -# Syntax check — must pass with zero errors -python -m compileall src/ routes/ core/ conf/ - -# Full test suite — must match baseline pass rate -python -m pytest tests/ -x -q - -# Import shim verification — existing import paths must still work -python -c "from src.tool_implementations import do_search_chats; print('OK')" - -# App startup smoke test (if backend touched) -timeout 5 python app.py 2>&1 | head -5 || true -``` - ---- - -## 9. Open Questions - -1. Is `#2538` (specs ground truth) the canonical behavior map baseline, and should this inventory be kept in sync with those specs once merged? -2. Should route grouping follow the domain map proposed here, or is there a different taxonomy preferred by maintainers? -3. For the `tool_implementations.py` split (#3629), is the tool categorization in §5.2 acceptable, or should it follow a different grouping? -4. Should compatibility shims (`__init__.py`) be temporary (removed in a follow-up wave) or permanent? -5. Should an ADR (Architecture Decision Record) document be started to track decisions made during this process? - ---- - -## 10. Future Direction (NOT current state) - -The following are **future refactor targets** (candidate directions **pending maintainer agreement**, not committed), recorded here so this inventory does not imply they exist today. None of them are present in the current `dev` tree: - -- `main.py` — proposed rename of the `app.py` entrypoint. Today the app boots via `app.py`. -- `src/agent/` — proposed package to hold `agent_loop.py` submodules (prompt/classifier/verifier/runaway/context). Today `agent_loop.py` is a single flat file in `src/`. -- `src/infra/`, `src/domain/`, `src/pkg/`, `src/api/` — proposed layered reorganization of the flat `src/` directory (slice 4 in §6). - -These become real only when the corresponding slices land. - ---- - -## Appendix A: File Listing - -### `src/` (95 files — 61 shown; run `ls src/*.py` for the full list) - -``` -agent_loop.py tool_implementations.py tool_schemas.py -tool_index.py tool_security.py tool_policy.py -tool_utils.py builtin_actions.py task_scheduler.py -llm_core.py model_context.py model_discovery.py -session_search.py context_budget.py context_compactor.py -ai_interaction.py action_intents.py agent_runs.py -app_helpers.py app_initializer.py config.py -database.py memory.py memory_provider.py -secret_storage.py prompt_security.py url_security.py -url_safety.py rate_limiter.py cleanup_service.py -readiness.py service_health.py exceptions.py -request_models.py assistant_log.py bg_monitor.py -builtin_mcp.py chat_helpers.py chroma_client.py -document_processor.py embedding_lanes.py deep_research.py -research_handler.py research_utils.py personal_docs.py -rag_manager.py rag_singleton.py topic_analyzer.py -visual_report.py youtube_handler.py pdf_forms.py -pdf_form_doc.py pdf_runtime.py caldav_writeback.py -email_thread_parser.py text_helpers.py user_time.py -teacher_escalation.py cookbook_serve_lifecycle.py -chatgpt_subscription.py mcp_manager.py -``` - -### `routes/` (54 files) - -``` -__init__.py _validators.py -auth_routes.py api_token_routes.py device_flow.py -chat_routes.py chat_helpers.py shell_routes.py -codex_routes.py skills_routes.py -email_routes.py email_helpers.py email_pollers.py -cookbook_routes.py cookbook_helpers.py cookbook_output.py -model_routes.py assistant_routes.py copilot_routes.py -calendar_routes.py contacts_routes.py -document_routes.py document_helpers.py -gallery_routes.py gallery_helpers.py -task_routes.py session_routes.py -note_routes.py memory_routes.py research_routes.py -mcp_routes.py search_routes.py history_routes.py -webhook_routes.py workspace_routes.py upload_routes.py -vault_routes.py prefs_routes.py preset_routes.py -signature_routes.py personal_routes.py hwfit_routes.py -backup_routes.py cleanup_routes.py diagnostics_routes.py -embedding_routes.py emoji_routes.py font_routes.py -stt_routes.py tts_routes.py compare_routes.py -editor_draft_routes.py chatgpt_subscription_routes.py admin_wipe_routes.py -``` - -### `core/` (10 files) - -``` -__init__.py constants.py database.py models.py -auth.py middleware.py session_manager.py exceptions.py -atomic_io.py platform_compat.py -``` - ---- - -## Appendix B: Key Import Relationships - -``` -core/database.py ←── 102 importers (routes/*, src/*, core/*, tests/*) - ↑ - ├── routes/auth_routes.py - ├── routes/email_routes.py - ├── src/builtin_actions.py - ├── src/task_scheduler.py - ├── src/tool_implementations.py (inline) - └── ...97 more - -src/tool_implementations.py ←── 17 importers - ↑ - ├── src/agent_loop.py - ├── src/builtin_actions.py - ├── src/tool_index.py - ├── src/task_scheduler.py - ├── src/tool_policy.py - └── ...12 more (mostly tests) - -src/agent_loop.py ←── 22 importers - ↑ - ├── src/tool_policy.py - ├── src/teacher_escalation.py - ├── src/bg_monitor.py - ├── src/task_scheduler.py - └── 18 more (incl. tests) -``` +# Architecture runtime inventory disposition + +> [!NOTE] +> The dated architecture runtime inventory has been classified and transferred +> to [`discovery/architecture-runtime-inventory.md`](../discovery/architecture-runtime-inventory.md) +> after verification of its stable facts. +> +> Stable runtime structure and subsystem boundaries were transferred to the +> proposed canonical destination, +> [`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md), for maintainer review. + +## Classification + +| Content | Disposition | +|---|---| +| Stable runtime structure | Proposed canonical architecture document | +| Stable subsystem boundaries | Proposed canonical architecture document | +| Frontend module organization | `static/js/MODULE_SUMMARY.md` | +| Counts, rankings, and generated metrics | Non-canonical discovery inventory | +| Investigation context and unresolved questions | Non-canonical discovery material | +| Refactor options and prioritization | Issues, Plane, or discovery material | + +No dated metrics, prioritization, or investigation findings remain in this +specification path. + +## Review status + +- Findings and scope have been validated. +- Transfer verification has completed. +- The final canonical status of `docs/ARCHITECTURE.md` requires maintainer review. +- No implementation behaviour, test taxonomy, or runtime files were changed. diff --git a/tests/LAYOUT_INVENTORY.md b/tests/LAYOUT_INVENTORY.md index 34c719774..4a14961be 100644 --- a/tests/LAYOUT_INVENTORY.md +++ b/tests/LAYOUT_INVENTORY.md @@ -1,202 +1,152 @@ -# Test Layout Inventory +# Test layout inventory + +> [!NOTE] +> This document is a completed implementation snapshot for the first low-risk +> test-directory reorganization. It does not propose additional file moves and +> is not the canonical definition of the current test taxonomy. + +- **Original inventory:** #3712 +- **Parent tracker:** #2523 +- **Implemented by:** #3842 +- **Status:** completed historical snapshot ## Purpose -Inventory for the first low-risk split of the flat `tests/` directory -(issue #3712, parent #2523). This document only records *what* should move -first and *why*; it moves nothing. The actual move is a separate, mechanical -PR that relocates the listed files verbatim and changes no test content. +This document preserves the reasoning, scope, and validation contract for the +first mechanical reorganization of the formerly flat `tests/` directory. -The target layout and category definitions come from -[`TESTING_STANDARD.md`](./TESTING_STANDARD.md); the collection-time markers -come from [`_taxonomy.py`](./_taxonomy.py), which classifies by **filename -tokens only** (paths are ignored, except the `tests/helpers/` rule). A file -keeps its `area_*`/`sub_*` markers when moved into a subdirectory, and -`conftest.py` discovers marker names recursively (`rglob`), so a move does not -disturb marker registration or focused selection. +The original change moved 28 tests classified as `area_cli` into `tests/cli/` +without changing their assertions, helpers, taxonomy, or runtime behaviour. -## Current low-risk candidate groups +Current testing guidance and classification rules are maintained in: -Groups whose tests need no route/app setup and no real DB/session setup: +- [`README.md`](./README.md) +- [`TESTING_STANDARD.md`](./TESTING_STANDARD.md) +- [`_taxonomy.py`](./_taxonomy.py) -1. **CLI / script tests** (`area_cli`, 28 files) - load `scripts/` entry - points via `tests.helpers.cli_loader.load_script`; DB access is stubbed - with `tests.helpers.db_stubs` (`SessionLocal` is a plain stub attribute). - No `TestClient`, no FastAPI app import, no SQLite files. -2. **Helper self-tests** (`area_helpers`) - e.g. `test_helpers_import_state.py`, - `test_db_stubs_helper.py`. Safe but tiny (two files), and they test the - shared helpers from the #3685 audit (merged) that the rest of the suite - depends on; little payoff as a first slice. -3. **Pure unit / parsing tests** (`area_unit`) - `*_nonstring.py`, - `*_nondict.py`, parsing tests. Large and heterogeneous; some touch - provider/session modules, so the boundary is less crisp. -4. **Static checks** - e.g. `test_readme_ascii_fenced.py`, - `test_docs_no_orphan_images.py`. Safe but tiny and `uncategorized` in the - taxonomy, so a move buys little and matches no existing marker. +The current repository and test suite remain authoritative when this snapshot +and the implementation differ. -Not candidates for the first move (per #3712 guidance): security/owner-scope -tests, route/API tests, DB/session-heavy tests, auth/session concurrency -tests, and the taxonomy/runner infrastructure tests that changed recently -(#3491, #3556, #3659, #3711). +## Why CLI tests were selected -## Recommended first move +The CLI and script tests were selected as the first low-risk group because they +formed a narrow, mechanically identifiable boundary: -**CLI / script tests → `tests/cli/`** +- script entry points were loaded through shared CLI helpers; +- database access used controlled stubs rather than persistent sessions; +- the tests did not require FastAPI application or route setup; +- the tests did not require a real database; +- taxonomy classification could be derived consistently from filenames; +- absolute helper imports remained valid after relocation. -Why this group over the alternatives: +This reduced the risk relative to route, security, session, provider, and +heterogeneous unit-test groups. -- Lowest coupling: every file imports only the script under test (via - `cli_loader`) plus `tests.helpers` stubs - no app, no routes, no real DB. -- Crisp, machine-checkable boundary: the set is exactly the files classified - `area_cli` by `_taxonomy.py`, so before/after selection counts can be - compared mechanically. -- Already the planned target dir for this category in `TESTING_STANDARD.md` - (`tests/cli/`). -- Absolute imports (`from tests.helpers...`) and unique basenames mean no - import-order or module-name collisions after the move. -- Lower risk than helper self-tests (tiny group, little payoff), unit tests - (fuzzy boundary), or anything security/route/session-shaped. +## Completed implementation -## Files included in the first move +Repository history records the implementation as: -The 28 files classified `area_cli` (verified against `_taxonomy.py`): +- `83af3ca test: move area_cli tests into cli directory (#3842)` -Note: this inventory was refreshed against current `dev` after `tests/test_research_cli_status.py` was added to the `area_cli` set. +The following 28 files were included in that completed move: -- `tests/test_calendar_cli_name.py` -- `tests/test_contacts_cli_rows.py` -- `tests/test_cookbook_cli_state.py` -- `tests/test_docs_cli_content_length.py` -- `tests/test_gallery_cli_album_count.py` -- `tests/test_gallery_cli_preview.py` -- `tests/test_logs_cli_resolve_nonstring.py` -- `tests/test_mail_cli_read_empty_fetch.py` -- `tests/test_mail_cli_recipients.py` -- `tests/test_mcp_cli_env_serialize.py` -- `tests/test_mcp_cli_json.py` -- `tests/test_memory_cli_rows.py` -- `tests/test_notes_cli_items.py` -- `tests/test_personal_cli_rows.py` -- `tests/test_preset_cli_invalid_entries.py` -- `tests/test_preset_cli_set_corrupt_entry.py` -- `tests/test_preset_cli_store.py` -- `tests/test_research_cli_preview.py` -- `tests/test_research_cli_status_filter.py` -- `tests/test_research_cli_status.py` -- `tests/test_research_cli_store.py` -- `tests/test_sessions_cli.py` -- `tests/test_signature_cli_export.py` -- `tests/test_skills_cli_preview.py` -- `tests/test_skills_cli_rows.py` -- `tests/test_tasks_cli_preview.py` -- `tests/test_theme_cli_store.py` -- `tests/test_webhook_cli_mask.py` +- `tests/cli/test_calendar_cli_name.py` +- `tests/cli/test_contacts_cli_rows.py` +- `tests/cli/test_cookbook_cli_state.py` +- `tests/cli/test_docs_cli_content_length.py` +- `tests/cli/test_gallery_cli_album_count.py` +- `tests/cli/test_gallery_cli_preview.py` +- `tests/cli/test_logs_cli_resolve_nonstring.py` +- `tests/cli/test_mail_cli_read_empty_fetch.py` +- `tests/cli/test_mail_cli_recipients.py` +- `tests/cli/test_mcp_cli_env_serialize.py` +- `tests/cli/test_mcp_cli_json.py` +- `tests/cli/test_memory_cli_rows.py` +- `tests/cli/test_notes_cli_items.py` +- `tests/cli/test_personal_cli_rows.py` +- `tests/cli/test_preset_cli_invalid_entries.py` +- `tests/cli/test_preset_cli_set_corrupt_entry.py` +- `tests/cli/test_preset_cli_store.py` +- `tests/cli/test_research_cli_preview.py` +- `tests/cli/test_research_cli_status.py` +- `tests/cli/test_research_cli_status_filter.py` +- `tests/cli/test_research_cli_store.py` +- `tests/cli/test_sessions_cli.py` +- `tests/cli/test_signature_cli_export.py` +- `tests/cli/test_skills_cli_preview.py` +- `tests/cli/test_skills_cli_rows.py` +- `tests/cli/test_tasks_cli_preview.py` +- `tests/cli/test_theme_cli_store.py` +- `tests/cli/test_webhook_cli_mask.py` -## Files intentionally excluded +## Original exclusions -- `tests/test_backup_cli_security.py` - classifies as `area_security` - (security outranks cli in the taxonomy); moving it into `tests/cli/` would - make the directory disagree with its marker. It belongs with the security - group in a later phase. -- `tests/test_run_focus.py`, `tests/test_taxonomy.py` - taxonomy/runner - infrastructure tests, recently changed (#3556, #3659); they also pin - flat-layout paths (e.g. `tests/test_auth_config_lock_concurrency.py` in - `test_run_focus.py`), so they stay put. -- Script-like but `uncategorized` files - `test_pr_blocker_audit.py`, - `test_update_database_script.py`, `test_windows_update_script.py`, - `test_setup_admin_user.py`, `test_amd_gpu_check_args.py`, `test_hwfit_*.py`. - They exercise `scripts/` too, but moving them would make `tests/cli/` - diverge from the `area_cli` marker set. Reclassify or move them in a later, - separate slice. -- Everything else (security, routes, services, unit, js, helpers) - out of - scope for the first move by design. +The original inventory deliberately excluded: -## How this was verified +- `tests/test_backup_cli_security.py`, because its security classification + outranked the CLI token in its filename; +- taxonomy and focused-runner infrastructure, including + `tests/test_taxonomy.py` and `tests/test_run_focus.py`; +- script-oriented tests not classified as `area_cli`; +- route, service, security, session, JavaScript, helper, and broad unit-test + groups. -Read-only checks, run from the repo root on this branch. Note the real API is -`classify_test_path` (there is no `classify_test_file`). +These exclusions kept the implementation mechanical and prevented it from +mixing taxonomy changes, assertion changes, helper extraction, or unrelated +test-suite restructuring. + +## Current repository observation + +The current taxonomy classifies 30 files as `area_cli`: + +| Location | Files | +|---|---:| +| `tests/cli/` | 28 | +| Flat `tests/` directory | 2 | + +The two currently classified CLI tests outside `tests/cli/` are: + +- `tests/test_calendar_cli_overlap.py` +- `tests/test_memory_cli_add_nondict.py` + +These files were not part of the original 28-file implementation. + +Their presence is recorded solely to keep this snapshot accurate. This +documentation update does not recommend moving them, changing their taxonomy, +or opening implementation work. Any future test-layout change requires separate +maintainer agreement, scope, and validation. + +## Validation contract preserved by the original move + +The completed implementation was designed to preserve: + +- identical `area_cli` selection before and after relocation; +- successful direct execution of the relocated tests; +- successful whole-suite collection; +- unchanged taxonomy and focused-runner behaviour; +- no stale references to the former flat paths; +- no production-code changes. + +Relevant current verification commands include: ```bash -# Compute the area_cli set and confirm test_backup_cli_security.py is -# area_security. Expected: 28 files, then "security". -./venv/bin/python - <<'PY' -from pathlib import Path -from tests._taxonomy import classify_test_path - -cli = [p for p in sorted(Path("tests").glob("test_*.py")) - if classify_test_path(p).area == "cli"] -print(len(cli)) -for p in cli: - print(p) -print(classify_test_path("tests/test_backup_cli_security.py").area) -PY - -# Coupling check across the CLI files. Expected: the only hits are -# "SessionLocal" as stub attribute names passed to tests.helpers.db_stubs; -# no TestClient, FastAPI, create_app, sqlite, or dependency_overrides. -rg -n "TestClient|FastAPI|create_app|SessionLocal|sqlite|dependency_overrides" \ - tests/test_*cli*.py tests/test_sessions_cli.py - -# Hard-coded flat paths to the exact CLI files outside tests/. Expected: no matches. -./venv/bin/python - <<'PY2' > /tmp/area_cli_paths.txt -from pathlib import Path -from tests._taxonomy import classify_test_path - -for path in sorted(Path("tests").glob("test_*.py")): - if classify_test_path(path).area == "cli": - print(path) -PY2 - -rg -n -F -f /tmp/area_cli_paths.txt .github scripts docs \ - tests/README.md tests/TESTING_STANDARD.md pyproject.toml 2>/dev/null || true +venv/bin/python tests/run_focus.py --dry-run --area cli +venv/bin/python -m pytest -m area_cli -q +venv/bin/python -m pytest tests/cli/ -q +venv/bin/python -m pytest tests/test_taxonomy.py tests/test_run_focus.py -q +venv/bin/python -m pytest --collect-only -q ``` -Also checked by reading the code: `tests/conftest.py` registers sub-markers -from a recursive `rglob` scan, and `tests/_taxonomy.py` classifies by filename -tokens only (plus the `tests/helpers/` directory rule), so the markers of the -28 files do not change when they move into `tests/cli/`. - -## Validation for the future move PR - -Run with the project venv (`./venv/bin/python`); system `python3` may miss -pinned deps. Before the move, record the baseline; after, compare: - -```bash -# Selection must match the 28 files before and after the move. -./venv/bin/python tests/run_focus.py --dry-run --area cli -./venv/bin/python -m pytest -m area_cli -q - -# Moved files pass when targeted directly. -./venv/bin/python -m pytest tests/cli/ -q - -# Whole-suite collection still succeeds (catches import/path breakage). -./venv/bin/python -m pytest --collect-only -q - -# Taxonomy/runner infrastructure is unaffected. -./venv/bin/python -m pytest tests/test_taxonomy.py tests/test_run_focus.py -q - -# No stale flat-path references to the moved files. Expected: no matches -# outside tests/cli/ itself. -./venv/bin/python - <<'PY2' > /tmp/area_cli_paths.txt -from pathlib import Path -from tests._taxonomy import classify_test_path - -for path in sorted(Path("tests").glob("test_*.py")): - if classify_test_path(path).area == "cli": - print(path) -PY2 - -rg -n -F -f /tmp/area_cli_paths.txt .github scripts docs \ - tests/README.md tests/TESTING_STANDARD.md pyproject.toml 2>/dev/null || true -``` - -Pass criteria: identical test counts for `-m area_cli` before/after, zero -collection errors, and no changes outside the moved files. +These commands describe validation coverage. This snapshot does not claim that +they were executed as part of the present documentation update. ## Non-goals -- No file moves, renames, or deletions in this PR. -- No changes to `conftest.py`, `_taxonomy.py`, `run_focus.py`, helpers, - markers, CI workflows, or production code. -- No recommendation to split the whole suite at once; later groups get their - own inventory-then-move slices. +This document does not: + +- propose moving or renaming tests; +- redefine taxonomy classifications; +- change test behaviour or assertions; +- prescribe the next test-suite refactor; +- replace the testing standard; +- create implementation work from the two current residual files. diff --git a/tests/OVERSIZED_TEST_SPLIT_PLAN.md b/tests/OVERSIZED_TEST_SPLIT_PLAN.md index 4f81080a4..481424e58 100644 --- a/tests/OVERSIZED_TEST_SPLIT_PLAN.md +++ b/tests/OVERSIZED_TEST_SPLIT_PLAN.md @@ -1,5 +1,9 @@ # Oversized Test File Split Plan +> [!NOTE] +> This is generated planning material based on the current test tree. +> It does not define runtime behaviour or approve any test-file split. + ## Purpose This document plans future oversized test-file splits using current repo data. @@ -26,8 +30,8 @@ Future split PRs must still inspect each file manually before editing. ## Current summary -- test files scanned: 583 -- collected pytest items counted: 3586 +- test files scanned: 746 +- collected pytest items counted: 4764 - large-file threshold: 300 lines - large-collected threshold: 20 collected items @@ -35,123 +39,123 @@ Area distribution: | Value | Files | |---|---:| -| cli | 28 | +| cli | 30 | | helpers | 1 | -| js | 39 | -| routes | 23 | -| security | 77 | -| services | 144 | -| uncategorized | 234 | -| unit | 37 | +| js | 49 | +| routes | 35 | +| security | 88 | +| services | 197 | +| uncategorized | 306 | +| unit | 40 | Sub-area distribution: | Value | Files | |---|---:| -| api | 6 | +| api | 8 | | atomic | 3 | -| auth | 9 | -| calendar | 10 | -| cli | 28 | -| confinement | 7 | -| cookbook | 13 | -| document | 11 | -| email | 12 | -| embedding | 3 | -| gallery | 5 | +| auth | 11 | +| calendar | 12 | +| cli | 30 | +| confinement | 8 | +| cookbook | 20 | +| document | 13 | +| email | 23 | +| embedding | 6 | +| gallery | 6 | | history | 3 | -| js | 39 | -| llm | 16 | -| mcp | 8 | -| memory | 15 | +| js | 49 | +| llm | 21 | +| mcp | 12 | +| memory | 17 | | nondict | 7 | -| nonstring | 22 | -| owner | 14 | -| owner_scope | 23 | -| parse | 4 | -| provider | 6 | -| research | 16 | +| nonstring | 23 | +| owner | 15 | +| owner_scope | 27 | +| parse | 5 | +| provider | 16 | +| research | 17 | | route | 6 | -| routes | 9 | -| scheduler | 3 | +| routes | 19 | +| scheduler | 4 | | scope | 5 | | security | 9 | -| session | 16 | -| ssrf | 3 | -| webhook | 3 | +| session | 18 | +| ssrf | 6 | +| webhook | 4 | | xss | 5 | -Values below 2 files: 244 values covering 244 files. +Values below 2 files: 318 values covering 318 files. ## Top files by collected pytest items | File | Lines | Collected tests | Test defs | Test classes | Area | Sub-area | Signals | |---|---:|---:|---:|---:|---|---|---| -| `tests/test_model_routes.py` | 1778 | 139 | 116 | 10 | routes | routes | route/api, db/session, import-state, async/threading | -| `tests/test_security_regressions.py` | 1224 | 92 | 68 | 0 | security | security | route/api, db/session, import-state, security, filesystem, async/threading, ui/static | -| `tests/test_provider_classification.py` | 188 | 67 | 21 | 4 | services | provider | - | -| `tests/test_cookbook_helpers.py` | 912 | 65 | 65 | 0 | services | cookbook | route/api, filesystem, subprocess/script, async/threading, ui/static | -| `tests/test_shell_routes.py` | 481 | 63 | 48 | 8 | routes | routes | route/api, import-state, filesystem | +| `tests/test_model_routes.py` | 2099 | 163 | 134 | 10 | routes | routes | route/api, db/session, import-state, async/threading | +| `tests/test_security_regressions.py` | 1515 | 98 | 74 | 0 | security | security | route/api, db/session, import-state, security, filesystem, async/threading, ui/static | +| `tests/test_cookbook_helpers.py` | 958 | 69 | 69 | 0 | services | cookbook | route/api, filesystem, subprocess/script, async/threading, ui/static | +| `tests/test_shell_routes.py` | 538 | 69 | 53 | 9 | routes | routes | route/api, import-state, filesystem | +| `tests/test_endpoint_probing.py` | 497 | 61 | 36 | 7 | uncategorized | endpoint_probing | route/api, db/session, import-state | | `tests/test_pr_blocker_audit.py` | 964 | 58 | 58 | 0 | uncategorized | pr_blocker_audit | import-state, security, filesystem | -| `tests/test_provider_endpoints.py` | 241 | 58 | 18 | 1 | services | provider | subprocess/script | -| `tests/test_agent_loop.py` | 469 | 52 | 52 | 5 | uncategorized | agent_loop | db/session, import-state | -| `tests/test_service_health.py` | 472 | 47 | 42 | 0 | uncategorized | service_health | async/threading | -| `tests/test_run_focus.py` | 399 | 47 | 44 | 0 | uncategorized | run_focus | security, filesystem, subprocess/script, ui/static | -| `tests/test_llm_core_temperature.py` | 196 | 41 | 17 | 0 | services | llm | - | -| `tests/test_endpoint_probing.py` | 411 | 34 | 30 | 6 | uncategorized | endpoint_probing | route/api, db/session, import-state | +| `tests/test_research_routes_path_confinement.py` | 565 | 58 | 26 | 0 | security | confinement | route/api, security, filesystem, async/threading | +| `tests/test_run_focus.py` | 492 | 55 | 52 | 0 | uncategorized | run_focus | security, filesystem, subprocess/script, ui/static | +| `tests/test_agent_loop.py` | 500 | 54 | 54 | 5 | uncategorized | agent_loop | db/session, import-state | +| `tests/test_manage_mcp_command_allowlist.py` | 168 | 45 | 10 | 0 | services | mcp | db/session, subprocess/script, async/threading, ui/static | +| `tests/test_provider_classification.py` | 115 | 42 | 9 | 2 | services | provider | - | +| `tests/test_review_regressions.py` | 1429 | 41 | 41 | 0 | uncategorized | review_regressions | route/api, db/session, import-state, filesystem, async/threading | +| `tests/test_chat_helpers.py` | 582 | 37 | 24 | 0 | uncategorized | chat_helpers | route/api, filesystem, async/threading | +| `tests/test_model_context.py` | 314 | 36 | 34 | 4 | uncategorized | model_context | db/session, import-state | +| `tests/test_email_oauth.py` | 798 | 34 | 29 | 0 | services | email | route/api, db/session, security, async/threading | | `tests/test_llm_core_anthropic_temp_omit.py` | 94 | 32 | 6 | 0 | services | llm | db/session | -| `tests/test_chat_helpers.py` | 264 | 31 | 18 | 0 | uncategorized | chat_helpers | route/api | -| `tests/test_provider_detection.py` | 148 | 31 | 31 | 5 | services | provider | - | -| `tests/test_model_context.py` | 251 | 30 | 30 | 4 | uncategorized | model_context | db/session, import-state | -| `tests/test_endpoint_resolver.py` | 148 | 30 | 30 | 6 | uncategorized | endpoint_resolver | - | -| `tests/test_embedding_lanes.py` | 1104 | 29 | 29 | 0 | services | embedding | filesystem | +| `tests/test_redos_verdict_continuation.py` | 80 | 30 | 5 | 0 | uncategorized | redos_verdict_continuation | route/api | | `tests/test_upload_limits_centralized.py` | 110 | 29 | 5 | 0 | uncategorized | upload_limits_centralized | import-state, filesystem | -| `tests/test_email_oauth.py` | 580 | 28 | 25 | 0 | services | email | route/api, db/session, security, async/threading | -| `tests/test_review_regressions.py` | 930 | 26 | 26 | 0 | uncategorized | review_regressions | route/api, db/session, import-state, filesystem, async/threading | -| `tests/test_rename_user_owner_sync.py` | 686 | 26 | 26 | 0 | security | owner | route/api, db/session, import-state, filesystem, async/threading | +| `tests/test_cookbook_docker_access.py` | 339 | 28 | 13 | 0 | services | cookbook | route/api, filesystem, async/threading | +| `tests/test_rename_user_owner_sync.py` | 740 | 27 | 27 | 0 | security | owner | route/api, db/session, import-state, filesystem, async/threading | +| `tests/test_taxonomy.py` | 151 | 27 | 17 | 0 | uncategorized | taxonomy | security, ui/static | +| `tests/test_endpoint_resolver_urls.py` | 95 | 27 | 23 | 3 | uncategorized | endpoint_resolver_urls | - | | `tests/test_helpers_import_state.py` | 426 | 26 | 26 | 0 | helpers | helpers | route/api, db/session, import-state | -| `tests/test_taxonomy.py` | 145 | 26 | 16 | 0 | uncategorized | taxonomy | security, ui/static | -| `tests/test_tool_path_confinement.py` | 282 | 24 | 24 | 0 | security | confinement | import-state, filesystem, async/threading | -| `tests/test_copilot.py` | 170 | 23 | 16 | 0 | uncategorized | copilot | - | -| `tests/test_research_utils.py` | 97 | 23 | 23 | 2 | services | research | - | -| `tests/test_api_chat_security.py` | 401 | 22 | 8 | 0 | security | security | route/api, db/session, import-state, filesystem, async/threading | -| `tests/test_tool_support_heuristic.py` | 166 | 22 | 22 | 3 | uncategorized | tool_support_heuristic | - | -| `tests/test_platform_compat.py` | 318 | 21 | 21 | 0 | uncategorized | platform_compat | import-state, filesystem, subprocess/script | +| `tests/test_workspace_confine.py` | 527 | 25 | 25 | 0 | uncategorized | workspace_confine | route/api, filesystem, subprocess/script, async/threading | +| `tests/test_chat_route_tool_policy.py` | 344 | 25 | 23 | 0 | routes | route | filesystem, ui/static | +| `tests/test_tool_path_confinement.py` | 303 | 25 | 25 | 0 | security | confinement | import-state, filesystem, async/threading | +| `tests/test_copilot.py` | 182 | 25 | 18 | 0 | uncategorized | copilot | - | +| `tests/test_llm_core_temperature_reasoning.py` | 133 | 25 | 9 | 0 | services | llm | - | +| `tests/test_provider_endpoints_url_building.py` | 86 | 25 | 3 | 0 | services | provider | - | +| `tests/test_bg_job_tools.py` | 174 | 23 | 16 | 0 | uncategorized | bg_job_tools | filesystem, async/threading | ## Top files by physical line count | File | Lines | Collected tests | Test defs | Test classes | Area | Sub-area | Signals | |---|---:|---:|---:|---:|---|---|---| -| `tests/test_model_routes.py` | 1778 | 139 | 116 | 10 | routes | routes | route/api, db/session, import-state, async/threading | -| `tests/test_security_regressions.py` | 1224 | 92 | 68 | 0 | security | security | route/api, db/session, import-state, security, filesystem, async/threading, ui/static | -| `tests/test_embedding_lanes.py` | 1104 | 29 | 29 | 0 | services | embedding | filesystem | +| `tests/test_model_routes.py` | 2099 | 163 | 134 | 10 | routes | routes | route/api, db/session, import-state, async/threading | +| `tests/test_security_regressions.py` | 1515 | 98 | 74 | 0 | security | security | route/api, db/session, import-state, security, filesystem, async/threading, ui/static | +| `tests/test_review_regressions.py` | 1429 | 41 | 41 | 0 | uncategorized | review_regressions | route/api, db/session, import-state, filesystem, async/threading | | `tests/test_pr_blocker_audit.py` | 964 | 58 | 58 | 0 | uncategorized | pr_blocker_audit | import-state, security, filesystem | -| `tests/test_review_regressions.py` | 930 | 26 | 26 | 0 | uncategorized | review_regressions | route/api, db/session, import-state, filesystem, async/threading | -| `tests/test_cookbook_helpers.py` | 912 | 65 | 65 | 0 | services | cookbook | route/api, filesystem, subprocess/script, async/threading, ui/static | -| `tests/test_rename_user_owner_sync.py` | 686 | 26 | 26 | 0 | security | owner | route/api, db/session, import-state, filesystem, async/threading | -| `tests/test_email_oauth.py` | 580 | 28 | 25 | 0 | services | email | route/api, db/session, security, async/threading | +| `tests/test_cookbook_helpers.py` | 958 | 69 | 69 | 0 | services | cookbook | route/api, filesystem, subprocess/script, async/threading, ui/static | +| `tests/test_upload_handler_cleanup.py` | 831 | 13 | 13 | 0 | uncategorized | upload_handler_cleanup | route/api, db/session, filesystem, async/threading | +| `tests/test_email_oauth.py` | 798 | 34 | 29 | 0 | services | email | route/api, db/session, security, async/threading | +| `tests/test_rename_user_owner_sync.py` | 740 | 27 | 27 | 0 | security | owner | route/api, db/session, import-state, filesystem, async/threading | +| `tests/test_email_owner_scope.py` | 656 | 13 | 13 | 0 | security | owner_scope | route/api, db/session, filesystem, async/threading | +| `tests/test_model_capability_readers.py` | 646 | 20 | 20 | 0 | uncategorized | model_capability_readers | - | +| `tests/test_chat_helpers.py` | 582 | 37 | 24 | 0 | uncategorized | chat_helpers | route/api, filesystem, async/threading | | `tests/test_api_token_routes.py` | 578 | 17 | 17 | 0 | routes | api_routes | route/api, db/session, import-state, async/threading | -| `tests/test_shell_routes.py` | 481 | 63 | 48 | 8 | routes | routes | route/api, import-state, filesystem | -| `tests/test_email_owner_scope.py` | 474 | 9 | 9 | 0 | security | owner_scope | route/api, db/session, filesystem, async/threading | -| `tests/test_service_health.py` | 472 | 47 | 42 | 0 | uncategorized | service_health | async/threading | -| `tests/test_agent_loop.py` | 469 | 52 | 52 | 5 | uncategorized | agent_loop | db/session, import-state | +| `tests/test_research_routes_path_confinement.py` | 565 | 58 | 26 | 0 | security | confinement | route/api, security, filesystem, async/threading | +| `tests/test_shell_routes.py` | 538 | 69 | 53 | 9 | routes | routes | route/api, import-state, filesystem | +| `tests/test_workspace_confine.py` | 527 | 25 | 25 | 0 | uncategorized | workspace_confine | route/api, filesystem, subprocess/script, async/threading | +| `tests/test_agent_loop.py` | 500 | 54 | 54 | 5 | uncategorized | agent_loop | db/session, import-state | +| `tests/test_endpoint_probing.py` | 497 | 61 | 36 | 7 | uncategorized | endpoint_probing | route/api, db/session, import-state | +| `tests/test_run_focus.py` | 492 | 55 | 52 | 0 | uncategorized | run_focus | security, filesystem, subprocess/script, ui/static | +| `tests/test_tool_policy.py` | 475 | 18 | 18 | 0 | uncategorized | tool_policy | import-state, async/threading | | `tests/test_kv_cache_invalidation_2927.py` | 463 | 8 | 8 | 0 | uncategorized | kv_cache_invalidation_2927 | route/api, db/session, import-state, async/threading | +| `tests/test_email_test_connection_oauth.py` | 436 | 9 | 7 | 0 | services | email | route/api, db/session, async/threading | | `tests/test_helpers_import_state.py` | 426 | 26 | 26 | 0 | helpers | helpers | route/api, db/session, import-state | | `tests/test_endpoint_owner_scope_followup.py` | 414 | 11 | 11 | 0 | security | owner_scope | route/api, db/session, filesystem | -| `tests/test_endpoint_probing.py` | 411 | 34 | 30 | 6 | uncategorized | endpoint_probing | route/api, db/session, import-state | +| `tests/test_api_chat_security.py` | 404 | 22 | 8 | 0 | security | security | route/api, db/session, import-state, filesystem, async/threading | | `tests/test_imap_leak_fixes.py` | 404 | 15 | 15 | 0 | uncategorized | imap_leak_fixes | route/api, db/session, security, filesystem | | `tests/test_companion_readonly.py` | 402 | 17 | 17 | 0 | uncategorized | companion_readonly | db/session, import-state | -| `tests/test_api_chat_security.py` | 401 | 22 | 8 | 0 | security | security | route/api, db/session, import-state, filesystem, async/threading | | `tests/test_upload_handler_atomicity.py` | 401 | 9 | 9 | 0 | uncategorized | upload_handler_atomicity | filesystem, async/threading | -| `tests/test_run_focus.py` | 399 | 47 | 44 | 0 | uncategorized | run_focus | security, filesystem, subprocess/script, ui/static | -| `tests/test_auth_regressions.py` | 375 | 15 | 15 | 0 | security | auth | route/api, db/session, import-state, async/threading | -| `tests/test_calendar_owner_scope.py` | 345 | 7 | 7 | 0 | security | owner_scope | route/api, db/session, import-state, filesystem, async/threading, ui/static | -| `tests/test_null_owner_gates.py` | 342 | 20 | 20 | 0 | security | owner | route/api, db/session, import-state | -| `tests/test_agent_migration_manifest.py` | 340 | 15 | 15 | 0 | uncategorized | agent_migration_manifest | import-state, filesystem | -| `tests/test_calendar_recurrence.py` | 338 | 19 | 19 | 0 | services | calendar | - | -| `tests/test_tool_policy.py` | 330 | 13 | 13 | 0 | uncategorized | tool_policy | import-state, async/threading | -| `tests/test_workspace_confine.py` | 328 | 18 | 18 | 0 | uncategorized | workspace_confine | route/api, filesystem, subprocess/script, async/threading | -| `tests/test_diffusion_server_security.py` | 325 | 14 | 14 | 0 | security | security | route/api, import-state, security, filesystem, async/threading, ui/static | +| `tests/test_auth_regressions.py` | 370 | 15 | 15 | 0 | security | auth | route/api, db/session, import-state, async/threading | +| `tests/test_calendar_recurrence.py` | 356 | 20 | 20 | 0 | services | calendar | - | +| `tests/test_task_cookbook_admin_gate.py` | 350 | 10 | 10 | 0 | services | cookbook | route/api, db/session, import-state, filesystem, async/threading | ## Split planning candidates @@ -165,53 +169,69 @@ These are planning candidates only. A later split PR still needs a focused manua | File | Why included | Setup/risk signals | Suggested handling | |---|---|---|---| -| `tests/test_model_routes.py` | 1778 lines, 139 collected tests | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_security_regressions.py` | 1224 lines, 92 collected tests | route/api, db/session, import-state, security, filesystem, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_provider_classification.py` | 67 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_cookbook_helpers.py` | 912 lines, 65 collected tests | route/api, filesystem, subprocess/script, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_shell_routes.py` | 481 lines, 63 collected tests | route/api, import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_model_routes.py` | 2099 lines, 163 collected tests | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_security_regressions.py` | 1515 lines, 98 collected tests | route/api, db/session, import-state, security, filesystem, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_cookbook_helpers.py` | 958 lines, 69 collected tests | route/api, filesystem, subprocess/script, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_shell_routes.py` | 538 lines, 69 collected tests | route/api, import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_endpoint_probing.py` | 497 lines, 61 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_pr_blocker_audit.py` | 964 lines, 58 collected tests | import-state, security, filesystem | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_provider_endpoints.py` | 58 collected tests | subprocess/script | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_agent_loop.py` | 469 lines, 52 collected tests | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_service_health.py` | 472 lines, 47 collected tests | async/threading | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_run_focus.py` | 399 lines, 47 collected tests | security, filesystem, subprocess/script, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_llm_core_temperature.py` | 41 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_endpoint_probing.py` | 411 lines, 34 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_research_routes_path_confinement.py` | 565 lines, 58 collected tests | route/api, security, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_run_focus.py` | 492 lines, 55 collected tests | security, filesystem, subprocess/script, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_agent_loop.py` | 500 lines, 54 collected tests | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_manage_mcp_command_allowlist.py` | 45 collected tests | db/session, subprocess/script, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_provider_classification.py` | 42 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_review_regressions.py` | 1429 lines, 41 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_chat_helpers.py` | 582 lines, 37 collected tests | route/api, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_model_context.py` | 314 lines, 36 collected tests | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_email_oauth.py` | 798 lines, 34 collected tests | route/api, db/session, security, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_llm_core_anthropic_temp_omit.py` | 32 collected tests | db/session | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_chat_helpers.py` | 31 collected tests | route/api | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_provider_detection.py` | 31 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_model_context.py` | 30 collected tests | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_endpoint_resolver.py` | 30 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_embedding_lanes.py` | 1104 lines, 29 collected tests | filesystem | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_redos_verdict_continuation.py` | 30 collected tests | route/api | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_upload_limits_centralized.py` | 29 collected tests | import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_email_oauth.py` | 580 lines, 28 collected tests | route/api, db/session, security, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_review_regressions.py` | 930 lines, 26 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_rename_user_owner_sync.py` | 686 lines, 26 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_cookbook_docker_access.py` | 339 lines, 28 collected tests | route/api, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_rename_user_owner_sync.py` | 740 lines, 27 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_taxonomy.py` | 27 collected tests | security, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_endpoint_resolver_urls.py` | 27 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | | `tests/test_helpers_import_state.py` | 426 lines, 26 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_taxonomy.py` | 26 collected tests | security, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_tool_path_confinement.py` | 24 collected tests | import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_copilot.py` | 23 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_workspace_confine.py` | 527 lines, 25 collected tests | route/api, filesystem, subprocess/script, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_chat_route_tool_policy.py` | 344 lines, 25 collected tests | filesystem, ui/static | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_tool_path_confinement.py` | 303 lines, 25 collected tests | import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_copilot.py` | 25 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_llm_core_temperature_reasoning.py` | 25 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_provider_endpoints_url_building.py` | 25 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_bg_job_tools.py` | 23 collected tests | filesystem, async/threading | Good first manual-review candidate if test themes are cohesive. | | `tests/test_research_utils.py` | 23 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_api_chat_security.py` | 401 lines, 22 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_api_chat_security.py` | 404 lines, 22 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_tool_support_heuristic.py` | 22 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_platform_compat.py` | 318 lines, 21 collected tests | import-state, filesystem, subprocess/script | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_platform_compat.py` | 322 lines, 21 collected tests | import-state, filesystem, subprocess/script | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_context_compactor.py` | 21 collected tests | db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_prompt_security.py` | 21 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | -| `tests/test_null_owner_gates.py` | 342 lines, 20 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_fenced_inline_args.py` | 21 collected tests | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_cache_affinity_local_only.py` | 21 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_model_capability_readers.py` | 646 lines, 20 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_calendar_recurrence.py` | 356 lines, 20 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_null_owner_gates.py` | 332 lines, 20 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_gallery_endpoint_hardening.py` | 326 lines, 20 collected tests | route/api, security, filesystem, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_redos_llm_parsers.py` | 20 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. | +| `tests/test_code_nav_tools.py` | 20 collected tests | db/session, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_youtube_handler_consolidation.py` | 20 collected tests | route/api, import-state | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_calendar_recurrence.py` | 338 lines | No obvious setup signals from static scan. | Plan split boundaries before editing. | -| `tests/test_workspace_confine.py` | 328 lines | route/api, filesystem, subprocess/script, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_tool_policy.py` | 475 lines | import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_fenced_example_not_executed_for_native_models.py` | 345 lines | async/threading | Plan split boundaries before editing. | | `tests/test_api_token_routes.py` | 578 lines | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_companion_readonly.py` | 402 lines | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_set_admin.py` | 317 lines | route/api, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_llm_core_ollama.py` | 324 lines | async/threading | Plan split boundaries before editing. | | `tests/test_imap_leak_fixes.py` | 404 lines | route/api, db/session, security, filesystem | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_auth_regressions.py` | 375 lines | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_auth_regressions.py` | 370 lines | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_agent_migration_manifest.py` | 340 lines | import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_mcp_email_decode_header_spaces.py` | 327 lines | route/api, db/session, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_diffusion_server_security.py` | 325 lines | route/api, import-state, security, filesystem, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_tool_policy.py` | 330 lines | import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_upload_handler_cleanup.py` | 831 lines | route/api, db/session, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_email_owner_scope.py` | 656 lines | route/api, db/session, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_upload_routes_owner_scope.py` | 344 lines | route/api, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_endpoint_owner_scope_followup.py` | 414 lines | route/api, db/session, filesystem | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_upload_routes_owner_scope.py` | 315 lines | route/api, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | -| `tests/test_email_owner_scope.py` | 474 lines | route/api, db/session, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_composer_arrow_up_recall_js.py` | 308 lines | filesystem, subprocess/script, ui/static | Plan split boundaries before editing. | +| `tests/test_task_cookbook_admin_gate.py` | 350 lines | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | +| `tests/test_email_test_connection_oauth.py` | 436 lines | route/api, db/session, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_upload_handler_atomicity.py` | 401 lines | filesystem, async/threading | Plan split boundaries before editing. | | `tests/test_kv_cache_invalidation_2927.py` | 463 lines | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. | | `tests/test_calendar_owner_scope.py` | 345 lines | route/api, db/session, import-state, filesystem, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. | @@ -226,28 +246,35 @@ Before using any of these files as a split target, first decide whether the taxo | File | Lines | Collected tests | Sub-area | Signals | Suggested follow-up | |---|---:|---:|---|---|---| +| `tests/test_endpoint_probing.py` | 497 | 61 | endpoint_probing | route/api, db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_pr_blocker_audit.py` | 964 | 58 | pr_blocker_audit | import-state, security, filesystem | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_agent_loop.py` | 469 | 52 | agent_loop | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_service_health.py` | 472 | 47 | service_health | async/threading | Review taxonomy mapping before using as a split target. | -| `tests/test_run_focus.py` | 399 | 47 | run_focus | security, filesystem, subprocess/script, ui/static | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_endpoint_probing.py` | 411 | 34 | endpoint_probing | route/api, db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_chat_helpers.py` | 264 | 31 | chat_helpers | route/api | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_model_context.py` | 251 | 30 | model_context | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_endpoint_resolver.py` | 148 | 30 | endpoint_resolver | - | Review taxonomy mapping before using as a split target. | +| `tests/test_run_focus.py` | 492 | 55 | run_focus | security, filesystem, subprocess/script, ui/static | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_agent_loop.py` | 500 | 54 | agent_loop | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_review_regressions.py` | 1429 | 41 | review_regressions | route/api, db/session, import-state, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_chat_helpers.py` | 582 | 37 | chat_helpers | route/api, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_model_context.py` | 314 | 36 | model_context | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_redos_verdict_continuation.py` | 80 | 30 | redos_verdict_continuation | route/api | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_upload_limits_centralized.py` | 110 | 29 | upload_limits_centralized | import-state, filesystem | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_review_regressions.py` | 930 | 26 | review_regressions | route/api, db/session, import-state, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_taxonomy.py` | 145 | 26 | taxonomy | security, ui/static | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_copilot.py` | 170 | 23 | copilot | - | Review taxonomy mapping before using as a split target. | +| `tests/test_taxonomy.py` | 151 | 27 | taxonomy | security, ui/static | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_endpoint_resolver_urls.py` | 95 | 27 | endpoint_resolver_urls | - | Review taxonomy mapping before using as a split target. | +| `tests/test_workspace_confine.py` | 527 | 25 | workspace_confine | route/api, filesystem, subprocess/script, async/threading | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_copilot.py` | 182 | 25 | copilot | - | Review taxonomy mapping before using as a split target. | +| `tests/test_bg_job_tools.py` | 174 | 23 | bg_job_tools | filesystem, async/threading | Review taxonomy mapping before using as a split target. | | `tests/test_tool_support_heuristic.py` | 166 | 22 | tool_support_heuristic | - | Review taxonomy mapping before using as a split target. | -| `tests/test_platform_compat.py` | 318 | 21 | platform_compat | import-state, filesystem, subprocess/script | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_platform_compat.py` | 322 | 21 | platform_compat | import-state, filesystem, subprocess/script | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_context_compactor.py` | 233 | 21 | context_compactor | db/session, import-state, async/threading | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_fenced_inline_args.py` | 186 | 21 | fenced_inline_args | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_cache_affinity_local_only.py` | 104 | 21 | cache_affinity_local_only | - | Review taxonomy mapping before using as a split target. | +| `tests/test_model_capability_readers.py` | 646 | 20 | model_capability_readers | - | Review taxonomy mapping before using as a split target. | +| `tests/test_code_nav_tools.py` | 198 | 20 | code_nav_tools | db/session, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_youtube_handler_consolidation.py` | 104 | 20 | youtube_handler_consolidation | route/api, import-state | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_workspace_confine.py` | 328 | 18 | workspace_confine | route/api, filesystem, subprocess/script, async/threading | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_tool_policy.py` | 475 | 18 | tool_policy | import-state, async/threading | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_fenced_example_not_executed_for_native_models.py` | 345 | 18 | fenced_example_not_executed_for_native_models | async/threading | Review taxonomy mapping before using as a split target. | | `tests/test_companion_readonly.py` | 402 | 17 | companion_readonly | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_set_admin.py` | 317 | 17 | set_admin | route/api, import-state, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_imap_leak_fixes.py` | 404 | 15 | imap_leak_fixes | route/api, db/session, security, filesystem | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_agent_migration_manifest.py` | 340 | 15 | agent_migration_manifest | import-state, filesystem | Review taxonomy and setup/risk boundaries before any split. | -| `tests/test_tool_policy.py` | 330 | 13 | tool_policy | import-state, async/threading | Review taxonomy and setup/risk boundaries before any split. | +| `tests/test_upload_handler_cleanup.py` | 831 | 13 | upload_handler_cleanup | route/api, db/session, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. | | `tests/test_upload_handler_atomicity.py` | 401 | 9 | upload_handler_atomicity | filesystem, async/threading | Review taxonomy mapping before using as a split target. | | `tests/test_kv_cache_invalidation_2927.py` | 463 | 8 | kv_cache_invalidation_2927 | route/api, db/session, import-state, async/threading | Review taxonomy and setup/risk boundaries before any split. | @@ -259,14 +286,16 @@ Files still in the `uncategorized` taxonomy area are listed separately below so | File | Lines | Collected tests | Area | Sub-area | Signals | Why this is a candidate | |---|---:|---:|---|---|---|---| -| `tests/test_provider_classification.py` | 188 | 67 | services | provider | - | 67 collected tests | -| `tests/test_provider_endpoints.py` | 241 | 58 | services | provider | subprocess/script | 58 collected tests | -| `tests/test_llm_core_temperature.py` | 196 | 41 | services | llm | - | 41 collected tests | -| `tests/test_provider_detection.py` | 148 | 31 | services | provider | - | 31 collected tests | -| `tests/test_embedding_lanes.py` | 1104 | 29 | services | embedding | filesystem | 1104 lines, 29 collected tests | +| `tests/test_provider_classification.py` | 115 | 42 | services | provider | - | 42 collected tests | +| `tests/test_chat_route_tool_policy.py` | 344 | 25 | routes | route | filesystem, ui/static | 344 lines, 25 collected tests | +| `tests/test_llm_core_temperature_reasoning.py` | 133 | 25 | services | llm | - | 25 collected tests | +| `tests/test_provider_endpoints_url_building.py` | 86 | 25 | services | provider | - | 25 collected tests | | `tests/test_research_utils.py` | 97 | 23 | services | research | - | 23 collected tests | | `tests/test_prompt_security.py` | 203 | 21 | security | security | - | 21 collected tests | -| `tests/test_calendar_recurrence.py` | 338 | 19 | services | calendar | - | 338 lines | +| `tests/test_calendar_recurrence.py` | 356 | 20 | services | calendar | - | 356 lines, 20 collected tests | +| `tests/test_redos_llm_parsers.py` | 201 | 20 | services | llm | - | 20 collected tests | +| `tests/test_llm_core_ollama.py` | 324 | 16 | services | llm | async/threading | 324 lines | +| `tests/test_composer_arrow_up_recall_js.py` | 308 | 11 | js | js | filesystem, subprocess/script, ui/static | 308 lines | ## High-risk candidates to defer first @@ -274,21 +303,21 @@ These files may still be split later, but not as the first implementation slice | File | Lines | Collected tests | High-risk signals | |---|---:|---:|---| -| `tests/test_model_routes.py` | 1778 | 139 | db/session, import-state, route/api | -| `tests/test_security_regressions.py` | 1224 | 92 | db/session, import-state, route/api, security | -| `tests/test_cookbook_helpers.py` | 912 | 65 | route/api | -| `tests/test_shell_routes.py` | 481 | 63 | import-state, route/api | +| `tests/test_model_routes.py` | 2099 | 163 | db/session, import-state, route/api | +| `tests/test_security_regressions.py` | 1515 | 98 | db/session, import-state, route/api, security | +| `tests/test_cookbook_helpers.py` | 958 | 69 | route/api | +| `tests/test_shell_routes.py` | 538 | 69 | import-state, route/api | +| `tests/test_endpoint_probing.py` | 497 | 61 | db/session, import-state, route/api | | `tests/test_pr_blocker_audit.py` | 964 | 58 | import-state, security | -| `tests/test_agent_loop.py` | 469 | 52 | db/session, import-state | -| `tests/test_run_focus.py` | 399 | 47 | security | -| `tests/test_endpoint_probing.py` | 411 | 34 | db/session, import-state, route/api | +| `tests/test_research_routes_path_confinement.py` | 565 | 58 | route/api, security | +| `tests/test_run_focus.py` | 492 | 55 | security | +| `tests/test_agent_loop.py` | 500 | 54 | db/session, import-state | +| `tests/test_manage_mcp_command_allowlist.py` | 168 | 45 | db/session | +| `tests/test_review_regressions.py` | 1429 | 41 | db/session, import-state, route/api | +| `tests/test_chat_helpers.py` | 582 | 37 | route/api | +| `tests/test_model_context.py` | 314 | 36 | db/session, import-state | +| `tests/test_email_oauth.py` | 798 | 34 | db/session, route/api, security | | `tests/test_llm_core_anthropic_temp_omit.py` | 94 | 32 | db/session | -| `tests/test_chat_helpers.py` | 264 | 31 | route/api | -| `tests/test_model_context.py` | 251 | 30 | db/session, import-state | -| `tests/test_upload_limits_centralized.py` | 110 | 29 | import-state | -| `tests/test_email_oauth.py` | 580 | 28 | db/session, route/api, security | -| `tests/test_review_regressions.py` | 930 | 26 | db/session, import-state, route/api | -| `tests/test_rename_user_owner_sync.py` | 686 | 26 | db/session, import-state, route/api | ## Rules for future split PRs @@ -313,7 +342,7 @@ Do not start a split PR from this planning issue alone if the file's boundaries This document was generated with: ```bash -.venv/bin/python tests/tools/build_oversized_test_split_plan.py +venv/bin/python tests/tools/build_oversized_test_split_plan.py ``` ## Freshness check @@ -321,6 +350,6 @@ This document was generated with: After editing the builder or rebasing the branch, regenerate the plan and confirm no unexpected plan drift: ```bash -.venv/bin/python tests/tools/build_oversized_test_split_plan.py +venv/bin/python tests/tools/build_oversized_test_split_plan.py git diff --exit-code -- tests/OVERSIZED_TEST_SPLIT_PLAN.md ``` diff --git a/tests/tools/build_oversized_test_split_plan.py b/tests/tools/build_oversized_test_split_plan.py index 855945c1c..671b646d5 100644 --- a/tests/tools/build_oversized_test_split_plan.py +++ b/tests/tools/build_oversized_test_split_plan.py @@ -441,6 +441,10 @@ def write_report(metrics: list[FileMetric], node_count_total: int) -> None: lines = [ "# Oversized Test File Split Plan", "", + "> [!NOTE]", + "> This is generated planning material based on the current test tree.", + "> It does not define runtime behaviour or approve any test-file split.", + "", "## Purpose", "", "This document plans future oversized test-file splits using current repo data.", @@ -508,7 +512,7 @@ def write_report(metrics: list[FileMetric], node_count_total: int) -> None: "This document was generated with:", "", "```bash", - ".venv/bin/python tests/tools/build_oversized_test_split_plan.py", + "venv/bin/python tests/tools/build_oversized_test_split_plan.py", "```", "", "## Freshness check", @@ -516,7 +520,7 @@ def write_report(metrics: list[FileMetric], node_count_total: int) -> None: "After editing the builder or rebasing the branch, regenerate the plan and confirm no unexpected plan drift:", "", "```bash", - ".venv/bin/python tests/tools/build_oversized_test_split_plan.py", + "venv/bin/python tests/tools/build_oversized_test_split_plan.py", "git diff --exit-code -- tests/OVERSIZED_TEST_SPLIT_PLAN.md", "```", "", From 01790c2f08233f1b8d15457fbe7dd0d089d7b8ef Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:11:34 +0100 Subject: [PATCH 09/43] fix(mcp): keep built-in servers on SDK v1 (#5820) --- requirements.txt | 5 ++++- tests/test_mcp_dependency_compatibility.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/test_mcp_dependency_compatibility.py diff --git a/requirements.txt b/requirements.txt index be5f5d450..3c5114f53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,10 @@ python-dateutil caldav cryptography bcrypt -mcp +# Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a +# breaking rewrite, so keep fresh installs on the maintained v1 line until the +# servers are migrated together. +mcp<2 pyotp qrcode[pil] croniter diff --git a/tests/test_mcp_dependency_compatibility.py b/tests/test_mcp_dependency_compatibility.py new file mode 100644 index 000000000..9efefe4fe --- /dev/null +++ b/tests/test_mcp_dependency_compatibility.py @@ -0,0 +1,15 @@ +"""Regression coverage for the built-in MCP servers' SDK compatibility line.""" + +from pathlib import Path + + +REQUIREMENTS = Path(__file__).resolve().parents[1] / "requirements.txt" + + +def test_mcp_requirement_excludes_breaking_v2_sdk(): + requirements = [ + line.split("#", 1)[0].strip().replace(" ", "") + for line in REQUIREMENTS.read_text(encoding="utf-8").splitlines() + ] + + assert "mcp<2" in requirements From d190980f4779108571cfc75b988ea6a5f15b6606 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Tue, 28 Jul 2026 18:28:31 +0100 Subject: [PATCH 10/43] docs: classify test planning documents --- tests/LAYOUT_INVENTORY.md | 17 +++++++++++++++++ tests/OVERSIZED_TEST_SPLIT_PLAN.md | 4 ++++ tests/tools/build_oversized_test_split_plan.py | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/tests/LAYOUT_INVENTORY.md b/tests/LAYOUT_INVENTORY.md index 4a14961be..60a9b7c63 100644 --- a/tests/LAYOUT_INVENTORY.md +++ b/tests/LAYOUT_INVENTORY.md @@ -10,6 +10,23 @@ - **Implemented by:** #3842 - **Status:** completed historical snapshot +- **Durable testing policy:** [`TESTING_STANDARD.md`](./TESTING_STANDARD.md) +- **Current testing mechanics:** [`README.md`](./README.md) + +## Disposition + +This document is retained as a non-canonical historical record of the completed +CLI test relocation. + +Reusable testing rules are maintained in `TESTING_STANDARD.md`; current helper, +taxonomy-runner, and execution mechanics are maintained in `README.md`, the test +suite, and its supporting code. + +Counts, residual-file observations, and path listings in this document are +snapshot evidence only. They must not be maintained as durable repository truth. + +Any additional test relocation requires a separately approved issue or work item. + ## Purpose This document preserves the reasoning, scope, and validation contract for the diff --git a/tests/OVERSIZED_TEST_SPLIT_PLAN.md b/tests/OVERSIZED_TEST_SPLIT_PLAN.md index 481424e58..442b6c83c 100644 --- a/tests/OVERSIZED_TEST_SPLIT_PLAN.md +++ b/tests/OVERSIZED_TEST_SPLIT_PLAN.md @@ -2,6 +2,10 @@ > [!NOTE] > This is generated planning material based on the current test tree. +> +> It is non-canonical and disposable. Durable testing policy belongs in +> `tests/TESTING_STANDARD.md`; active split selection and prioritization +> belong in approved issues or the project tracker. > It does not define runtime behaviour or approve any test-file split. ## Purpose diff --git a/tests/tools/build_oversized_test_split_plan.py b/tests/tools/build_oversized_test_split_plan.py index 671b646d5..ac2499fdc 100644 --- a/tests/tools/build_oversized_test_split_plan.py +++ b/tests/tools/build_oversized_test_split_plan.py @@ -443,6 +443,10 @@ def write_report(metrics: list[FileMetric], node_count_total: int) -> None: "", "> [!NOTE]", "> This is generated planning material based on the current test tree.", + ">", + "> It is non-canonical and disposable. Durable testing policy belongs in", + "> `tests/TESTING_STANDARD.md`; active split selection and prioritization", + "> belong in approved issues or the project tracker.", "> It does not define runtime behaviour or approve any test-file split.", "", "## Purpose", From 5104a9a96710f682d641a1e10c4977ae196ab912 Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Tue, 28 Jul 2026 21:34:03 +0300 Subject: [PATCH 11/43] feat(tts): implement TTS cache size limit and eviction policy --- .env.example | 1 + services/tts/tts_service.py | 35 +++++++++ tests/test_tts_service_enforce_cache_limit.py | 73 +++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 tests/test_tts_service_enforce_cache_limit.py diff --git a/.env.example b/.env.example index d23276eb8..4eb4695e0 100644 --- a/.env.example +++ b/.env.example @@ -189,6 +189,7 @@ SEARXNG_INSTANCE=http://localhost:8080 # ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400 # email compose attachment (25 MB) # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400 # speech-to-text audio (25 MB) # ODYSSEUS_ICS_MAX_BYTES=10485760 # calendar .ics import (10 MB) +# ODYSSEUS_TTS_CACHE_MAX_BYTES=52428800 # TTS cache (500 MB) # ============================================================ # Host Docker access (explicit opt-in) diff --git a/services/tts/tts_service.py b/services/tts/tts_service.py index 2120d7720..e1c67d4da 100644 --- a/services/tts/tts_service.py +++ b/services/tts/tts_service.py @@ -2,6 +2,7 @@ """Multi-provider TTS service — dispatches to local Kokoro, OpenAI-compatible API, or browser.""" import io +import os import wave import logging import hashlib @@ -41,6 +42,11 @@ class TTSService: self.cache_dir = Path(cache_dir) self.cache_dir.mkdir(parents=True, exist_ok=True) self._kokoro = None # lazy-init + + try: + self.max_cache_bytes = int(os.getenv("ODYSSEUS_TTS_CACHE_MAX_BYTES", 500 * 1024 * 1024)) + except ValueError: + self.max_cache_bytes = 500 * 1024 * 1024 # ── Settings ── @@ -89,6 +95,35 @@ class TTSService: ext = ".mp3" if (len(data) >= 3 and (data[:3] == b'ID3' or (data[0] == 0xff and (data[1] & 0xe0) == 0xe0))) else ".wav" (self.cache_dir / f"{key}{ext}").write_bytes(data) + self._enforce_cache_limit() + + def _enforce_cache_limit(self): + """Evicts oldest files if the cache exceeds the configured byte limit.""" + if self.max_cache_bytes <= 0: + return + + files = [f for f in self.cache_dir.glob("*.*") if f.is_file()] + total_size = sum(f.stat().st_size for f in files) + + if total_size > self.max_cache_bytes: + logger.info(f"TTS cache ({total_size} bytes) exceeded limit ({self.max_cache_bytes} bytes). Evicting oldest files.") + + # Sort files by modification time (oldest first) + files.sort(key=lambda f: f.stat().st_mtime) + + # Trim down to 80% of max capacity so we aren't constantly triggering this on every new generation + target_size = self.max_cache_bytes * 0.8 + + while files and total_size > target_size: + f = files.pop(0) + try: + size = f.stat().st_size + f.unlink() + total_size -= size + except FileNotFoundError: + # File was deleted by another process + continue + def clear_cache(self): count = 0 for f in self.cache_dir.glob("*.*"): diff --git a/tests/test_tts_service_enforce_cache_limit.py b/tests/test_tts_service_enforce_cache_limit.py new file mode 100644 index 000000000..f4a31c75b --- /dev/null +++ b/tests/test_tts_service_enforce_cache_limit.py @@ -0,0 +1,73 @@ +import os +import time +from pathlib import Path +import pytest + +# Adjust the import path if your file is directly in ./services instead of ./services/tts +from services.tts.tts_service import TTSService + +def test_cache_under_limit(tmp_path, monkeypatch): + """Test that writing a file under the size limit does not trigger eviction.""" + # Set a tiny limit: 100 bytes + monkeypatch.setenv("TTS_CACHE_MAX_BYTES", "100") + + # Initialize service with pytest's temporary directory + service = TTSService(cache_dir=str(tmp_path)) + + # Write a 40-byte file (under the 100-byte limit) + service._put_cache("test_key", b"x" * 40) + + # Verify the file was written and nothing was deleted + files = list(tmp_path.glob("*.*")) + assert len(files) == 1 + assert sum(f.stat().st_size for f in files) == 40 + +def test_cache_exceeds_limit_triggers_eviction(tmp_path, monkeypatch): + """Test that exceeding the limit evicts the oldest files down to 80% capacity.""" + # Set limit to 100 bytes. 80% target capacity will be 80 bytes. + monkeypatch.setenv("TTS_CACHE_MAX_BYTES", "100") + service = TTSService(cache_dir=str(tmp_path)) + + # 1. Setup: Manually create two older files (40 bytes each) + file1 = tmp_path / "oldest.wav" + file2 = tmp_path / "middle.wav" + + file1.write_bytes(b"a" * 40) + file2.write_bytes(b"b" * 40) + + # Spoof timestamps so file1 is explicitly older than file2 + now = time.time() + os.utime(file1, (now - 100, now - 100)) # 100 seconds ago + os.utime(file2, (now - 50, now - 50)) # 50 seconds ago + + # 2. Action: Write a 3rd file using the service method (40 bytes) + # Total cache is now 120 bytes, which exceeds 100. + # It should delete oldest (file1) to drop to 80 bytes (which matches the 80% target). + service._put_cache("newest", b"c" * 40) + + # 3. Assertions + # The newest file should exist (saved as .wav because it lacks MP3 magic bytes) + newest_file = tmp_path / "newest.wav" + + assert not file1.exists(), "The oldest file should have been evicted." + assert file2.exists(), "The middle file should still exist." + assert newest_file.exists(), "The newest file should have been saved." + + # Verify the final directory size is <= 80 bytes + total_size = sum(f.stat().st_size for f in tmp_path.glob("*.*")) + assert total_size <= 80 + +def test_cache_limit_disabled(tmp_path, monkeypatch): + """Test that setting max bytes to 0 disables eviction.""" + monkeypatch.setenv("TTS_CACHE_MAX_BYTES", "0") + service = TTSService(cache_dir=str(tmp_path)) + + # Write 3 large files that would normally trigger eviction + service._put_cache("file1", b"x" * 1000) + service._put_cache("file2", b"x" * 1000) + service._put_cache("file3", b"x" * 1000) + + # Ensure nothing was deleted + files = list(tmp_path.glob("*.*")) + assert len(files) == 3 + assert sum(f.stat().st_size for f in files) == 3000 \ No newline at end of file From 98e4d8451bdcdf2f682bb39a54cccfeb0299ab9a Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Tue, 28 Jul 2026 22:00:54 +0300 Subject: [PATCH 12/43] fix(tests): update environment variable for TTS cache limit to include ODYSSEUS prefix --- tests/test_tts_service_enforce_cache_limit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_tts_service_enforce_cache_limit.py b/tests/test_tts_service_enforce_cache_limit.py index f4a31c75b..8a726557b 100644 --- a/tests/test_tts_service_enforce_cache_limit.py +++ b/tests/test_tts_service_enforce_cache_limit.py @@ -9,7 +9,7 @@ from services.tts.tts_service import TTSService def test_cache_under_limit(tmp_path, monkeypatch): """Test that writing a file under the size limit does not trigger eviction.""" # Set a tiny limit: 100 bytes - monkeypatch.setenv("TTS_CACHE_MAX_BYTES", "100") + monkeypatch.setenv("ODYSSEUS_TTS_CACHE_MAX_BYTES", "100") # Initialize service with pytest's temporary directory service = TTSService(cache_dir=str(tmp_path)) @@ -25,7 +25,7 @@ def test_cache_under_limit(tmp_path, monkeypatch): def test_cache_exceeds_limit_triggers_eviction(tmp_path, monkeypatch): """Test that exceeding the limit evicts the oldest files down to 80% capacity.""" # Set limit to 100 bytes. 80% target capacity will be 80 bytes. - monkeypatch.setenv("TTS_CACHE_MAX_BYTES", "100") + monkeypatch.setenv("ODYSSEUS_TTS_CACHE_MAX_BYTES", "100") service = TTSService(cache_dir=str(tmp_path)) # 1. Setup: Manually create two older files (40 bytes each) @@ -59,7 +59,7 @@ def test_cache_exceeds_limit_triggers_eviction(tmp_path, monkeypatch): def test_cache_limit_disabled(tmp_path, monkeypatch): """Test that setting max bytes to 0 disables eviction.""" - monkeypatch.setenv("TTS_CACHE_MAX_BYTES", "0") + monkeypatch.setenv("ODYSSEUS_TTS_CACHE_MAX_BYTES", "0") service = TTSService(cache_dir=str(tmp_path)) # Write 3 large files that would normally trigger eviction From 6bc6ee9c7e9d798c5c09e82311a1f75e075b0ae7 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Tue, 28 Jul 2026 20:26:38 +0100 Subject: [PATCH 13/43] docs: reconcile security reporting guidance --- SECURITY.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 1fa5b0b3b..38265a1da 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -37,4 +37,6 @@ Only `.env.example`, docs, source, tests, and static assets should be committed. ## Reporting -Please report vulnerabilities privately via GitHub security advisories if available, or by opening a minimal issue that does not disclose exploit details. +Report security vulnerabilities privately through [GitHub Security Advisories](https://github.com/odysseus-dev/odysseus/security/advisories/new). + +Do not open a public issue or discussion, and do not disclose exploit details publicly. From 25a4d134b1611b5329675ce212070bedcb8daad1 Mon Sep 17 00:00:00 2001 From: "Tal.Yuan" <ydonghao@users.noreply.github.com> Date: Wed, 29 Jul 2026 04:26:29 +0800 Subject: [PATCH 14/43] refactor(routes): move search domain into routes/search/ subpackage (#5779) Slice 2j of the route-domain reorganization (#4082/#4071). Moves search_routes.py into routes/search/, leaving a backward-compat sys.modules shim. Pure file reorganization, no behavior change. --- app.py | 2 +- routes/search/__init__.py | 5 ++ routes/search/search_routes.py | 111 +++++++++++++++++++++++++++++ routes/search_routes.py | 116 +++---------------------------- tests/test_search_routes_shim.py | 11 +++ 5 files changed, 137 insertions(+), 108 deletions(-) create mode 100644 routes/search/__init__.py create mode 100644 routes/search/search_routes.py create mode 100644 tests/test_search_routes_shim.py diff --git a/app.py b/app.py index e740ad518..2ae5ec761 100644 --- a/app.py +++ b/app.py @@ -692,7 +692,7 @@ from routes.history.history_routes import setup_history_routes app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler)) # Search -from routes.search_routes import setup_search_routes +from routes.search.search_routes import setup_search_routes app.include_router(setup_search_routes(config)) # Presets diff --git a/routes/search/__init__.py b/routes/search/__init__.py new file mode 100644 index 000000000..ea051bbe0 --- /dev/null +++ b/routes/search/__init__.py @@ -0,0 +1,5 @@ +"""Search route domain package (slice 2j, #4082/#4071). + +Contains search_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/search_routes.py re-exports from here. +""" diff --git a/routes/search/search_routes.py b/routes/search/search_routes.py new file mode 100644 index 000000000..1effb7b8f --- /dev/null +++ b/routes/search/search_routes.py @@ -0,0 +1,111 @@ +"""Search routes — /api/search/config GET, /api/search POST.""" + +import logging +from typing import Dict, Any + +from fastapi import APIRouter, Request + +import time + +from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO +from services.search.core import _call_provider +from services.search.providers import _get_provider_key, _get_search_instance + +logger = logging.getLogger(__name__) + + +async def _request_values(request: Request) -> Dict[str, Any]: + """Accept JSON, form data, or query params for search endpoints. + + The browser UI posts FormData, while the agent's generic app_api tool + posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler + runs, which made the model think SearXNG was broken. + """ + values: Dict[str, Any] = dict(request.query_params) + content_type = (request.headers.get("content-type") or "").lower() + try: + if "application/json" in content_type: + body = await request.json() + if isinstance(body, dict): + values.update(body) + else: + form = await request.form() + values.update(dict(form)) + except Exception: + pass + return values + + +def setup_search_routes(config) -> APIRouter: + router = APIRouter(tags=["search"]) + + @router.get("/api/search/config") + async def get_search_settings() -> Dict[str, Any]: + return get_search_config() + + @router.post("/api/search") + async def do_web_search(request: Request) -> Dict[str, Any]: + """Standalone web search — returns context string + source list. + + Used by Compare mode to pre-search once and share results across panes. + """ + values = await _request_values(request) + query = str(values.get("query") or values.get("q") or "").strip() + if not query: + return {"context": "", "sources": [], "error": "query is required"} + time_filter = values.get("time_filter") or values.get("freshness") + if time_filter is not None: + time_filter = str(time_filter).strip() or None + try: + context, sources = comprehensive_web_search( + query, return_sources=True, time_filter=time_filter, + ) + return {"context": context, "sources": sources} + except Exception as e: + logger.error(f"Standalone web search failed: {e}") + return {"context": "", "sources": [], "error": str(e)} + + @router.get("/api/search/providers") + async def list_search_providers(): + """Return available search providers with config status.""" + providers = [] + for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items(): + if pid == "disabled": + continue + available = True + if needs_key and not _get_provider_key(pid): + available = False + if needs_url and pid == "searxng" and not _get_search_instance(): + available = False + providers.append({ + "id": pid, + "label": label, + "available": available, + }) + return providers + + @router.post("/api/search/query") + async def search_with_provider(request: Request) -> Dict[str, Any]: + """Search using a specific provider. Used by compare search mode.""" + values = await _request_values(request) + query = str(values.get("query") or values.get("q") or "").strip() + provider = str(values.get("provider") or "").strip() + try: + count = int(values.get("count") or values.get("limit") or 10) + except Exception: + count = 10 + if not query: + return {"results": [], "provider": provider, "error": "query is required"} + if provider not in PROVIDER_INFO or provider == "disabled": + return {"results": [], "provider": provider, "error": "Unknown provider"} + t0 = time.time() + try: + results = _call_provider(provider, query, min(count, 20)) + elapsed = round(time.time() - t0, 2) + return {"results": results, "provider": provider, "time": elapsed} + except Exception as e: + elapsed = round(time.time() - t0, 2) + logger.error(f"Search provider {provider} failed: {e}") + return {"results": [], "provider": provider, "time": elapsed, "error": str(e)} + + return router diff --git a/routes/search_routes.py b/routes/search_routes.py index 1effb7b8f..03b94438b 100644 --- a/routes/search_routes.py +++ b/routes/search_routes.py @@ -1,111 +1,13 @@ -"""Search routes — /api/search/config GET, /api/search POST.""" +"""Backward-compat shim — canonical location is routes/search/search_routes.py. -import logging -from typing import Dict, Any +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.search_routes`` and ``from routes.search_routes import X`` +keep resolving to the canonical module. Keeps existing import paths working +after slice 2j (#4082/#4071). +""" -from fastapi import APIRouter, Request +import sys as _sys -import time +from routes.search import search_routes as _canonical # noqa: F401 -from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO -from services.search.core import _call_provider -from services.search.providers import _get_provider_key, _get_search_instance - -logger = logging.getLogger(__name__) - - -async def _request_values(request: Request) -> Dict[str, Any]: - """Accept JSON, form data, or query params for search endpoints. - - The browser UI posts FormData, while the agent's generic app_api tool - posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler - runs, which made the model think SearXNG was broken. - """ - values: Dict[str, Any] = dict(request.query_params) - content_type = (request.headers.get("content-type") or "").lower() - try: - if "application/json" in content_type: - body = await request.json() - if isinstance(body, dict): - values.update(body) - else: - form = await request.form() - values.update(dict(form)) - except Exception: - pass - return values - - -def setup_search_routes(config) -> APIRouter: - router = APIRouter(tags=["search"]) - - @router.get("/api/search/config") - async def get_search_settings() -> Dict[str, Any]: - return get_search_config() - - @router.post("/api/search") - async def do_web_search(request: Request) -> Dict[str, Any]: - """Standalone web search — returns context string + source list. - - Used by Compare mode to pre-search once and share results across panes. - """ - values = await _request_values(request) - query = str(values.get("query") or values.get("q") or "").strip() - if not query: - return {"context": "", "sources": [], "error": "query is required"} - time_filter = values.get("time_filter") or values.get("freshness") - if time_filter is not None: - time_filter = str(time_filter).strip() or None - try: - context, sources = comprehensive_web_search( - query, return_sources=True, time_filter=time_filter, - ) - return {"context": context, "sources": sources} - except Exception as e: - logger.error(f"Standalone web search failed: {e}") - return {"context": "", "sources": [], "error": str(e)} - - @router.get("/api/search/providers") - async def list_search_providers(): - """Return available search providers with config status.""" - providers = [] - for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items(): - if pid == "disabled": - continue - available = True - if needs_key and not _get_provider_key(pid): - available = False - if needs_url and pid == "searxng" and not _get_search_instance(): - available = False - providers.append({ - "id": pid, - "label": label, - "available": available, - }) - return providers - - @router.post("/api/search/query") - async def search_with_provider(request: Request) -> Dict[str, Any]: - """Search using a specific provider. Used by compare search mode.""" - values = await _request_values(request) - query = str(values.get("query") or values.get("q") or "").strip() - provider = str(values.get("provider") or "").strip() - try: - count = int(values.get("count") or values.get("limit") or 10) - except Exception: - count = 10 - if not query: - return {"results": [], "provider": provider, "error": "query is required"} - if provider not in PROVIDER_INFO or provider == "disabled": - return {"results": [], "provider": provider, "error": "Unknown provider"} - t0 = time.time() - try: - results = _call_provider(provider, query, min(count, 20)) - elapsed = round(time.time() - t0, 2) - return {"results": results, "provider": provider, "time": elapsed} - except Exception as e: - elapsed = round(time.time() - t0, 2) - logger.error(f"Search provider {provider} failed: {e}") - return {"results": [], "provider": provider, "time": elapsed, "error": str(e)} - - return router +_sys.modules[__name__] = _canonical diff --git a/tests/test_search_routes_shim.py b/tests/test_search_routes_shim.py new file mode 100644 index 000000000..a8b278488 --- /dev/null +++ b/tests/test_search_routes_shim.py @@ -0,0 +1,11 @@ +"""Regression test for the search route shim (slice 2j, #4082/#4071).""" + +import importlib + +import routes.search_routes as _shim_search # noqa: F401 + + +def test_legacy_and_canonical_search_module_are_same_object(): + legacy = importlib.import_module("routes.search_routes") + canonical = importlib.import_module("routes.search.search_routes") + assert legacy is canonical From 7b4c83089b8437a831e44e445f857ebfa0714e2a Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Tue, 28 Jul 2026 22:15:13 +0100 Subject: [PATCH 15/43] docs: improve README and setup navigation --- README.md | 12 ++++++++---- docs/setup.md | 51 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b8d2017d7..b790f4e2d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# Odysseus + <p align="center"> <img src="docs/odysseus-wordmark.png" alt="Odysseus" width="238"> </p> @@ -9,6 +11,8 @@ <p align="center"> <a href="#quick-start">Quick Start</a> · <a href="docs/setup.md">Setup Guide</a> · + <a href="docs/ARCHITECTURE.md">Architecture</a> · + <a href="SECURITY.md">Security</a> · <a href="CONTRIBUTING.md">Contributing</a> · <a href="ROADMAP.md">Roadmap</a> </p> @@ -51,15 +55,15 @@ Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration ## Demo -A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/index.html). +Explore the interface through the [interactive product tour](docs/index.html). ## Contributing -Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, documentation, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md), the [public roadmap](ROADMAP.md), and the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues). +Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, documentation, and small focused refactors. Read the [contributing guide](CONTRIBUTING.md), review the [public roadmap](ROADMAP.md), and browse the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues). ## Security -Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly. Deployment details are in the [setup guide](docs/setup.md#security-notes). +Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model or service ports publicly. Read the [security policy](SECURITY.md) and the [deployment security guidance](docs/setup.md#security-notes). ## Star History @@ -73,4 +77,4 @@ Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled ## License -AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md). +Licensed under AGPL-3.0-or-later. See the [license](LICENSE) and [acknowledgments](ACKNOWLEDGMENTS.md). diff --git a/docs/setup.md b/docs/setup.md index 61381461b..ab7265698 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,6 +1,28 @@ # Odysseus Setup Guide -This page keeps the detailed install, deployment, troubleshooting, and configuration notes out of the front README. +This guide covers installation, deployment, troubleshooting, and configuration. +For a minimal Docker installation, start with the +[repository README](../README.md#quick-start). + +## On this page + +- [Quick Start](#quick-start) + - [Docker](#docker-recommended) + - [Native Linux and macOS](#native-linux--macos) + - [Apple Silicon](#apple-silicon) + - [Native Windows](#native-windows) +- [Troubleshooting and advanced setup](#troubleshooting--advanced-setup) +- [Security notes](#security-notes) +- [Configuration](#configuration) +- [Architecture](#architecture) +- [Data and backups](#data) + +Related guidance: + +- [Security policy](../SECURITY.md) +- [Architecture overview](ARCHITECTURE.md) +- [Backup and restore guide](backup-restore.md) +- [Contributing guide](../CONTRIBUTING.md) ## Quick Start @@ -15,8 +37,8 @@ On first setup, Odysseus creates an admin account (`admin` unless For Docker installs, the same line is in `docker compose logs odysseus`. Use that for the first login, then change it in **Settings**. -Contributing? See [CONTRIBUTING.md](../CONTRIBUTING.md) for setup, testing, and -pull request guidelines. +Contributing? See the [contributing guide](../CONTRIBUTING.md) for development +setup, testing, and pull request guidelines. ### Docker (recommended) ```bash @@ -441,8 +463,8 @@ uv pip sync requirements.lock # reproduce it exactly la ### Outlook / Office 365 email Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox -passwords will fail. See [email-outlook.md](email-outlook.md) for the -current limitation and the planned integration direction. +passwords will fail. See the [Outlook and Microsoft 365 email guide](email-outlook.md) +for the current limitation and planned integration direction. ## Security Notes Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console. @@ -530,19 +552,16 @@ npx -y @playwright/mcp@latest --version That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup. ## Architecture -``` -app.py # FastAPI entry point -core/ auth, database, middleware, constants -src/ llm_core, agent_loop, agent_tools, chat_processor, search/ -routes/ chat, session, document, memory, model … endpoints -services/ docs, memory, search, hwfit (Cookbook) … -static/ index.html + app.js + style.css + js/ (modular front-end) -docs/ landing page (index.html) + preview clips -``` + +For stable high-level runtime structure, subsystem boundaries, and documentation +authority, see the [architecture overview](ARCHITECTURE.md). + +Source code, tests, and configuration remain authoritative for +implementation-sensitive behaviour. ## Data All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents), `memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`. -To back up or restore everything in `data/`, see the -[Backup & Restore guide](backup-restore.md). +To protect or recover this data, follow the +[backup and restore guide](backup-restore.md). From 53ce95049db686588544776191c83ea5469232ad Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <alexandremagteixeira@gmail.com> Date: Tue, 28 Jul 2026 23:02:40 +0100 Subject: [PATCH 16/43] docs: validate attachment lifecycle contract --- docs/attachments.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/attachments.md b/docs/attachments.md index 93f9e0ffe..4c0aec5fb 100644 --- a/docs/attachments.md +++ b/docs/attachments.md @@ -1,7 +1,13 @@ # Attachment References and Upload Storage -Odysseus stores uploaded bytes once under the configured upload directory and -passes stable references through chat history, tools, and future artifact work. +> [!NOTE] +> This document records the current attachment-reference and upload-lifecycle +> contract proposed for maintainer acceptance. Source code, tests, and +> configuration remain authoritative for implementation-sensitive behaviour. + +Odysseus stores chat and document attachment bytes under the configured upload +directory and passes stable references through chat history, document flows, and +tool context. The goal is to avoid duplicating large inline media payloads in `chat_messages.content` or the SQLite FTS index. @@ -54,6 +60,32 @@ External MCP/custom tools should treat the URI and attachment ID as the stable contract and request bytes through an owner-checked server path, not by assuming host filesystem layout. +## Implementation evidence + +The current contract is implemented primarily through: + +- `src/upload_handler.py` for upload metadata, owner-aware resolution, + reservations, cleanup, and deletion; +- `src/attachment_refs.py` for compact persisted references and search-index + sanitization; +- `src/document_processor.py` for resolving attachments into chat/model context; +- `src/tool_execution.py` for attachment manifests exposed to tools; +- `routes/upload_routes.py` and `routes/document_helpers.py` for upload and + retrieval paths. + +Focused regression coverage includes: + +- `tests/test_attachment_refs.py`; +- `tests/test_upload_handler_cleanup.py`; +- `tests/test_replace_messages_upload_reservations.py`; +- `tests/test_resolve_upload_path_nondict.py`; +- `tests/test_chat_preprocess_tool_policy.py`; +- the upload, attachment, and PDF-marker cases in + `tests/test_security_regressions.py`. + +These tests cover compact persistence, owner isolation, path containment, +cleanup safety, reservation-before-write behaviour, and traversal resistance. + ## Retention and Deletion Current retention behavior is conservative: From 61c138d9e7f50eaaf73185d1249f8debe809094b Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Wed, 29 Jul 2026 12:26:09 +0300 Subject: [PATCH 17/43] fixed .env.example ODYSSEUS_TTS_CACHE_MAX_BYTES into correct 500 MBs --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 4eb4695e0..2d1be3373 100644 --- a/.env.example +++ b/.env.example @@ -189,7 +189,7 @@ SEARXNG_INSTANCE=http://localhost:8080 # ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400 # email compose attachment (25 MB) # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400 # speech-to-text audio (25 MB) # ODYSSEUS_ICS_MAX_BYTES=10485760 # calendar .ics import (10 MB) -# ODYSSEUS_TTS_CACHE_MAX_BYTES=52428800 # TTS cache (500 MB) +# ODYSSEUS_TTS_CACHE_MAX_BYTES=524288000 # TTS cache (500 MB) # ============================================================ # Host Docker access (explicit opt-in) From 46905ab9b0f4a2ecf85996b23f47f678501792e4 Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Wed, 29 Jul 2026 12:32:43 +0300 Subject: [PATCH 18/43] added ODYSSEUS_TTS_CACHE_MAX_BYTES env variable to docker compose files --- docker-compose.gpu-amd.yml | 1 + docker-compose.gpu-nvidia.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index 91e223e05..9699fc038 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -67,6 +67,7 @@ services: - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index e8c2fd032..804a0a14e 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -66,6 +66,7 @@ services: - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} From 9914651cc9f1756fd1610952af42de8848b6015d Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Wed, 29 Jul 2026 12:41:31 +0300 Subject: [PATCH 19/43] improve cache eviction logic to handle file access errors and ensure stability --- services/tts/tts_service.py | 64 ++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/services/tts/tts_service.py b/services/tts/tts_service.py index e1c67d4da..c7f787954 100644 --- a/services/tts/tts_service.py +++ b/services/tts/tts_service.py @@ -98,31 +98,49 @@ class TTSService: self._enforce_cache_limit() def _enforce_cache_limit(self): - """Evicts oldest files if the cache exceeds the configured byte limit.""" - if self.max_cache_bytes <= 0: - return + """Evicts oldest files if the cache exceeds the configured byte limit.""" + if self.max_cache_bytes <= 0: + return - files = [f for f in self.cache_dir.glob("*.*") if f.is_file()] - total_size = sum(f.stat().st_size for f in files) + try: + files = [] + total_size = 0 - if total_size > self.max_cache_bytes: - logger.info(f"TTS cache ({total_size} bytes) exceeded limit ({self.max_cache_bytes} bytes). Evicting oldest files.") - - # Sort files by modification time (oldest first) - files.sort(key=lambda f: f.stat().st_mtime) - - # Trim down to 80% of max capacity so we aren't constantly triggering this on every new generation - target_size = self.max_cache_bytes * 0.8 - - while files and total_size > target_size: - f = files.pop(0) - try: - size = f.stat().st_size - f.unlink() - total_size -= size - except FileNotFoundError: - # File was deleted by another process - continue + # Safely scan files and sum sizes, ignoring files deleted mid-scan + for f in self.cache_dir.glob("*.*"): + try: + if f.is_file(): + files.append(f) + total_size += f.stat().st_size + except OSError: + continue + + if total_size > self.max_cache_bytes: + logger.info( + f"TTS cache ({total_size} bytes) exceeded limit ({self.max_cache_bytes} bytes). Evicting oldest files." + ) + + # Sort files by modification time (oldest first) + try: + files.sort(key=lambda f: f.stat().st_mtime) + except OSError as e: + logger.warning(f"Failed to sort cache files by mtime: {e}") + + # Trim down to 80% of max capacity + target_size = self.max_cache_bytes * 0.8 + + while files and total_size > target_size: + f = files.pop(0) + try: + size = f.stat().st_size + f.unlink() + total_size -= size + except OSError as e: + logger.warning(f"Failed to evict cache file {f}: {e}") + continue + + except Exception as e: + logger.warning(f"Error enforcing TTS cache limit: {e}", exc_info=True) def clear_cache(self): count = 0 From d183fe545b25766badbf5ada731ccd0fe7b0c594 Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Wed, 29 Jul 2026 12:42:20 +0300 Subject: [PATCH 20/43] add test for cache eviction handling unlink errors gracefully --- tests/test_tts_service_enforce_cache_limit.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_tts_service_enforce_cache_limit.py b/tests/test_tts_service_enforce_cache_limit.py index 8a726557b..1da9d16c0 100644 --- a/tests/test_tts_service_enforce_cache_limit.py +++ b/tests/test_tts_service_enforce_cache_limit.py @@ -70,4 +70,28 @@ def test_cache_limit_disabled(tmp_path, monkeypatch): # Ensure nothing was deleted files = list(tmp_path.glob("*.*")) assert len(files) == 3 - assert sum(f.stat().st_size for f in files) == 3000 \ No newline at end of file + assert sum(f.stat().st_size for f in files) == 3000 + +def test_cache_eviction_handles_unlink_error_gracefully(tmp_path, monkeypatch): + """Test that if unlinking a file fails, _put_cache still succeeds without raising.""" + service = TTSService(cache_dir=str(tmp_path)) + service.max_cache_bytes = 50 + + # Create a file to evict + old_file = tmp_path / "old.wav" + old_file.write_bytes(b"x" * 40) + + # Monkeypatch unlink on Path objects to simulate a PermissionError / file-lock failure + def mock_unlink(self_path): + raise OSError("Permission denied / file locked") + + monkeypatch.setattr(Path, "unlink", mock_unlink) + + # Writing a new file triggers eviction which encounters the mocked unlink error + try: + service._put_cache("new_key", b"y" * 40) + except Exception as e: + pytest.fail(f"_put_cache raised an exception during failed eviction: {e}") + + # The new file should still be written successfully + assert (tmp_path / "new_key.wav").exists() \ No newline at end of file From 2e631ad8160c76c42f757471d372cffa59ac88e4 Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Wed, 29 Jul 2026 12:47:55 +0300 Subject: [PATCH 21/43] improve cache size calculation by filtering file types --- services/tts/tts_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/tts/tts_service.py b/services/tts/tts_service.py index c7f787954..dd37865a7 100644 --- a/services/tts/tts_service.py +++ b/services/tts/tts_service.py @@ -107,9 +107,9 @@ class TTSService: total_size = 0 # Safely scan files and sum sizes, ignoring files deleted mid-scan - for f in self.cache_dir.glob("*.*"): + for f in self.cache_dir.iterdir(): try: - if f.is_file(): + if f.is_file() and f.suffix.lower() in (".mp3", ".wav"): files.append(f) total_size += f.stat().st_size except OSError: From 9297bed5b9574ea5ba13039821a51d77a2afaccd Mon Sep 17 00:00:00 2001 From: Boody <bodebi639@gmail.com> Date: Wed, 29 Jul 2026 12:54:55 +0300 Subject: [PATCH 22/43] add ODYSSEUS_TTS_CACHE_MAX_BYTES environment variable to docker-compose --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index b1f2c37ee..b0efb4439 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,7 @@ services: - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} From 3250a4ce68ffb05e6d5023ca55c6c2189d548686 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:04:28 +0100 Subject: [PATCH 23/43] fix(ci): clear review label when issues close (#5813) The issue-close lifecycle change is narrowly scoped and correct. Closed issues remove the stale \`ready for review\` label and return before normal validation can restore it. Focused regressions cover closure and subsequent edits to a closed issue. The branch was updated onto current \`dev\`. The focused test, merged-result validation, diff checks, and GitHub CI passed. No blocking review threads remain. --- .github/scripts/check-issue-description.js | 13 ++- .github/workflows/issue-description-check.yml | 2 +- tests/test_issue_description_check.py | 86 +++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 tests/test_issue_description_check.py diff --git a/.github/scripts/check-issue-description.js b/.github/scripts/check-issue-description.js index a76ca29ab..63162b0d7 100644 --- a/.github/scripts/check-issue-description.js +++ b/.github/scripts/check-issue-description.js @@ -153,6 +153,16 @@ module.exports = async ({ github, context, core }) => { } } + const LABEL_BAD = 'needs more info'; + const LABEL_GOOD = 'ready for review'; + + // Closed issues are no longer awaiting review. + // This also prevents later edits to closed issues from restoring the label. + if (issue.state === 'closed') { + await dropLabel(LABEL_GOOD); + return; + } + // ── Find existing bot comment to update in-place ────────────────────────── const MARKER = '<!-- issue-description-check -->'; const { data: comments } = await github.rest.issues.listComments({ @@ -160,9 +170,6 @@ module.exports = async ({ github, context, core }) => { }); const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(MARKER)); - const LABEL_BAD = 'needs more info'; - const LABEL_GOOD = 'ready for review'; - if (failures.length === 0) { if (existing) { await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id }); diff --git a/.github/workflows/issue-description-check.yml b/.github/workflows/issue-description-check.yml index 52e9dddae..5ce6037f0 100644 --- a/.github/workflows/issue-description-check.yml +++ b/.github/workflows/issue-description-check.yml @@ -2,7 +2,7 @@ name: ci / issue description check on: issues: - types: [opened, edited, reopened] + types: [opened, edited, reopened, closed] permissions: issues: write diff --git a/tests/test_issue_description_check.py b/tests/test_issue_description_check.py new file mode 100644 index 000000000..196f21cfc --- /dev/null +++ b/tests/test_issue_description_check.py @@ -0,0 +1,86 @@ +"""Regression coverage for issue-description label lifecycle events.""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + + +_REPO = Path(__file__).resolve().parent.parent +_CHECKER = _REPO / ".github" / "scripts" / "check-issue-description.js" +_WORKFLOW = _REPO / ".github" / "workflows" / "issue-description-check.yml" +pytestmark = pytest.mark.skipif(not shutil.which("node"), reason="node not on PATH") + + +def _run_closed_issue(action): + harness = r""" +const checkIssueDescription = require(process.argv[1]); +const action = process.argv[2]; +const calls = []; +const unexpected = (name) => async () => { + throw new Error(`${name} should not be called for a closed issue`); +}; + +const github = { + rest: { + issues: { + removeLabel: async (params) => calls.push({ method: 'removeLabel', params }), + getLabel: unexpected('getLabel'), + addLabels: unexpected('addLabels'), + listComments: unexpected('listComments'), + createComment: unexpected('createComment'), + updateComment: unexpected('updateComment'), + deleteComment: unexpected('deleteComment'), + }, + }, +}; +const context = { + payload: { + action, + issue: { number: 42, state: 'closed', body: '', labels: [] }, + }, + repo: { owner: 'odysseus-dev', repo: 'odysseus' }, +}; +const core = { + warning: unexpected('core.warning'), + setFailed: unexpected('core.setFailed'), +}; + +checkIssueDescription({ github, context, core }) + .then(() => process.stdout.write(JSON.stringify(calls))) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); +""" + proc = subprocess.run( + ["node", "-e", harness, str(_CHECKER), action], + capture_output=True, + text=True, + cwd=str(_REPO), + timeout=30, + ) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def test_workflow_handles_issue_closures(): + workflow = _WORKFLOW.read_text() + assert "types: [opened, edited, reopened, closed]" in workflow + + +@pytest.mark.parametrize("action", ["closed", "edited"]) +def test_closed_issue_only_drops_ready_for_review(action): + assert _run_closed_issue(action) == [ + { + "method": "removeLabel", + "params": { + "owner": "odysseus-dev", + "repo": "odysseus", + "issue_number": 42, + "name": "ready for review", + }, + } + ] From 6a84398e75835e899d792953e0b4e55ac40d9ccf Mon Sep 17 00:00:00 2001 From: holden093 <kevin@nixit.it> Date: Thu, 30 Jul 2026 10:06:31 +0200 Subject: [PATCH 24/43] fix(skills): use utility model for skill tests instead of chat default (#5746) Skill tests are background automation tasks (like auto-naming and memory audit) and should use the configured utility model. Previously they resolved via resolve_endpoint("default") which returned the chat model, bypassing the utility model entirely. This completes the sweep started in PR #4027 which fixed auto-naming and memory audit but missed skill tests. --- routes/skills_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/skills_routes.py b/routes/skills_routes.py index 711baa2e5..00bef589f 100644 --- a/routes/skills_routes.py +++ b/routes/skills_routes.py @@ -1409,7 +1409,7 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: # Prefer the configured DEFAULT (→ Utility) model — not the current chat # session's model. Fall back to the caller's session model only if unset. - url, model, headers = resolve_endpoint("default", owner=user) + url, model, headers = resolve_endpoint("utility", owner=user) if not url or not model: url = url or ((body.get("endpoint_url") or "").strip() or None) model = model or ((body.get("model") or "").strip() or None) From f23221420fea7df82d9ca532cf6b16a4868b1136 Mon Sep 17 00:00:00 2001 From: Husam <husamemad60@gmail.com> Date: Thu, 30 Jul 2026 11:54:59 +0300 Subject: [PATCH 25/43] fix(skills): replace deprecated utcnow in skill timestamp helper (#5777) * fix(skills): replace deprecated utcnow in skill timestamp helper _now_iso() builds the 'created' value in skill frontmatter. datetime.utcnow() returns a naive datetime and has been deprecated since Python 3.12, scheduled for removal. Switch to the timezone-aware datetime.now(timezone.utc), keeping the serialized YYYY-MM-DDTHH:MM:SSZ shape unchanged so existing skill files keep parsing. timezone.utc is used rather than the datetime.UTC alias, which is 3.11+ only. Adds regression tests covering the deprecation, the serialized shape, and UTC correctness under a non-UTC local timezone -- the last guards against a bare datetime.now(), which yields the same shape but local wall time. Fixes #5697 * test(skills): skip timezone mutation where unsupported --------- Co-authored-by: Alexandre Teixeira <alexandremagteixeira@gmail.com> --- services/memory/skill_format.py | 4 +- tests/test_skill_format_timestamp.py | 58 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 tests/test_skill_format_timestamp.py diff --git a/services/memory/skill_format.py b/services/memory/skill_format.py index 2b2dfb1b3..628474b04 100644 --- a/services/memory/skill_format.py +++ b/services/memory/skill_format.py @@ -50,7 +50,7 @@ import json import logging import re from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) @@ -441,4 +441,4 @@ class Skill: def _now_iso() -> str: - return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/tests/test_skill_format_timestamp.py b/tests/test_skill_format_timestamp.py new file mode 100644 index 000000000..a9309bdc1 --- /dev/null +++ b/tests/test_skill_format_timestamp.py @@ -0,0 +1,58 @@ +"""Regression for issue #5697 — skill timestamps must not use ``datetime.utcnow()``. + +``_now_iso()`` builds the ``created`` value in skill frontmatter. ``utcnow()`` +returns a *naive* datetime and has been deprecated since Python 3.12, scheduled +for removal. The replacement must stay timezone-aware while keeping the +serialized ``YYYY-MM-DDTHH:MM:SSZ`` shape, so skill files written by older +versions keep parsing. + +The UTC check matters on its own: a bare ``datetime.now()`` also produces the +right shape, but emits local wall time, which would silently backdate or +postdate skills for every user outside UTC. +""" + +import os +import re +import time +import warnings +from datetime import datetime, timezone + +import pytest + +from services.memory.skill_format import _now_iso + +_ISO_Z = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + + +def test_now_iso_keeps_serialized_shape(): + assert _ISO_Z.match(_now_iso()) + + +def test_now_iso_emits_no_deprecation_warning(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _now_iso() + assert not [w for w in caught if issubclass(w.category, DeprecationWarning)] + + +@pytest.mark.skipif( + not hasattr(time, "tzset"), + reason="time.tzset is unavailable on this platform", +) +def test_now_iso_is_utc_not_local_time(): + """Pin UTC under a non-UTC local timezone, where the two visibly diverge.""" + original_tz = os.environ.get("TZ") + os.environ["TZ"] = "Asia/Amman" # UTC+3, never UTC + time.tzset() + try: + emitted = datetime.strptime(_now_iso(), "%Y-%m-%dT%H:%M:%SZ").replace( + tzinfo=timezone.utc + ) + drift = abs((emitted - datetime.now(timezone.utc)).total_seconds()) + assert drift < 60, f"timestamp is {drift}s off UTC — local time leaked in" + finally: + if original_tz is None: + os.environ.pop("TZ", None) + else: + os.environ["TZ"] = original_tz + time.tzset() From 578312200ac1828bc3bae039eac3c827036fdc69 Mon Sep 17 00:00:00 2001 From: Husam <husamemad60@gmail.com> Date: Thu, 30 Jul 2026 12:48:31 +0300 Subject: [PATCH 26/43] fix(markdown): restore extracted blocks verbatim so $& and $$ survive (#5768) The placeholder-restore pass in mdToHtml put code, math, mermaid and allowed-HTML blocks back with a string replacement, so String.replace read `$&`, `` $` ``, `$'` and `$$` in the *replacement* as substitution patterns. A fenced block containing them rendered corrupted: `$&` re-inserted the placeholder (`perl -pe 's/world/$& again/'` became `s/world/___CODE_BLOCK_0___amp; again/`), `` $` `` and `$'` spliced in the surrounding document, and `$$` collapsed to a single `$`. Pass a function replacer at all four sites, matching the inline-code site below them, which was already fixed this way. A function's return value is inserted verbatim with no `$` interpretation. The inline-code comment claimed `echo $1` would be read as a back-reference; with a string search value there are no capture groups, so `$1` is already literal. Reworded to name the four sequences that do corrupt. Fixes #5663 --- static/js/markdown.js | 20 ++++++++----- tests/test_markdown_rendering_js.py | 44 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/static/js/markdown.js b/static/js/markdown.js index 8735b83e7..f249facc9 100644 --- a/static/js/markdown.js +++ b/static/js/markdown.js @@ -758,30 +758,36 @@ export function mdToHtml(src, opts) { // Remove empty paragraphs s = s.replace(/<p><\/p>/g, ''); + // Every restore below passes a function replacer rather than the block string + // itself. With a string replacement, `String.replace` reads `$&`, `` $` ``, + // `$'` and `$$` in the *replacement* as substitution patterns, so a restored + // block containing them is corrupted: `$&` re-inserts the placeholder, `` $` `` + // and `$'` splice in the surrounding document, and `$$` collapses to `$`. Those + // sequences are ordinary content in fenced code (`perl -pe 's/x/$& y/'`, + // `echo "$$USD"`). A function replacer inserts its return value verbatim. + // CRITICAL: Restore allowed HTML blocks first allowedHtmlBlocks.forEach((block, index) => { - s = s.replace(`___ALLOWED_HTML_${index}___`, block); + s = s.replace(`___ALLOWED_HTML_${index}___`, () => block); }); // Restore math blocks mathBlocks.forEach((block, index) => { - s = s.replace(`___MATH_BLOCK_${index}___`, block); + s = s.replace(`___MATH_BLOCK_${index}___`, () => block); }); // Restore mermaid diagram blocks mermaidBlocks.forEach((block, index) => { - s = s.replace(`___MERMAID_BLOCK_${index}___`, block); + s = s.replace(`___MERMAID_BLOCK_${index}___`, () => block); }); // CRITICAL: Restore code blocks at the end codeBlocks.forEach((block, index) => { - s = s.replace(`___CODE_BLOCK_${index}___`, block); + s = s.replace(`___CODE_BLOCK_${index}___`, () => block); }); // Restore inline code spans last, so placeholders carried inside restored - // <a>/allowed-HTML blocks are resolved too. The function replacer keeps the - // escaped code literal — e.g. a shell snippet like `echo $1` is not treated - // as a regex back-reference. + // <a>/allowed-HTML blocks are resolved too. inlineCodeBlocks.forEach((block, index) => { s = s.replace(`___INLINE_CODE_${index}___`, () => block); }); diff --git a/tests/test_markdown_rendering_js.py b/tests/test_markdown_rendering_js.py index 2ffe8914f..536789b89 100644 --- a/tests/test_markdown_rendering_js.py +++ b/tests/test_markdown_rendering_js.py @@ -214,6 +214,50 @@ def test_inline_code_content_is_html_escaped(node_available): assert "<b>" not in html +def test_fenced_code_keeps_dollar_ampersand(node_available): + # Issue #5663: the block-restore pass used a string replacement, so `$&` in a + # restored block was read as "the matched text" and re-inserted the + # placeholder. `perl -pe 's/world/$& again/'` rendered as + # "s/world/___CODE_BLOCK_0___amp; again/" — the trailing "amp;" is the orphan + # left behind after `$&` consumed the `$&` of the escaped `$&`. + html = _run_markdown_case( + "```sh\necho \"hello world\" | perl -pe 's/world/$& again/'\n```" + ) + + assert "___CODE_BLOCK_" not in html + assert "s/world/$& again/" in html + assert "amp; again" not in html.replace("$& again", "") + + +def test_fenced_code_keeps_dollar_backtick_and_quote(node_available): + # `` $` `` and `$'` splice the text before/after the placeholder into the + # block. Unlike `$&` these leave no placeholder behind — the characters just + # vanish — so assert the content survives verbatim. + html = _run_markdown_case("```sh\nsed \"s/$`/x/\" && sed \"s/$'/y/\"\n```") + + assert "___CODE_BLOCK_" not in html + assert "s/$`/x/" in html + assert "s/$'/y/" in html + + +def test_fenced_code_keeps_double_dollar(node_available): + # `$$` collapsed to a single `$` in the restored block. + html = _run_markdown_case('```sh\necho "$$USD and $$"\n```') + + assert "$$USD and $$" in html + + +def test_mermaid_block_keeps_dollar_ampersand(node_available): + # The mermaid restore site had the same hazard: a node label containing `$&` + # re-inserted the ___MERMAID_BLOCK_n___ placeholder into the diagram source, + # which then fails to parse. The math and allowed-HTML sites are fixed the + # same way; they need KaTeX/sanitizer conditions this harness doesn't set up. + html = _run_markdown_case('```mermaid\ngraph TD; A["$&"] --> B;\n```') + + assert "___MERMAID_BLOCK_" not in html + assert "$&" in html + + def test_currency_dollar_amounts_are_not_rendered_as_math(node_available): # "$5 to $10" used to pair the two dollar signs as inline-math delimiters # and render "5 to" through KaTeX. Pandoc-style rules now reject it: the From 84709a00d979c396dfb34f5d60b2992efb5a9284 Mon Sep 17 00:00:00 2001 From: Husam <husamemad60@gmail.com> Date: Thu, 30 Jul 2026 13:30:00 +0300 Subject: [PATCH 27/43] fix(llm): omit temperature for major-only Opus ids (claude-opus-5) (#5761) The version pattern in _anthropic_rejects_temperature() required a minor component, so major-only ids like `claude-opus-5` never matched and the guard reported that the model accepts `temperature`. Anthropic rejects the field outright on Opus 4.7+, so every such call returned HTTP 400 and the stream aborted with zero tokens ("the model returned an empty response"). Make the minor optional and read a missing minor as `.0`. The major is also capped at 1-2 digits with a no-trailing-digit lookahead, mirroring the minor: once the minor is optional, a greedy major would swallow the date in `claude-3-opus-20240229` and read it as version 20240229, dropping temperature from a model that accepts it. Fixes #5753 Co-authored-by: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> --- src/llm_core.py | 26 ++++++++++++++++------ tests/test_llm_core_anthropic_temp_omit.py | 26 +++++++++++++++++++++- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/llm_core.py b/src/llm_core.py index 4dec32376..3e84c1060 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -1237,15 +1237,27 @@ def _anthropic_rejects_temperature(model: str) -> bool: return False # `(?<![a-z])` anchors "opus" to a word boundary so a substring match like # `oct-opus`/`octopus-4-8` can't be read as Opus (it would otherwise strip - # temperature). Cap the minor at 1-2 digits and forbid a trailing digit so a - # dated id like `claude-opus-4-20250514` (Opus 4.0) parses as major-only (no - # minor match, kept) instead of reading the date `20250514` as a giant minor - # that would falsely test >= 4.7. Dated 4.7+ snapshots (`claude-opus-4-7- - # 20260201`) keep their explicit minor and are still matched. - match = re.search(r"(?<![a-z])opus[-_]?(\d+)[-_.](\d{1,2})(?!\d)", model.lower()) + # temperature). Both version components are capped at 1-2 digits and forbid a + # trailing digit, so an 8-digit date can never be read as a version number: + # `claude-opus-4-20250514` (Opus 4.0) parses as major-only rather than reading + # `20250514` as a giant minor, and `claude-3-opus-20240229` (legacy Claude 3 + # Opus, date directly after "opus-") fails to match at all rather than reading + # the date as a giant major. Dated 4.7+ snapshots (`claude-opus-4-7-20260201`) + # keep their explicit minor and are still matched. + # + # The minor is optional and a missing minor reads as `.0`, so major-only ids + # like `claude-opus-5` are correctly treated as >= 4.7 (issue #5753). Without + # this, every Opus 5 call kept `temperature` and failed with HTTP 400 — visible + # only on paths that pass a temperature, e.g. scheduled tasks inheriting + # `stream_agent_loop`'s 0.3 default, which returned empty responses. + match = re.search( + r"(?<![a-z])opus[-_]?(\d{1,2})(?!\d)(?:[-_.](\d{1,2})(?!\d))?", model.lower() + ) if not match: return False - return (int(match.group(1)), int(match.group(2))) >= (4, 7) + major = int(match.group(1)) + minor = int(match.group(2)) if match.group(2) else 0 + return (major, minor) >= (4, 7) # Reasoning effort level sent to Mistral thinking-capable models. Mistral's # API accepts "high", "medium", "low", "none" — see diff --git a/tests/test_llm_core_anthropic_temp_omit.py b/tests/test_llm_core_anthropic_temp_omit.py index 2274f1dc9..f7d26aef0 100644 --- a/tests/test_llm_core_anthropic_temp_omit.py +++ b/tests/test_llm_core_anthropic_temp_omit.py @@ -29,6 +29,13 @@ from src.llm_core import _anthropic_rejects_temperature, _build_anthropic_payloa "anthropic/claude-opus-4-7", # tolerate a provider-prefixed id "claude-opus-4-10", # future minor still >= 4.7 "claude-opus-5-0", # future major + # Major-only ids: a missing minor reads as `.0`, so these are >= 4.7 too + # (issue #5753). Before the fix the version pattern required a minor, so + # these fell through to "accepts temperature" and every call 400'd. + "claude-opus-5", + "claude-opus-5-20260101", # major-only + dated snapshot + "anthropic/claude-opus-5", # major-only behind a provider prefix + "claude-opus-6", # future major-only ], ) def test_opus_47_plus_rejects_temperature(model): @@ -48,7 +55,10 @@ def test_opus_47_plus_rejects_temperature(model): "claude-opus-4-6-20251201", # dated 4.6 snapshot — older, still keeps temperature "claude-sonnet-4-6", "claude-3-5-sonnet", - "claude-3-opus-20240229", # legacy Claude 3 Opus — no opus-N-M pattern, kept + "claude-3-opus-20240229", # legacy Claude 3 Opus — date directly after + # "opus-", so the major must not swallow it as version 20240229 (that is + # what makes capping the major at 1-2 digits necessary once the minor + # became optional in #5753). "claude-haiku-4-5", "claude-x", "octopus-4-8", # "opus" only as a substring of another word — must not match @@ -87,6 +97,20 @@ def test_payload_keeps_temperature_for_older_models(): assert _payload("claude-3-5-sonnet", 1.2)["temperature"] == 1.0 +def test_payload_omits_temperature_for_major_only_opus_5(): + # Issue #5753: the scheduled-task path calls stream_agent_loop() without a + # temperature and inherits its 0.3 default, so `claude-opus-5` 400'd on every + # run and surfaced as "the model returned an empty response". Interactive chat + # leaves temperature None and never hit it. + assert "temperature" not in _payload("claude-opus-5", 0.3) + + +def test_payload_keeps_temperature_for_legacy_claude_3_opus(): + # Guards the major-digit cap: `opus-20240229` must not parse as version + # 20240229, or Claude 3 Opus would silently lose the caller's temperature. + assert _payload("claude-3-opus-20240229", 0.5)["temperature"] == 0.5 + + def test_payload_keeps_temperature_for_dated_opus_4_0(): # Anthropic's dated id for Opus 4.0 (claude-opus-4-20250514) is in this repo's # ANTHROPIC_MODELS list. The date must not be misread as a >= 4.7 minor, or the From 28c333e64780ae2fbb7f0757d752e8bb5fb74990 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:24:39 +0100 Subject: [PATCH 28/43] fix(email): preserve OAuth SMTP security (#5802) --- static/js/settings.js | 2 ++ tests/test_email_oauth_connect_smtp_security.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/test_email_oauth_connect_smtp_security.py diff --git a/static/js/settings.js b/static/js/settings.js index 72936adee..3819c83fa 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -3031,12 +3031,14 @@ async function initEmailAccountsSettings() { const body = { name: el('eaf-name').value.trim() || el('eaf-from').value.trim(), from_address: el('eaf-from').value.trim(), + display_name: el('eaf-display-name').value.trim(), imap_host: el('eaf-imap-host').value.trim(), imap_port: parseInt(el('eaf-imap-port').value) || 993, imap_user: el('eaf-imap-user').value.trim(), imap_starttls: el('eaf-imap-starttls').checked, smtp_host: el('eaf-smtp-host').value.trim(), smtp_port: parseInt(el('eaf-smtp-port').value) || 587, + smtp_security: el('eaf-smtp-security').value, smtp_user: el('eaf-imap-user').value.trim(), }; if (!body.name) { el('eaf-msg').textContent = 'Enter a Name or Email first'; el('eaf-msg').style.color = 'var(--red)'; return; } diff --git a/tests/test_email_oauth_connect_smtp_security.py b/tests/test_email_oauth_connect_smtp_security.py new file mode 100644 index 000000000..21c4224a6 --- /dev/null +++ b/tests/test_email_oauth_connect_smtp_security.py @@ -0,0 +1,15 @@ +"""Regression coverage for SMTP security saved before Google OAuth.""" + +from pathlib import Path + + +_REPO = Path(__file__).resolve().parents[1] + + +def test_email_tab_oauth_connect_persists_selected_smtp_security(): + source = (_REPO / "static" / "js" / "settings.js").read_text(encoding="utf-8") + start = source.index("el('eaf-oauth-btn').addEventListener") + handler_body = source[start:source.index("if (!body.name)", start)] + + assert "smtp_security: el('eaf-smtp-security').value" in handler_body + assert "display_name: el('eaf-display-name').value.trim()" in handler_body From 25c9e735ef5ce605f47f8f666ac6689056d2c10c Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:57:07 +0100 Subject: [PATCH 29/43] fix(email): open settings after OAuth callback (#5803) --- static/js/settings.js | 45 +++++++++++---------- tests/test_email_oauth_settings_redirect.py | 19 +++++++++ 2 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 tests/test_email_oauth_settings_redirect.py diff --git a/static/js/settings.js b/static/js/settings.js index 3819c83fa..540acff00 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -5790,29 +5790,30 @@ export function close() { window.history.replaceState(null, '', clean); const success = sp.has('email_oauth_success'); const errMsg = sp.get('email_oauth_error') || ''; - // Open settings → integrations after the app has initialised. - function _tryOpen() { - if (window.settingsModule && typeof window.settingsModule.open === 'function') { - window.settingsModule.open('integrations'); - // Brief toast-style banner. - const banner = document.createElement('div'); - banner.textContent = success - ? '✓ Google account connected — email is ready' - : `Google OAuth failed: ${errMsg || 'unknown error'}`; - Object.assign(banner.style, { - position: 'fixed', bottom: '24px', left: '50%', transform: 'translateX(-50%)', - background: success ? 'var(--accent, #50fa7b)' : 'var(--red, #ff5555)', - color: '#000', padding: '8px 18px', borderRadius: '6px', fontSize: '12px', - fontWeight: '600', zIndex: '99999', pointerEvents: 'none', - boxShadow: '0 2px 12px rgba(0,0,0,0.3)', - }); - document.body.appendChild(banner); - setTimeout(() => banner.remove(), 4000); - } else { - setTimeout(_tryOpen, 100); - } + // Open settings → integrations once the document is ready. This module owns + // the open() API, so it does not need to wait for a window-level alias. + function _showResult() { + open('integrations'); + // Brief toast-style banner. + const banner = document.createElement('div'); + banner.textContent = success + ? 'Google account connected — email is ready' + : `Google OAuth failed: ${errMsg || 'unknown error'}`; + Object.assign(banner.style, { + position: 'fixed', bottom: '24px', left: '50%', transform: 'translateX(-50%)', + background: success ? 'var(--accent, #50fa7b)' : 'var(--red, #ff5555)', + color: '#000', padding: '8px 18px', borderRadius: '6px', fontSize: '12px', + fontWeight: '600', zIndex: '99999', pointerEvents: 'none', + boxShadow: '0 2px 12px rgba(0,0,0,0.3)', + }); + document.body.appendChild(banner); + setTimeout(() => banner.remove(), 4000); + } + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', _showResult, { once: true }); + } else { + _showResult(); } - _tryOpen(); })(); const settingsModule = { open, close, initIntegrations, initUnifiedIntegrations, syncAdminVisibility, refreshAiModelEndpoints }; diff --git a/tests/test_email_oauth_settings_redirect.py b/tests/test_email_oauth_settings_redirect.py new file mode 100644 index 000000000..f7d588132 --- /dev/null +++ b/tests/test_email_oauth_settings_redirect.py @@ -0,0 +1,19 @@ +"""Regression coverage for the settings UI after Google OAuth redirects.""" + +from pathlib import Path + + +_REPO = Path(__file__).resolve().parents[1] + + +def test_oauth_redirect_uses_the_module_local_settings_api(): + source = (_REPO / "static" / "js" / "settings.js").read_text(encoding="utf-8") + handler = source[ + source.index("(function _handleOauthRedirect"): + source.index("const settingsModule =") + ] + + assert "open('integrations');" in handler + assert "window.settingsModule" not in handler + assert "window.__odysseusAppStarted" not in handler + assert "document.addEventListener('DOMContentLoaded', _showResult, { once: true })" in handler From 0de76c4056a0f0fb8f24b3bf2972fa5a185d7b0c Mon Sep 17 00:00:00 2001 From: "Tal.Yuan" <ydonghao@users.noreply.github.com> Date: Tue, 4 Aug 2026 02:44:00 +0800 Subject: [PATCH 30/43] refactor(routes): move vault domain into routes/vault/ subpackage (#5780) Slice 2k of the route-domain reorganization (#4082/#4071). Moves vault_routes.py into routes/vault/, leaving a backward-compat sys.modules shim. Pure file reorganization, no behavior change. --- app.py | 2 +- routes/vault/__init__.py | 5 + routes/vault/vault_routes.py | 242 +++++++++++++++++++++++++++++++ routes/vault_routes.py | 246 ++------------------------------ tests/test_vault_routes_shim.py | 11 ++ 5 files changed, 268 insertions(+), 238 deletions(-) create mode 100644 routes/vault/__init__.py create mode 100644 routes/vault/vault_routes.py create mode 100644 tests/test_vault_routes_shim.py diff --git a/app.py b/app.py index 2ae5ec761..5fb2da54d 100644 --- a/app.py +++ b/app.py @@ -852,7 +852,7 @@ app.include_router(setup_codex_routes( )) app.include_router(setup_claude_routes()) -from routes.vault_routes import setup_vault_routes +from routes.vault.vault_routes import setup_vault_routes app.include_router(setup_vault_routes()) # Contacts (CardDAV) diff --git a/routes/vault/__init__.py b/routes/vault/__init__.py new file mode 100644 index 000000000..8aa82701d --- /dev/null +++ b/routes/vault/__init__.py @@ -0,0 +1,5 @@ +"""Vault route domain package (slice 2k, #4082/#4071). + +Contains vault_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/vault_routes.py re-exports from here. +""" diff --git a/routes/vault/vault_routes.py b/routes/vault/vault_routes.py new file mode 100644 index 000000000..7e97500f0 --- /dev/null +++ b/routes/vault/vault_routes.py @@ -0,0 +1,242 @@ +""" +vault_routes.py + +Vaultwarden / Bitwarden CLI integration — config and unlock endpoints. +Stores the BW_SESSION key in data/vault.json with restrictive permissions. +""" + +import json +import logging +import os +import shutil +import asyncio +from pathlib import Path +from datetime import datetime +from fastapi import APIRouter, Request +from pydantic import BaseModel + +from core.middleware import require_admin +from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool +from src.constants import VAULT_FILE as _VAULT_FILE + +logger = logging.getLogger(__name__) + +VAULT_FILE = Path(_VAULT_FILE) + + +def _find_bw() -> str: + """Locate the bw binary, checking PATH and common npm-global locations. + + On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by + which_tool via PATHEXT. + """ + p = which_tool("bw") + if p: + return p + if IS_WINDOWS: + appdata = os.environ.get("APPDATA", os.path.expanduser("~")) + for candidate in ( + os.path.join(appdata, "npm", "bw.cmd"), + os.path.join(appdata, "npm", "bw.exe"), + ): + if os.path.isfile(candidate): + return candidate + return "bw" + home = os.path.expanduser("~") + for candidate in ( + f"{home}/.npm-global/bin/bw", + f"{home}/.nvm/versions/node/*/bin/bw", + "/usr/local/bin/bw", + "/opt/homebrew/bin/bw", + ): + if "*" in candidate: + import glob + for m in glob.glob(candidate): + if os.path.isfile(m) and os.access(m, os.X_OK): + return m + elif os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below) + + +def _load_config() -> dict: + if VAULT_FILE.exists(): + try: + data = json.loads(VAULT_FILE.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except Exception: + pass + return {} + + +def _save_config(cfg: dict): + VAULT_FILE.parent.mkdir(parents=True, exist_ok=True) + VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir + # is ACL-restricted already). + safe_chmod(str(VAULT_FILE), 0o600) + + +async def _run_bw(args: list, session: str = None, input_text: str = None, + bw_password: str = None) -> tuple: + env = {} + env.update(os.environ) + if session: + env["BW_SESSION"] = session + # Secrets must never be passed as argv — process arguments are world-readable + # via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv + # support for bw commands that need it; unlock/login callers should prefer + # stdin so the master password is not left in the child environment either. + if bw_password is not None: + env["BW_PASSWORD"] = bw_password + bw_path = _find_bw() + try: + proc = await asyncio.create_subprocess_exec( + bw_path, *args, + stdin=asyncio.subprocess.PIPE if input_text else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + except FileNotFoundError: + return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127 + except Exception as e: + return "", f"Failed to launch bw: {e}", 1 + try: + stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None) + except Exception as e: + return "", f"bw subprocess error: {e}", 1 + return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode + + +class VaultConfig(BaseModel): + server_url: str = "" + email: str = "" + + +class VaultUnlockRequest(BaseModel): + master_password: str + + +class VaultLoginRequest(BaseModel): + email: str + master_password: str + + +def setup_vault_routes(): + router = APIRouter(prefix="/api/vault", tags=["vault"]) + + @router.get("/config") + async def get_config(request: Request): + """Return vault config (no sensitive fields).""" + require_admin(request) + cfg = _load_config() + return { + "server_url": cfg.get("server_url", ""), + "email": cfg.get("email", ""), + "unlocked": bool(cfg.get("session")), + "unlocked_at": cfg.get("unlocked_at", ""), + "bw_installed": await _check_bw_installed(), + } + + @router.post("/config") + async def save_config(req: VaultConfig, request: Request): + """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden.""" + require_admin(request) + cfg = _load_config() + cfg["server_url"] = req.server_url.strip().rstrip("/") + cfg["email"] = req.email.strip() + + if cfg["server_url"]: + _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]]) + if rc != 0: + return {"ok": False, "error": f"bw config failed: {stderr[:300]}"} + + _save_config(cfg) + return {"ok": True} + + @router.post("/login") + async def login(req: VaultLoginRequest, request: Request): + """Log in to Vaultwarden (required once per account).""" + require_admin(request) + cfg = _load_config() + # Update email + cfg["email"] = req.email + _save_config(cfg) + + stdout, stderr, rc = await _run_bw( + ["login", req.email, "--raw"], + input_text=req.master_password + "\n", + ) + if rc != 0: + # Already logged in is OK + if "already logged in" in stderr.lower(): + return {"ok": True, "already": True} + return {"ok": False, "error": f"Login failed: {stderr[:300]}"} + # bw login --raw prints session key on success (when 2FA disabled) + if stdout: + cfg["session"] = stdout + cfg["unlocked_at"] = datetime.utcnow().isoformat() + _save_config(cfg) + return {"ok": True} + + @router.post("/unlock") + async def unlock(req: VaultUnlockRequest, request: Request): + """Unlock the vault and save the session key.""" + require_admin(request) + # Pass the master password on stdin, not argv. argv is visible through + # `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in + # the child process environment. + stdout, stderr, rc = await _run_bw( + ["unlock", "--raw"], + input_text=req.master_password + "\n", + ) + if rc != 0: + return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"} + session = stdout.strip() + if not session: + return {"ok": False, "error": "bw returned empty session"} + cfg = _load_config() + cfg["session"] = session + cfg["unlocked_at"] = datetime.utcnow().isoformat() + _save_config(cfg) + return {"ok": True, "message": "Vault unlocked"} + + @router.post("/lock") + async def lock(request: Request): + """Lock the vault (clear session from config).""" + require_admin(request) + cfg = _load_config() + cfg.pop("session", None) + cfg.pop("unlocked_at", None) + _save_config(cfg) + # Also tell bw to lock + await _run_bw(["lock"]) + return {"ok": True, "message": "Vault locked"} + + @router.post("/logout") + async def logout(request: Request): + """Log out of the Bitwarden CLI completely.""" + require_admin(request) + await _run_bw(["logout"]) + cfg = _load_config() + cfg.pop("session", None) + cfg.pop("email", None) + cfg.pop("unlocked_at", None) + _save_config(cfg) + return {"ok": True} + + return router + + +async def _check_bw_installed() -> bool: + try: + proc = await asyncio.create_subprocess_exec( + _find_bw(), "--version", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await proc.communicate() + return proc.returncode == 0 + except Exception: + return False diff --git a/routes/vault_routes.py b/routes/vault_routes.py index 7e97500f0..cfed2ba39 100644 --- a/routes/vault_routes.py +++ b/routes/vault_routes.py @@ -1,242 +1,14 @@ -""" -vault_routes.py +"""Backward-compat shim — canonical location is routes/vault/vault_routes.py. -Vaultwarden / Bitwarden CLI integration — config and unlock endpoints. -Stores the BW_SESSION key in data/vault.json with restrictive permissions. +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.vault_routes``, ``from routes.vault_routes import X``, +and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used +by test_vault_password_not_in_argv.py all operate on the *same* object. +Keeps existing import paths working after slice 2k (#4082/#4071). """ -import json -import logging -import os -import shutil -import asyncio -from pathlib import Path -from datetime import datetime -from fastapi import APIRouter, Request -from pydantic import BaseModel +import sys as _sys -from core.middleware import require_admin -from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool -from src.constants import VAULT_FILE as _VAULT_FILE +from routes.vault import vault_routes as _canonical # noqa: F401 -logger = logging.getLogger(__name__) - -VAULT_FILE = Path(_VAULT_FILE) - - -def _find_bw() -> str: - """Locate the bw binary, checking PATH and common npm-global locations. - - On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by - which_tool via PATHEXT. - """ - p = which_tool("bw") - if p: - return p - if IS_WINDOWS: - appdata = os.environ.get("APPDATA", os.path.expanduser("~")) - for candidate in ( - os.path.join(appdata, "npm", "bw.cmd"), - os.path.join(appdata, "npm", "bw.exe"), - ): - if os.path.isfile(candidate): - return candidate - return "bw" - home = os.path.expanduser("~") - for candidate in ( - f"{home}/.npm-global/bin/bw", - f"{home}/.nvm/versions/node/*/bin/bw", - "/usr/local/bin/bw", - "/opt/homebrew/bin/bw", - ): - if "*" in candidate: - import glob - for m in glob.glob(candidate): - if os.path.isfile(m) and os.access(m, os.X_OK): - return m - elif os.path.isfile(candidate) and os.access(candidate, os.X_OK): - return candidate - return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below) - - -def _load_config() -> dict: - if VAULT_FILE.exists(): - try: - data = json.loads(VAULT_FILE.read_text(encoding="utf-8")) - return data if isinstance(data, dict) else {} - except Exception: - pass - return {} - - -def _save_config(cfg: dict): - VAULT_FILE.parent.mkdir(parents=True, exist_ok=True) - VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8") - # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir - # is ACL-restricted already). - safe_chmod(str(VAULT_FILE), 0o600) - - -async def _run_bw(args: list, session: str = None, input_text: str = None, - bw_password: str = None) -> tuple: - env = {} - env.update(os.environ) - if session: - env["BW_SESSION"] = session - # Secrets must never be passed as argv — process arguments are world-readable - # via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv - # support for bw commands that need it; unlock/login callers should prefer - # stdin so the master password is not left in the child environment either. - if bw_password is not None: - env["BW_PASSWORD"] = bw_password - bw_path = _find_bw() - try: - proc = await asyncio.create_subprocess_exec( - bw_path, *args, - stdin=asyncio.subprocess.PIPE if input_text else None, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - except FileNotFoundError: - return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127 - except Exception as e: - return "", f"Failed to launch bw: {e}", 1 - try: - stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None) - except Exception as e: - return "", f"bw subprocess error: {e}", 1 - return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode - - -class VaultConfig(BaseModel): - server_url: str = "" - email: str = "" - - -class VaultUnlockRequest(BaseModel): - master_password: str - - -class VaultLoginRequest(BaseModel): - email: str - master_password: str - - -def setup_vault_routes(): - router = APIRouter(prefix="/api/vault", tags=["vault"]) - - @router.get("/config") - async def get_config(request: Request): - """Return vault config (no sensitive fields).""" - require_admin(request) - cfg = _load_config() - return { - "server_url": cfg.get("server_url", ""), - "email": cfg.get("email", ""), - "unlocked": bool(cfg.get("session")), - "unlocked_at": cfg.get("unlocked_at", ""), - "bw_installed": await _check_bw_installed(), - } - - @router.post("/config") - async def save_config(req: VaultConfig, request: Request): - """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden.""" - require_admin(request) - cfg = _load_config() - cfg["server_url"] = req.server_url.strip().rstrip("/") - cfg["email"] = req.email.strip() - - if cfg["server_url"]: - _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]]) - if rc != 0: - return {"ok": False, "error": f"bw config failed: {stderr[:300]}"} - - _save_config(cfg) - return {"ok": True} - - @router.post("/login") - async def login(req: VaultLoginRequest, request: Request): - """Log in to Vaultwarden (required once per account).""" - require_admin(request) - cfg = _load_config() - # Update email - cfg["email"] = req.email - _save_config(cfg) - - stdout, stderr, rc = await _run_bw( - ["login", req.email, "--raw"], - input_text=req.master_password + "\n", - ) - if rc != 0: - # Already logged in is OK - if "already logged in" in stderr.lower(): - return {"ok": True, "already": True} - return {"ok": False, "error": f"Login failed: {stderr[:300]}"} - # bw login --raw prints session key on success (when 2FA disabled) - if stdout: - cfg["session"] = stdout - cfg["unlocked_at"] = datetime.utcnow().isoformat() - _save_config(cfg) - return {"ok": True} - - @router.post("/unlock") - async def unlock(req: VaultUnlockRequest, request: Request): - """Unlock the vault and save the session key.""" - require_admin(request) - # Pass the master password on stdin, not argv. argv is visible through - # `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in - # the child process environment. - stdout, stderr, rc = await _run_bw( - ["unlock", "--raw"], - input_text=req.master_password + "\n", - ) - if rc != 0: - return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"} - session = stdout.strip() - if not session: - return {"ok": False, "error": "bw returned empty session"} - cfg = _load_config() - cfg["session"] = session - cfg["unlocked_at"] = datetime.utcnow().isoformat() - _save_config(cfg) - return {"ok": True, "message": "Vault unlocked"} - - @router.post("/lock") - async def lock(request: Request): - """Lock the vault (clear session from config).""" - require_admin(request) - cfg = _load_config() - cfg.pop("session", None) - cfg.pop("unlocked_at", None) - _save_config(cfg) - # Also tell bw to lock - await _run_bw(["lock"]) - return {"ok": True, "message": "Vault locked"} - - @router.post("/logout") - async def logout(request: Request): - """Log out of the Bitwarden CLI completely.""" - require_admin(request) - await _run_bw(["logout"]) - cfg = _load_config() - cfg.pop("session", None) - cfg.pop("email", None) - cfg.pop("unlocked_at", None) - _save_config(cfg) - return {"ok": True} - - return router - - -async def _check_bw_installed() -> bool: - try: - proc = await asyncio.create_subprocess_exec( - _find_bw(), "--version", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - await proc.communicate() - return proc.returncode == 0 - except Exception: - return False +_sys.modules[__name__] = _canonical diff --git a/tests/test_vault_routes_shim.py b/tests/test_vault_routes_shim.py new file mode 100644 index 000000000..9577395f7 --- /dev/null +++ b/tests/test_vault_routes_shim.py @@ -0,0 +1,11 @@ +"""Regression test for the vault route shim (slice 2k, #4082/#4071).""" + +import importlib + +import routes.vault_routes as _shim_vault # noqa: F401 + + +def test_legacy_and_canonical_vault_module_are_same_object(): + legacy = importlib.import_module("routes.vault_routes") + canonical = importlib.import_module("routes.vault.vault_routes") + assert legacy is canonical From fb8c391a8893be254a9ce4e2e954ea297c573554 Mon Sep 17 00:00:00 2001 From: "Tal.Yuan" <ydonghao@users.noreply.github.com> Date: Tue, 4 Aug 2026 02:44:31 +0800 Subject: [PATCH 31/43] refactor(routes): move webhook domain into routes/webhook/ subpackage (#5781) Slice 2l of the route-domain reorganization (#4082/#4071). Moves webhook_routes.py into routes/webhook/, leaving a backward-compat sys.modules shim. Pure file reorganization, no behavior change. One source-introspection test repointed (test_api_chat_security.py). --- app.py | 2 +- routes/webhook/__init__.py | 5 + routes/webhook/webhook_routes.py | 395 +++++++++++++++++++++++++++++ routes/webhook_routes.py | 403 +----------------------------- tests/test_api_chat_security.py | 2 +- tests/test_webhook_routes_shim.py | 11 + 6 files changed, 425 insertions(+), 393 deletions(-) create mode 100644 routes/webhook/__init__.py create mode 100644 routes/webhook/webhook_routes.py create mode 100644 tests/test_webhook_routes_shim.py diff --git a/app.py b/app.py index 5fb2da54d..c85d425fb 100644 --- a/app.py +++ b/app.py @@ -820,7 +820,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr) logger.info("AI interaction tools initialized (session, memory, RAG, UI control)") # Webhooks -from routes.webhook_routes import setup_webhook_routes +from routes.webhook.webhook_routes import setup_webhook_routes app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager)) # API Tokens diff --git a/routes/webhook/__init__.py b/routes/webhook/__init__.py new file mode 100644 index 000000000..e51389e3a --- /dev/null +++ b/routes/webhook/__init__.py @@ -0,0 +1,5 @@ +"""Webhook route domain package (slice 2l, #4082/#4071). + +Contains webhook_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/webhook_routes.py re-exports from here. +""" diff --git a/routes/webhook/webhook_routes.py b/routes/webhook/webhook_routes.py new file mode 100644 index 000000000..8d3a704c6 --- /dev/null +++ b/routes/webhook/webhook_routes.py @@ -0,0 +1,395 @@ +"""Webhook, API Token, and sync chat routes.""" + +import uuid +import logging +from typing import Optional + +import httpx +from fastapi import APIRouter, HTTPException, Request, Form +from pydantic import BaseModel, Field + +from core.database import SessionLocal, Webhook, ModelEndpoint +from src.auth_helpers import owner_filter +from src.url_security import validate_public_http_url +from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api", tags=["webhooks"]) + +# Input limits +MAX_NAME_LEN = 100 +MAX_URL_LEN = 2048 +MAX_SECRET_LEN = 256 +MAX_MESSAGE_LEN = 32_000 + + +from core.middleware import require_admin as _require_admin + + +def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]): + """First enabled ModelEndpoint visible to token_owner — their own rows plus + legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would + let a chat-scoped token fall back onto another user's private endpoint and + silently spend that owner's API key/quota. Prefer owner rows before shared + rows. Fails closed to null-owner rows only when token_owner is absent. + Does not validate base_url — admin-configured local/LAN endpoints remain allowed. + """ + query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712 + if token_owner: + query = owner_filter(query, ModelEndpoint, token_owner) + return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first() + return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711 + + +def _caller_owns_session(sess_owner, caller) -> bool: + """Strict session-ownership gate for the token-authenticated sync-chat + endpoint (`POST /api/v1/chat`). + + Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner + gates in notes/calendar/gallery: a caller may resume a session ONLY when + its owner matches them exactly. A null/empty session owner (legacy or + migrated rows) is deliberately NOT resumable by an arbitrary token — the + old ``sess_owner and sess_owner != caller`` form skipped the check whenever + ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile + device) could resume such a session, inject a message, and read back its + history and reuse the owner's endpoint credentials. Fail closed: an + unresolvable caller also returns False. + """ + if not caller: + return False + return sess_owner == caller + + +def setup_webhook_routes( + webhook_manager: WebhookManager, + auth_manager, + session_manager=None, + api_key_manager=None, +) -> APIRouter: + + @router.get("/webhooks") + def list_webhooks(request: Request): + _require_admin(request) + db = SessionLocal() + try: + hooks = db.query(Webhook).all() + return [ + { + "id": w.id, + "name": w.name, + "url": w.url, + "has_secret": bool(w.secret), + "events": w.events.split(",") if w.events else [], + "is_active": w.is_active, + "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None, + "last_status_code": w.last_status_code, + "last_error": w.last_error, + "created_at": w.created_at.isoformat() if w.created_at else None, + } + for w in hooks + ] + finally: + db.close() + + @router.post("/webhooks") + def create_webhook( + request: Request, + name: str = Form(""), + url: str = Form(""), + secret: str = Form(""), + events: str = Form(""), + ): + _require_admin(request) + name = name.strip()[:MAX_NAME_LEN] + if not name: + raise HTTPException(400, "Webhook name is required") + try: + url = validate_webhook_url(url) + except ValueError as e: + raise HTTPException(400, str(e)) + try: + events = validate_events(events) + except ValueError as e: + raise HTTPException(400, str(e)) + + secret_val = secret.strip()[:MAX_SECRET_LEN] or None + # Encrypt the secret at rest using the same Fernet key as API keys + encrypted_secret = None + if secret_val and api_key_manager: + encrypted_secret = api_key_manager.encrypt_api_key(secret_val) + elif secret_val: + encrypted_secret = secret_val # Fallback if no encryption available + + webhook_id = str(uuid.uuid4())[:8] + db = SessionLocal() + try: + db.add(Webhook( + id=webhook_id, + name=name, + url=url, + secret=encrypted_secret, + events=events, + is_active=True, + )) + db.commit() + finally: + db.close() + + return {"id": webhook_id, "name": name} + + @router.post("/webhooks/{webhook_id}/test") + async def test_webhook(request: Request, webhook_id: str): + _require_admin(request) + db = SessionLocal() + try: + wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() + if not wh: + raise HTTPException(404, "Webhook not found") + url, secret = wh.url, wh.secret + finally: + db.close() + + await webhook_manager.deliver_test(webhook_id, url, secret) + return {"status": "sent"} + + @router.patch("/webhooks/{webhook_id}") + def toggle_webhook(request: Request, webhook_id: str): + _require_admin(request) + db = SessionLocal() + try: + wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() + if not wh: + raise HTTPException(404, "Webhook not found") + wh.is_active = not wh.is_active + db.commit() + return {"id": webhook_id, "is_active": wh.is_active} + finally: + db.close() + + @router.delete("/webhooks/{webhook_id}") + def delete_webhook(request: Request, webhook_id: str): + _require_admin(request) + db = SessionLocal() + try: + deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete() + db.commit() + if not deleted: + raise HTTPException(404, "Webhook not found") + finally: + db.close() + return {"status": "deleted"} + + # ================================================================ + # Sync Chat Endpoint (for n8n / Make / Activepieces) + # ================================================================ + + # Known provider base URLs — auto-resolved from api_key prefix or model name + KNOWN_PROVIDERS = { + "deepseek": "https://api.deepseek.com/v1", + "openai": "https://api.openai.com/v1", + "mistral": "https://api.mistral.ai/v1", + "groq": "https://api.groq.com/openai/v1", + "together": "https://api.together.xyz/v1", + "openrouter": "https://openrouter.ai/api/v1", + "ollama": "https://ollama.com/api", + "opencode-zen": "https://opencode.ai/zen/v1", + "opencode-go": "https://opencode.ai/zen/go/v1", + "fireworks": "https://api.fireworks.ai/inference/v1", + "venice": "https://api.venice.ai/api/v1", + "kimi-code": "https://api.kimi.com/coding/v1", + "kimicode": "https://api.kimi.com/coding/v1", + } + + # Model prefix → provider mapping for auto-detection + MODEL_PROVIDER_MAP = { + "deepseek": "deepseek", + "gpt-": "openai", + "o1": "openai", + "o3": "openai", + "o4": "openai", + "mistral": "mistral", + "llama": "groq", + "mixtral": "groq", + "kimi-for-coding": "kimi-code", + "kimi": "kimi-code", + } + + def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]: + """Try to auto-resolve a base URL from provider name or model prefix.""" + if provider and provider.lower() in KNOWN_PROVIDERS: + return KNOWN_PROVIDERS[provider.lower()] + if model: + model_lower = model.lower() + for prefix, prov in MODEL_PROVIDER_MAP.items(): + if model_lower.startswith(prefix): + return KNOWN_PROVIDERS[prov] + return None + + class SyncChatRequest(BaseModel): + message: str = Field(..., max_length=MAX_MESSAGE_LEN) + model: Optional[str] = Field(None, max_length=200) + session: Optional[str] = Field(None, max_length=100) + api_key: Optional[str] = Field(None, max_length=256) + base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN) + provider: Optional[str] = Field(None, max_length=50) + + @router.post("/v1/chat") + async def sync_chat(request: Request, body: SyncChatRequest): + if not getattr(request.state, "api_token", False): + raise HTTPException(403, "This endpoint requires an API token") + scopes = set(getattr(request.state, "api_token_scopes", []) or []) + if "chat" not in scopes: + raise HTTPException(403, "API token is not scoped for chat") + token_owner = getattr(request.state, "api_token_owner", None) + + from core.models import ChatMessage + from src.llm_core import llm_call_async + from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base + + message = body.message.strip() + if not message: + raise HTTPException(400, "Message is required") + + session_id = body.session + sess = None + + # --- Case 1: Resume an existing session --- + if session_id and session_manager: + try: + sess = session_manager.get_session(session_id) + except (KeyError, Exception): + raise HTTPException(404, "Session not found") + # SECURITY: verify the API-token's user owns this session — without + # this any token holder could resume any user's chat by passing its + # ID. The token's user is on request.state.user (set by API-token + # middleware); fall back to require_user if not present. + try: + from src.auth_helpers import get_current_user as _gcu + _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request) + except Exception: + _tok_user = None + # Strict ownership (see _caller_owns_session): fail closed so a + # null-owner / cross-owner session can't be resumed by an arbitrary + # chat-scoped token. + _sess_owner = getattr(sess, "owner", None) + if not _caller_owns_session(_sess_owner, _tok_user): + raise HTTPException(404, "Session not found") + + # --- Case 2: Direct API key + model (no pre-configured endpoint needed) --- + if not sess and body.api_key: + api_key = body.api_key.strip() + model = body.model or "deepseek-chat" + + # Validate only token-supplied direct base_url; auto-resolved known-provider + # URLs are not subject to extra local/LAN blocking beyond existing provider logic. + direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None + if direct_base_url: + try: + base_url = validate_public_http_url(direct_base_url) + except ValueError as e: + detail = str(e).replace("URL", "base_url", 1) + raise HTTPException(400, detail) + else: + base_url = _resolve_base_url(model, body.provider) + if not base_url: + raise HTTPException(400, + "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') " + "or provider ('deepseek', 'openai', 'groq', etc.)") + base_url = normalize_base(base_url) + endpoint_url = build_chat_url(base_url) + + if not session_manager: + raise HTTPException(500, "Session manager not available") + + sid = str(uuid.uuid4()) + sess = session_manager.create_session( + session_id=sid, name="API Chat", endpoint_url=endpoint_url, + model=model, owner=token_owner, + ) + sess.headers = build_headers(api_key, base_url) + session_manager.save_sessions() + session_id = sid + + # --- Case 3: Fall back to first configured ModelEndpoint --- + if not sess: + db = SessionLocal() + try: + ep = _select_api_chat_fallback_endpoint(db, token_owner) + finally: + db.close() + + if not ep: + raise HTTPException(400, + "No session, api_key, or configured endpoints. " + "Pass api_key + model, or configure an endpoint in Admin.") + + base_url = normalize_base(ep.base_url) + endpoint_url = build_chat_url(base_url) + model = body.model or "auto" + api_key = ep.api_key + if getattr(ep, "provider_auth_id", None): + try: + from src.endpoint_resolver import resolve_endpoint_runtime + base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner) + endpoint_url = build_chat_url(base_url) + except Exception: + raise HTTPException(500, "Could not resolve endpoint credentials") + + if model == "auto": + try: + async with httpx.AsyncClient(timeout=5) as client: + models_url = build_models_url(base_url) + hdrs = build_headers(api_key, base_url) + if models_url: + resp = await client.get(models_url, headers=hdrs) + resp.raise_for_status() + data = resp.json() + items = data if isinstance(data, list) else (data.get("data") or []) + ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")] + if not ids and isinstance(data, dict): + ids = [ + m.get("name") or m.get("model") + for m in (data.get("models") or []) + if m.get("name") or m.get("model") + ] + else: + import json as _json + ids = _json.loads(ep.cached_models or "[]") + model = ids[0] if ids else "auto" + except Exception: + raise HTTPException(500, "Could not discover models from endpoint") + + if not session_manager: + raise HTTPException(500, "Session manager not available") + + sid = str(uuid.uuid4()) + sess = session_manager.create_session( + session_id=sid, name="API Chat", endpoint_url=endpoint_url, + model=model, owner=token_owner, + ) + if api_key: + sess.headers = build_headers(api_key, base_url) + session_manager.save_sessions() + session_id = sid + + # --- Send message and get response --- + sess.add_message(ChatMessage("user", message)) + + messages = [{"role": m.role, "content": m.content} for m in sess.history] + + reply = await llm_call_async( + sess.endpoint_url, sess.model, messages, + headers=sess.headers, timeout=120, + ) + sess.add_message(ChatMessage("assistant", reply)) + session_manager.save_sessions() + + webhook_manager.fire_and_forget("chat.completed", { + "session_id": session_id, "model": sess.model, + "user_message": message[:2000], "response": reply[:2000], + }) + + return {"response": reply, "session_id": session_id, "model": sess.model} + + return router diff --git a/routes/webhook_routes.py b/routes/webhook_routes.py index 8d3a704c6..7c5e0453e 100644 --- a/routes/webhook_routes.py +++ b/routes/webhook_routes.py @@ -1,395 +1,16 @@ -"""Webhook, API Token, and sync chat routes.""" +"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py. -import uuid -import logging -from typing import Optional +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``, +``importlib.import_module("routes.webhook_routes")``, and the +``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod, +...)`` pattern used by test_null_owner_gates.py all operate on the *same* +object. Keeps existing import paths working after slice 2l (#4082/#4071). +Source-introspection tests read the canonical file by path. +""" -import httpx -from fastapi import APIRouter, HTTPException, Request, Form -from pydantic import BaseModel, Field +import sys as _sys -from core.database import SessionLocal, Webhook, ModelEndpoint -from src.auth_helpers import owner_filter -from src.url_security import validate_public_http_url -from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events +from routes.webhook import webhook_routes as _canonical # noqa: F401 -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/api", tags=["webhooks"]) - -# Input limits -MAX_NAME_LEN = 100 -MAX_URL_LEN = 2048 -MAX_SECRET_LEN = 256 -MAX_MESSAGE_LEN = 32_000 - - -from core.middleware import require_admin as _require_admin - - -def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]): - """First enabled ModelEndpoint visible to token_owner — their own rows plus - legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would - let a chat-scoped token fall back onto another user's private endpoint and - silently spend that owner's API key/quota. Prefer owner rows before shared - rows. Fails closed to null-owner rows only when token_owner is absent. - Does not validate base_url — admin-configured local/LAN endpoints remain allowed. - """ - query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712 - if token_owner: - query = owner_filter(query, ModelEndpoint, token_owner) - return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first() - return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711 - - -def _caller_owns_session(sess_owner, caller) -> bool: - """Strict session-ownership gate for the token-authenticated sync-chat - endpoint (`POST /api/v1/chat`). - - Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner - gates in notes/calendar/gallery: a caller may resume a session ONLY when - its owner matches them exactly. A null/empty session owner (legacy or - migrated rows) is deliberately NOT resumable by an arbitrary token — the - old ``sess_owner and sess_owner != caller`` form skipped the check whenever - ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile - device) could resume such a session, inject a message, and read back its - history and reuse the owner's endpoint credentials. Fail closed: an - unresolvable caller also returns False. - """ - if not caller: - return False - return sess_owner == caller - - -def setup_webhook_routes( - webhook_manager: WebhookManager, - auth_manager, - session_manager=None, - api_key_manager=None, -) -> APIRouter: - - @router.get("/webhooks") - def list_webhooks(request: Request): - _require_admin(request) - db = SessionLocal() - try: - hooks = db.query(Webhook).all() - return [ - { - "id": w.id, - "name": w.name, - "url": w.url, - "has_secret": bool(w.secret), - "events": w.events.split(",") if w.events else [], - "is_active": w.is_active, - "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None, - "last_status_code": w.last_status_code, - "last_error": w.last_error, - "created_at": w.created_at.isoformat() if w.created_at else None, - } - for w in hooks - ] - finally: - db.close() - - @router.post("/webhooks") - def create_webhook( - request: Request, - name: str = Form(""), - url: str = Form(""), - secret: str = Form(""), - events: str = Form(""), - ): - _require_admin(request) - name = name.strip()[:MAX_NAME_LEN] - if not name: - raise HTTPException(400, "Webhook name is required") - try: - url = validate_webhook_url(url) - except ValueError as e: - raise HTTPException(400, str(e)) - try: - events = validate_events(events) - except ValueError as e: - raise HTTPException(400, str(e)) - - secret_val = secret.strip()[:MAX_SECRET_LEN] or None - # Encrypt the secret at rest using the same Fernet key as API keys - encrypted_secret = None - if secret_val and api_key_manager: - encrypted_secret = api_key_manager.encrypt_api_key(secret_val) - elif secret_val: - encrypted_secret = secret_val # Fallback if no encryption available - - webhook_id = str(uuid.uuid4())[:8] - db = SessionLocal() - try: - db.add(Webhook( - id=webhook_id, - name=name, - url=url, - secret=encrypted_secret, - events=events, - is_active=True, - )) - db.commit() - finally: - db.close() - - return {"id": webhook_id, "name": name} - - @router.post("/webhooks/{webhook_id}/test") - async def test_webhook(request: Request, webhook_id: str): - _require_admin(request) - db = SessionLocal() - try: - wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() - if not wh: - raise HTTPException(404, "Webhook not found") - url, secret = wh.url, wh.secret - finally: - db.close() - - await webhook_manager.deliver_test(webhook_id, url, secret) - return {"status": "sent"} - - @router.patch("/webhooks/{webhook_id}") - def toggle_webhook(request: Request, webhook_id: str): - _require_admin(request) - db = SessionLocal() - try: - wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() - if not wh: - raise HTTPException(404, "Webhook not found") - wh.is_active = not wh.is_active - db.commit() - return {"id": webhook_id, "is_active": wh.is_active} - finally: - db.close() - - @router.delete("/webhooks/{webhook_id}") - def delete_webhook(request: Request, webhook_id: str): - _require_admin(request) - db = SessionLocal() - try: - deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete() - db.commit() - if not deleted: - raise HTTPException(404, "Webhook not found") - finally: - db.close() - return {"status": "deleted"} - - # ================================================================ - # Sync Chat Endpoint (for n8n / Make / Activepieces) - # ================================================================ - - # Known provider base URLs — auto-resolved from api_key prefix or model name - KNOWN_PROVIDERS = { - "deepseek": "https://api.deepseek.com/v1", - "openai": "https://api.openai.com/v1", - "mistral": "https://api.mistral.ai/v1", - "groq": "https://api.groq.com/openai/v1", - "together": "https://api.together.xyz/v1", - "openrouter": "https://openrouter.ai/api/v1", - "ollama": "https://ollama.com/api", - "opencode-zen": "https://opencode.ai/zen/v1", - "opencode-go": "https://opencode.ai/zen/go/v1", - "fireworks": "https://api.fireworks.ai/inference/v1", - "venice": "https://api.venice.ai/api/v1", - "kimi-code": "https://api.kimi.com/coding/v1", - "kimicode": "https://api.kimi.com/coding/v1", - } - - # Model prefix → provider mapping for auto-detection - MODEL_PROVIDER_MAP = { - "deepseek": "deepseek", - "gpt-": "openai", - "o1": "openai", - "o3": "openai", - "o4": "openai", - "mistral": "mistral", - "llama": "groq", - "mixtral": "groq", - "kimi-for-coding": "kimi-code", - "kimi": "kimi-code", - } - - def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]: - """Try to auto-resolve a base URL from provider name or model prefix.""" - if provider and provider.lower() in KNOWN_PROVIDERS: - return KNOWN_PROVIDERS[provider.lower()] - if model: - model_lower = model.lower() - for prefix, prov in MODEL_PROVIDER_MAP.items(): - if model_lower.startswith(prefix): - return KNOWN_PROVIDERS[prov] - return None - - class SyncChatRequest(BaseModel): - message: str = Field(..., max_length=MAX_MESSAGE_LEN) - model: Optional[str] = Field(None, max_length=200) - session: Optional[str] = Field(None, max_length=100) - api_key: Optional[str] = Field(None, max_length=256) - base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN) - provider: Optional[str] = Field(None, max_length=50) - - @router.post("/v1/chat") - async def sync_chat(request: Request, body: SyncChatRequest): - if not getattr(request.state, "api_token", False): - raise HTTPException(403, "This endpoint requires an API token") - scopes = set(getattr(request.state, "api_token_scopes", []) or []) - if "chat" not in scopes: - raise HTTPException(403, "API token is not scoped for chat") - token_owner = getattr(request.state, "api_token_owner", None) - - from core.models import ChatMessage - from src.llm_core import llm_call_async - from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base - - message = body.message.strip() - if not message: - raise HTTPException(400, "Message is required") - - session_id = body.session - sess = None - - # --- Case 1: Resume an existing session --- - if session_id and session_manager: - try: - sess = session_manager.get_session(session_id) - except (KeyError, Exception): - raise HTTPException(404, "Session not found") - # SECURITY: verify the API-token's user owns this session — without - # this any token holder could resume any user's chat by passing its - # ID. The token's user is on request.state.user (set by API-token - # middleware); fall back to require_user if not present. - try: - from src.auth_helpers import get_current_user as _gcu - _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request) - except Exception: - _tok_user = None - # Strict ownership (see _caller_owns_session): fail closed so a - # null-owner / cross-owner session can't be resumed by an arbitrary - # chat-scoped token. - _sess_owner = getattr(sess, "owner", None) - if not _caller_owns_session(_sess_owner, _tok_user): - raise HTTPException(404, "Session not found") - - # --- Case 2: Direct API key + model (no pre-configured endpoint needed) --- - if not sess and body.api_key: - api_key = body.api_key.strip() - model = body.model or "deepseek-chat" - - # Validate only token-supplied direct base_url; auto-resolved known-provider - # URLs are not subject to extra local/LAN blocking beyond existing provider logic. - direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None - if direct_base_url: - try: - base_url = validate_public_http_url(direct_base_url) - except ValueError as e: - detail = str(e).replace("URL", "base_url", 1) - raise HTTPException(400, detail) - else: - base_url = _resolve_base_url(model, body.provider) - if not base_url: - raise HTTPException(400, - "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') " - "or provider ('deepseek', 'openai', 'groq', etc.)") - base_url = normalize_base(base_url) - endpoint_url = build_chat_url(base_url) - - if not session_manager: - raise HTTPException(500, "Session manager not available") - - sid = str(uuid.uuid4()) - sess = session_manager.create_session( - session_id=sid, name="API Chat", endpoint_url=endpoint_url, - model=model, owner=token_owner, - ) - sess.headers = build_headers(api_key, base_url) - session_manager.save_sessions() - session_id = sid - - # --- Case 3: Fall back to first configured ModelEndpoint --- - if not sess: - db = SessionLocal() - try: - ep = _select_api_chat_fallback_endpoint(db, token_owner) - finally: - db.close() - - if not ep: - raise HTTPException(400, - "No session, api_key, or configured endpoints. " - "Pass api_key + model, or configure an endpoint in Admin.") - - base_url = normalize_base(ep.base_url) - endpoint_url = build_chat_url(base_url) - model = body.model or "auto" - api_key = ep.api_key - if getattr(ep, "provider_auth_id", None): - try: - from src.endpoint_resolver import resolve_endpoint_runtime - base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner) - endpoint_url = build_chat_url(base_url) - except Exception: - raise HTTPException(500, "Could not resolve endpoint credentials") - - if model == "auto": - try: - async with httpx.AsyncClient(timeout=5) as client: - models_url = build_models_url(base_url) - hdrs = build_headers(api_key, base_url) - if models_url: - resp = await client.get(models_url, headers=hdrs) - resp.raise_for_status() - data = resp.json() - items = data if isinstance(data, list) else (data.get("data") or []) - ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")] - if not ids and isinstance(data, dict): - ids = [ - m.get("name") or m.get("model") - for m in (data.get("models") or []) - if m.get("name") or m.get("model") - ] - else: - import json as _json - ids = _json.loads(ep.cached_models or "[]") - model = ids[0] if ids else "auto" - except Exception: - raise HTTPException(500, "Could not discover models from endpoint") - - if not session_manager: - raise HTTPException(500, "Session manager not available") - - sid = str(uuid.uuid4()) - sess = session_manager.create_session( - session_id=sid, name="API Chat", endpoint_url=endpoint_url, - model=model, owner=token_owner, - ) - if api_key: - sess.headers = build_headers(api_key, base_url) - session_manager.save_sessions() - session_id = sid - - # --- Send message and get response --- - sess.add_message(ChatMessage("user", message)) - - messages = [{"role": m.role, "content": m.content} for m in sess.history] - - reply = await llm_call_async( - sess.endpoint_url, sess.model, messages, - headers=sess.headers, timeout=120, - ) - sess.add_message(ChatMessage("assistant", reply)) - session_manager.save_sessions() - - webhook_manager.fire_and_forget("chat.completed", { - "session_id": session_id, "model": sess.model, - "user_message": message[:2000], "response": reply[:2000], - }) - - return {"response": reply, "session_id": session_id, "model": sess.model} - - return router +_sys.modules[__name__] = _canonical diff --git a/tests/test_api_chat_security.py b/tests/test_api_chat_security.py index 7dcec324e..d92a31620 100644 --- a/tests/test_api_chat_security.py +++ b/tests/test_api_chat_security.py @@ -76,7 +76,7 @@ def _load_webhook_routes_for_test(monkeypatch): module_name = "routes.webhook_routes_under_test" spec = importlib.util.spec_from_file_location( module_name, - Path(__file__).resolve().parent.parent / "routes" / "webhook_routes.py", + Path(__file__).resolve().parent.parent / "routes" / "webhook" / "webhook_routes.py", ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) diff --git a/tests/test_webhook_routes_shim.py b/tests/test_webhook_routes_shim.py new file mode 100644 index 000000000..f6312e8e6 --- /dev/null +++ b/tests/test_webhook_routes_shim.py @@ -0,0 +1,11 @@ +"""Regression test for the webhook route shim (slice 2l, #4082/#4071).""" + +import importlib + +import routes.webhook_routes as _shim_webhook # noqa: F401 + + +def test_legacy_and_canonical_webhook_module_are_same_object(): + legacy = importlib.import_module("routes.webhook_routes") + canonical = importlib.import_module("routes.webhook.webhook_routes") + assert legacy is canonical From bb719f217a77b19d89d26f96168cf463ab73b6ba Mon Sep 17 00:00:00 2001 From: "Tal.Yuan" <ydonghao@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:54:55 +0800 Subject: [PATCH 32/43] refactor(routes): move document domain into routes/document/ subpackage (#5885) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice 2m of the route-domain reorganization (#4082/#4071, per specs/architecture-runtime-inventory.md §6.3). Moves document_routes.py (1810 lines) and document_helpers.py (243 lines) into routes/document/, leaving backward-compat sys.modules shims at the old paths. Pure file reorganization, no behavior change. Both shims use sys.modules replacement so the `import ... as droutes` + `droutes.SessionLocal = ...` / `monkeypatch.setattr(droutes, ...)` pattern in multiple tests, and the `sys.modules.pop("routes.document_helpers")` + re-import pattern in test_security_regressions.py, all reach the canonical modules. The canonical document_routes.py imports helpers from the canonical path (routes.document.document_helpers), not the legacy shim. Three source-introspection test sites repointed to the new canonical path: - test_imap_mailbox_quoting.py - test_model_helper_owner_scope.py - test_vision_owner_scope.py (shared with other domains; document entry repointed) Adds tests/test_document_routes_shim.py to pin the sys.modules shim contract for both modules. Verified: compileall clean; full suite 4789 passed, 3 skipped. --- app.py | 2 +- routes/document/__init__.py | 6 + routes/document/document_helpers.py | 243 ++++ routes/document/document_routes.py | 1810 +++++++++++++++++++++++ routes/document_helpers.py | 249 +--- routes/document_routes.py | 1819 +----------------------- tests/test_document_routes_shim.py | 29 + tests/test_imap_mailbox_quoting.py | 2 +- tests/test_model_helper_owner_scope.py | 2 +- tests/test_vision_owner_scope.py | 2 +- 10 files changed, 2115 insertions(+), 2049 deletions(-) create mode 100644 routes/document/__init__.py create mode 100644 routes/document/document_helpers.py create mode 100644 routes/document/document_routes.py create mode 100644 tests/test_document_routes_shim.py diff --git a/app.py b/app.py index c85d425fb..8363ba4e9 100644 --- a/app.py +++ b/app.py @@ -739,7 +739,7 @@ app.include_router(setup_stt_routes(stt_service)) logger.info("STT service initialized (provider managed via settings)") # Documents (artifacts/canvas) -from routes.document_routes import setup_document_routes +from routes.document.document_routes import setup_document_routes document_router = setup_document_routes(session_manager, upload_handler) app.include_router(document_router) diff --git a/routes/document/__init__.py b/routes/document/__init__.py new file mode 100644 index 000000000..7f79ce1bb --- /dev/null +++ b/routes/document/__init__.py @@ -0,0 +1,6 @@ +"""Document route domain package (slice 2m, #4082/#4071). + +Contains document_routes.py and document_helpers.py, migrated from the flat +routes/ directory. Backward-compat shims at routes/document_routes.py and +routes/document_helpers.py re-export from here. +""" diff --git a/routes/document/document_helpers.py b/routes/document/document_helpers.py new file mode 100644 index 000000000..a0c2d08eb --- /dev/null +++ b/routes/document/document_helpers.py @@ -0,0 +1,243 @@ +"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py.""" + +"""Document routes — CRUD for living documents with version history.""" + +import logging +import os +import re +from typing import Any, Dict, Optional + +from fastapi import HTTPException, Request +from pydantic import BaseModel + +from core.database import Document, DocumentVersion +from core.database import Session as DbSession +from src.auth_helpers import _auth_disabled +from src.upload_handler import UploadHandler + +logger = logging.getLogger(__name__) + + +# ---- Request schemas ---- + +class DocumentCreate(BaseModel): + session_id: Optional[str] = None + title: str = "Untitled" + language: Optional[str] = None + content: str = "" + +class DocumentUpdate(BaseModel): + content: str + summary: Optional[str] = None + force_version: bool = False + +class DocumentPatch(BaseModel): + title: Optional[str] = None + language: Optional[str] = None + session_id: Optional[str] = None # link/unlink document to a session + + +# ---- Helpers ---- + +def _doc_to_dict(doc: Document) -> Dict[str, Any]: + return { + "id": doc.id, + "session_id": doc.session_id, + "title": doc.title, + "language": doc.language, + "current_content": doc.current_content, + "version_count": doc.version_count, + "is_active": doc.is_active, + "archived": bool(getattr(doc, "archived", False)), + "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None, + "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None, + # Source-email provenance (set when doc was created from an email + # attachment) — drives the "Send signed reply" menu item. + "source_email_uid": getattr(doc, "source_email_uid", None), + "source_email_folder": getattr(doc, "source_email_folder", None), + "source_email_account_id": getattr(doc, "source_email_account_id", None), + "source_email_message_id": getattr(doc, "source_email_message_id", None), + } + +def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]: + return { + "id": v.id, + "document_id": v.document_id, + "version_number": v.version_number, + "content": v.content, + "summary": v.summary, + "source": v.source, + "created_at": v.created_at.isoformat() if v.created_at else None, + } + + +def _verify_doc_owner(db, doc: Document, user: str): + """Verify `user` owns this document. Raise 404 if not. + + Documents now carry their own `owner` column, so a doc whose session + was deleted (session_id → NULL) can still prove ownership and stay + openable / cloneable. We trust that column first and only fall back to + the session join for any not-yet-backfilled legacy row. + """ + if user is None: + if _auth_disabled(): + return # Single-user / no-auth mode: allow access + raise HTTPException(403, "Authentication required") + if doc.owner is not None: + if doc.owner != user: + raise HTTPException(404, "Document not found") + return + # Legacy fallback: derive ownership from the linked session. + if not doc.session_id: + raise HTTPException(404, "Document not found") + session = db.query(DbSession).filter(DbSession.id == doc.session_id).first() + if not session or session.owner != user: + raise HTTPException(404, "Document not found") + + +def _owner_session_filter(q, user): + """Restrict a documents query to those owned by `user`. + + Documents now carry their own `owner` column (backfilled at boot from + the linked session, or assigned to the admin user for legacy/orphaned + docs). We filter on that directly rather than on a session join, so a + document whose session was deleted (session_id → NULL) still shows up + for its owner instead of silently vanishing from the Library + search. + + The owner backfill runs in init_db before the app serves requests, so + by the time this filter is live there are no NULL-owner rows to leak; + we therefore match the owner strictly for authenticated callers.""" + if not user: + if user == "" or _auth_disabled(): + return q + return q.filter(False) + return q.filter(Document.owner == user) + + + +def _slug(name: str) -> str: + """Filesystem-friendly version of a document title. + + Whitespace becomes underscores; other unsafe punctuation is dropped. + Preserves letters, digits, dot, hyphen, underscore. Idempotent. + """ + import re as _re + s = (name or "").strip() + # Drop the trailing extension if the title happens to include one + s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE) + s = _re.sub(r'\s+', '_', s) + s = _re.sub(r'[^A-Za-z0-9._-]', '', s) + s = _re.sub(r'_+', '_', s).strip('_') + return s or "form" + + +# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units). +_PDF_RENDER_SCALE = 2.0 + + +def _upload_path_inside(upload_dir: str, path: str) -> bool: + base = os.path.realpath(upload_dir) + p = os.path.realpath(path) + try: + return os.path.commonpath([base, p]) == base + except Exception: + return False + + +def _resolve_user_upload_path( + upload_handler: Any, + upload_id: str, + owner: Optional[str], + auth_manager=None, +) -> Optional[str]: + """Resolve an upload id to a filesystem path the caller may read.""" + if upload_handler is None: + return None + resolved = upload_handler.resolve_upload( + upload_id, + owner=owner, + auth_manager=auth_manager, + ) + if not isinstance(resolved, dict) or not resolved: + return None + path = resolved.get("path") + upload_dir = getattr(upload_handler, "upload_dir", None) + if path and upload_dir and not _upload_path_inside(upload_dir, path): + logger.warning("Upload path outside upload directory: %s", path) + return None + return path + + +def _locate_upload( + upload_dir: str, + file_id: str, + owner: Optional[str] = None, + auth_manager=None, + upload_handler: Any = None, +): + """Find an upload by its filename ID via UploadHandler.resolve_upload.""" + if upload_handler is None: + from src.upload_handler import UploadHandler + + base_dir = os.path.dirname(os.path.abspath(upload_dir)) + upload_handler = UploadHandler(base_dir, upload_dir) + return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager) + + +def _assert_pdf_marker_upload_owned( + request: Request, + content: str, + user: Optional[str], + upload_handler: Any, +) -> None: + """Reject document content whose pdf_source marker points at another user's upload.""" + if upload_handler is None: + return + from src.pdf_form_doc import find_source_upload_id + + upload_id = find_source_upload_id(content or "") + if not upload_id: + return + auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) + if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager): + raise HTTPException( + 400, + "Document PDF marker references an upload you do not own", + ) + + +def _derive_title(content: str) -> str: + """Derive a title from document content.""" + import re + if not isinstance(content, str): + return "Untitled" + text = content.strip() + if not text: + return "Untitled" + + # Markdown header + md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE) + if md: + title = md.group(1).strip() + if len(title) > 50: + title = title[:48] + "…" + return title + + # HTML heading + html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE) + if html: + title = html.group(1).strip() + if len(title) > 50: + title = title[:48] + "…" + return title + + # First non-empty line (if short enough) + for line in text.split('\n'): + line = line.strip() + if line and 2 <= len(line) <= 60: + title = re.sub(r'[:#*`]+$', '', line).strip() + if title and len(title) > 50: + title = title[:48] + "…" + return title or "Untitled" + + return "Untitled" diff --git a/routes/document/document_routes.py b/routes/document/document_routes.py new file mode 100644 index 000000000..dae8b09fa --- /dev/null +++ b/routes/document/document_routes.py @@ -0,0 +1,1810 @@ +"""Document routes — CRUD for living documents with version history.""" + +import uuid +import logging +from datetime import datetime, timezone +from typing import Dict, Any, List, Optional + +from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form + +from sqlalchemy import case, func, or_ +from core.database import SessionLocal, Document, DocumentVersion +from core.database import Session as DbSession +from src.auth_helpers import get_current_user, _auth_disabled +from src.constants import MAIL_ATTACHMENTS_DIR +from src.upload_handler import reserve_upload_references + +logger = logging.getLogger(__name__) + + +def _get_session_or_404(db, session_id: str, user: Optional[str]): + session = db.query(DbSession).filter(DbSession.id == session_id).first() + if not session: + raise HTTPException(404, "Session not found") + if user and session.owner != user: + raise HTTPException(404, "Session not found") + return session + + +def _aggregate_language_facets(lang_rows): + """Sum document counts per display language for the library facet. + + NULL-language and explicit "text" rows share the "text" bucket (the + language filter treats them as one), so they must be ADDED. The old dict + comprehension keyed both to "text", silently overwriting one group and + undercounting the facet versus what the filter actually returns. + """ + out = {} + for lang, cnt in lang_rows: + key = lang or "text" + out[key] = out.get(key, 0) + cnt + return out + + +def _library_language_for_document(doc: Document) -> str: + """Return the display language used by the document library. + + PDF documents are stored as markdown wrappers so the editor can preserve + extracted text, form fields, and annotations. The library should still + identify them as PDFs instead of exposing that internal wrapper format. + """ + from src.pdf_form_doc import find_source_upload_id + + if find_source_upload_id(doc.current_content or ""): + return "pdf" + return doc.language or "text" + + +def _email_source_key(content: str) -> tuple[str, str]: + """Return the source email identity embedded in an email draft document.""" + import re + + text = content or "" + uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text) + folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text) + uid = (uid_m.group(1).strip() if uid_m else "") + folder = (folder_m.group(1).strip() if folder_m else "INBOX") + return uid, folder + + +from routes.document_helpers import ( + DocumentCreate, DocumentUpdate, DocumentPatch, + _doc_to_dict, _version_to_dict, + _verify_doc_owner, _owner_session_filter, + _slug, _resolve_user_upload_path, _assert_pdf_marker_upload_owned, _derive_title, + _PDF_RENDER_SCALE, +) + + +def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: + router = APIRouter(tags=["documents"]) + + def _reserve_document_uploads(user: Optional[str], content: str) -> None: + missing_id = reserve_upload_references(upload_handler, user, content) + if missing_id: + raise HTTPException( + 409, + f"Referenced upload is no longer available: {missing_id}", + ) + + def _locate_current_user_upload(request: Request, upload_id: str, user: Optional[str]): + if upload_handler is None: + return None + auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) + return _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager) + + def _load_pdf_viewer_fitz(): + from src.pdf_runtime import load_pymupdf_for_pdf_viewer + + try: + return load_pymupdf_for_pdf_viewer() + except RuntimeError as exc: + raise HTTPException(503, str(exc)) from exc + + # ---- POST /api/document ---- + @router.post("/api/document") + async def create_document(request: Request, req: DocumentCreate) -> Dict[str, Any]: + from src.auth_helpers import require_privilege + user = require_privilege(request, "can_use_documents") + db = SessionLocal() + try: + # session_id is optional: a doc can be a session-less "library" doc + # (e.g. files imported from the library) — session_id is nullable and + # the doc is owner-stamped, so it lives in the library on its own. + session = None + if req.session_id: + # Match the lenient ownership model the rest of the app uses + # (see _owner_filter): only block when an AUTHENTICATED user is + # writing into a DIFFERENT user's session. In single-user / + # unconfigured / localhost-bypass mode, falsey users preserve + # the existing lenient path. + session = _get_session_or_404(db, req.session_id, user) + + # If no language was supplied (e.g. cloning a doc whose language + # was never set), detect it from the content rather than storing + # NULL — which made the editor fall back to plain text. Defaults + # to markdown for prose. + language = req.language + if not language: + from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content + language = _sniff_doc_language(req.content) + else: + from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content + if _looks_like_email_document(req.content, req.title): + language = "email" + + _reserve_document_uploads(user, req.content) + _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler) + + # Reply drafts are keyed to the source email. If a UI/tool path tries + # to create a second draft for the same email in the same chat, + # update the existing draft instead so quoted thread history stays + # attached to the visible document. + if language == "email" and req.session_id: + source_uid, source_folder = _email_source_key(req.content) + if source_uid: + candidates = ( + db.query(Document) + .filter(Document.session_id == req.session_id) + .filter(Document.is_active == True) + .filter(Document.language == "email") + .order_by(Document.updated_at.desc()) + .limit(25) + .all() + ) + for existing in candidates: + old_uid, old_folder = _email_source_key(existing.current_content or "") + if old_uid != source_uid or old_folder != source_folder: + continue + merged = _coerce_email_document_content(existing.current_content or "", req.content) + if existing.current_content != merged: + new_ver = (existing.version_count or 1) + 1 + existing.current_content = merged + existing.title = req.title or existing.title + existing.version_count = new_ver + db.add(DocumentVersion( + id=str(uuid.uuid4()), + document_id=existing.id, + version_number=new_ver, + content=merged, + summary="Updated existing email draft", + source="user", + )) + db.commit() + db.refresh(existing) + return _doc_to_dict(existing) + + doc_id = str(uuid.uuid4()) + ver_id = str(uuid.uuid4()) + + doc = Document( + id=doc_id, + session_id=req.session_id, + title=req.title, + language=language, + current_content=req.content, + version_count=1, + is_active=True, + # Stamp ownership directly so the doc survives its session + # being deleted. Fall back to the session's owner when the + # request is unauthenticated (single-user / localhost bypass). + owner=user or (session.owner if session else None), + ) + ver = DocumentVersion( + id=ver_id, + document_id=doc_id, + version_number=1, + content=req.content, + summary="Initial version", + source="user", + ) + db.add(doc) + db.add(ver) + db.commit() + db.refresh(doc) + try: + from src.event_bus import fire_event + fire_event("document_created", doc.owner) + except Exception: + logger.debug("document_created event dispatch failed", exc_info=True) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"Failed to create document: {e}") + raise HTTPException(500, f"Failed to create document: {e}") + finally: + db.close() + + # ---- POST /api/documents/import-pdf ---- + @router.post("/api/documents/import-pdf") + async def import_pdf( + request: Request, + file: UploadFile = File(...), + session_id: Optional[str] = Form(None), + ) -> Dict[str, Any]: + """Upload a PDF and create the matching Document. + + Detects AcroForm fields — if any, creates a form-backed markdown doc + (clickable inputs in the PDF view). Otherwise creates a plain PDF doc + with a `pdf_source` marker so the viewer renders the pages without + overlays. + """ + from src.pdf_forms import has_form_fields, extract_fields + from src.pdf_form_doc import ( + save_field_sidecar, + create_form_markdown_document, + create_plain_pdf_document, + ) + from src.document_processor import _process_pdf, strip_pdf_content_marker + import os + + from src.auth_helpers import require_privilege + user = require_privilege(request, "can_use_documents") + + # session_id is optional — a library import isn't tied to a chat. When + # given, validate it; otherwise the PDF becomes a session-less library + # doc (the doc creators below already handle a missing session). + if session_id: + db = SessionLocal() + try: + _get_session_or_404(db, session_id, user) + finally: + db.close() + + if upload_handler is None: + raise HTTPException(500, "Upload handler not configured") + + client_ip = request.client.host if request.client else "unknown" + try: + meta = upload_handler.save_upload(file, client_ip, owner=user) + except HTTPException: + raise + except Exception as e: + logger.error(f"PDF import save_upload failed: {e}") + raise HTTPException(500, f"Upload failed: {e}") + + upload_id = meta["id"] + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(500, "Saved PDF could not be located") + + title = os.path.splitext(meta.get("original_name") or meta.get("name") or upload_id)[0] + try: + body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user)) + except Exception: + body_text = None + + is_form = False + try: + is_form = has_form_fields(pdf_path) + except Exception as e: + logger.warning(f"has_form_fields failed for {pdf_path}: {e}") + + if is_form: + fields = extract_fields(pdf_path) + save_field_sidecar(pdf_path, fields) + doc_id = create_form_markdown_document( + session_id=session_id, + fields=fields, + upload_id=upload_id, + title=title, + intro_text=body_text, + ) + else: + doc_id = create_plain_pdf_document( + session_id=session_id, + upload_id=upload_id, + title=title, + body_text=body_text, + ) + + if not doc_id: + raise HTTPException(500, "Failed to create document for PDF") + + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(500, "Created document not found") + # The PDF doc creators stamp owner from the session only; a + # session-less library import leaves owner NULL, which the Library's + # owner filter then hides. Stamp the requesting user so it shows. + if not doc.owner and user: + doc.owner = user + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + finally: + db.close() + + # ---- GET /api/documents/library ---- + @router.get("/api/documents/library") + async def documents_library( + request: Request, + search: Optional[str] = Query(None), + language: Optional[str] = Query(None), + sort: str = Query("recent"), + offset: int = Query(0, ge=0), + limit: int = Query(20, ge=1, le=50), + archived: bool = Query(False), + ) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + from sqlalchemy import or_ + pdf_marker_cond = or_( + Document.current_content.like('%<!-- pdf_source upload_id="%'), + Document.current_content.like('%<!-- pdf_form_source upload_id="%'), + ) + library_language_expr = case( + (pdf_marker_cond, "pdf"), + (Document.language.is_(None), "text"), + else_=Document.language, + ) + # Archived view shows ONLY archived docs; the default view excludes + # them (NULL = legacy rows that predate the column = not archived). + _arch_cond = (Document.archived == True) if archived else or_( + Document.archived == False, Document.archived.is_(None)) + # Language facet counts (owner-filtered). PDF documents are stored + # as markdown wrappers, so group by the library display language + # instead of the raw stored language. + lang_q = ( + db.query(library_language_expr, func.count(Document.id)) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == True).filter(_arch_cond) + ) + lang_q = _owner_session_filter(lang_q, user) + lang_rows = lang_q.group_by(library_language_expr).all() + languages = _aggregate_language_facets(lang_rows) + + # Session count (owner-filtered) + sc_q = ( + db.query(func.count(func.distinct(Document.session_id))) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == True).filter(_arch_cond) + ) + sc_q = _owner_session_filter(sc_q, user) + session_count = sc_q.scalar() + + # Base query + q = ( + db.query(Document, DbSession.name) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == True).filter(_arch_cond) + ) + q = _owner_session_filter(q, user) + + # Search filter — split on whitespace and require EACH term to + # match (title OR content). A single `%foo bar%` LIKE only matched + # the exact adjacent phrase, so any multi-word query with a space + # silently returned nothing. Per-term AND makes "machine learning" + # match docs containing both words regardless of position/order. + if search: + for tok in search.split(): + term = f"%{tok}%" + q = q.filter( + Document.title.ilike(term) | Document.current_content.ilike(term) + ) + + # Language filter. "pdf" is a display language derived from the + # source marker; "markdown" excludes those wrappers. + if language: + if language == "text": + q = q.filter((Document.language == None) | (Document.language == "text")) + elif language == "pdf": + q = q.filter(pdf_marker_cond) + else: + q = q.filter(Document.language == language) + if language == "markdown": + q = q.filter(~pdf_marker_cond) + + # Total before pagination + total = q.count() + + # Sorting + if sort == "oldest": + q = q.order_by(Document.created_at.asc()) + elif sort == "edits": + q = q.order_by(Document.version_count.desc()) + elif sort == "alpha": + q = q.order_by(Document.title.asc()) + else: # recent + q = q.order_by(Document.updated_at.desc()) + + rows = q.offset(offset).limit(limit).all() + + documents = [] + for doc, session_name in rows: + documents.append({ + "id": doc.id, + "session_id": doc.session_id, + "session_name": session_name, + "title": doc.title, + "language": _library_language_for_document(doc), + "preview": (doc.current_content or "")[:500], + "version_count": doc.version_count, + "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None, + "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None, + }) + + return { + "documents": documents, + "total": total, + "languages": languages, + "session_count": session_count, + } + except Exception as e: + logger.error(f"Failed to fetch document library: {e}") + raise HTTPException(500, f"Failed to fetch document library: {e}") + finally: + db.close() + + # ---- GET /api/documents/{session_id} ---- + @router.get("/api/documents/{session_id}") + async def list_documents(request: Request, session_id: str) -> List[Dict[str, Any]]: + user = get_current_user(request) + db = SessionLocal() + try: + if not user: + if not _auth_disabled(): + raise HTTPException(403, "Authentication required") + # v2 review HIGH-9: raise 403 explicitly when the caller + # can't see this session, instead of returning [] which the + # UI treats identically to "no docs" and silently masks + # auth failures. + _get_session_or_404(db, session_id, user) + q = db.query(Document).filter( + Document.session_id == session_id + ) + if user: + q = q.filter(or_(Document.owner == user, Document.owner.is_(None))) + docs = q.order_by(Document.created_at.desc()).all() + return [_doc_to_dict(d) for d in docs] + finally: + db.close() + + # ---- GET /api/document/{doc_id} ---- + @router.get("/api/document/{doc_id}") + async def get_document(request: Request, doc_id: str) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + return _doc_to_dict(doc) + finally: + db.close() + + # ---- POST /api/document/{doc_id}/archive — soft-archive / restore ---- + @router.post("/api/document/{doc_id}/archive") + async def archive_document(request: Request, doc_id: str, archived: bool = Query(True)) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + doc.archived = bool(archived) + db.commit() + return {"ok": True, "id": doc_id, "archived": doc.archived} + finally: + db.close() + + # ---- POST /api/document/{doc_id}/extract-pdf-text ---- + @router.post("/api/document/{doc_id}/extract-pdf-text") + async def extract_pdf_text(request: Request, doc_id: str) -> Dict[str, Any]: + """Re-run pypdf+VL text extraction against the PDF linked to this doc + and merge the result into the doc's markdown content. Idempotent — the + existing body (everything below the title heading) is replaced. + + Lets the AI see PDF contents for old docs that were imported before + text extraction was wired, plus for scanned/image-only PDFs where the + VL model picks up text the basic pypdf path missed.""" + import re + from src.document_processor import _process_pdf, strip_pdf_content_marker + from src.pdf_form_doc import find_source_upload_id + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + content = doc.current_content or "" + upload_id = find_source_upload_id(content) + if not upload_id: + raise HTTPException(400, "Document is not a PDF — no pdf_source marker found") + + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, "Source PDF could not be located") + + try: + body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user)) + except Exception as e: + logger.error(f"extract_pdf_text failed for {pdf_path}: {e}") + raise HTTPException(500, f"Extraction failed: {e}") + + if not body_text: + return {"ok": True, "id": doc_id, "extracted": False, "reason": "No readable content"} + + # Preserve everything up through the title (front-matter marker + + # first H1) and replace the rest with the freshly extracted text. + head_re = re.compile(r'^(<!--[^>]+-->\s*\n+#[^\n]*\n+)', re.MULTILINE) + head_match = head_re.match(content) + head = head_match.group(1) if head_match else (content.splitlines()[0] + "\n\n# " + (doc.title or "PDF") + "\n\n") + doc.current_content = head + body_text.strip() + "\n" + doc.version_count = (doc.version_count or 1) + 1 + db.add(DocumentVersion( + id=str(__import__("uuid").uuid4()), + document_id=doc_id, + version_number=doc.version_count, + content=doc.current_content, + summary="PDF text re-extracted (OCR)", + source="ocr", + )) + db.commit() + return {"ok": True, "id": doc_id, "extracted": True, "chars": len(body_text)} + finally: + db.close() + + # ---- POST /api/documents/export-zip — bundle selected docs into a .zip ---- + @router.post("/api/documents/export-zip") + async def documents_export_zip(request: Request): + """Zip the selected documents (each as a text file with the right + extension) — mirrors the gallery's bulk download-zip so multi-export + is one file instead of a blocked flood of individual downloads.""" + user = get_current_user(request) + try: + data = await request.json() + except Exception as e: + logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e) + data = {} + ids = data.get("ids") or [] + if not ids: + raise HTTPException(400, "No documents specified") + _ext = { + "javascript": ".js", "python": ".py", "html": ".html", "css": ".css", + "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh", + "sql": ".sql", "rust": ".rs", "go": ".go", "java": ".java", "c": ".c", + "cpp": ".cpp", "typescript": ".ts", "ruby": ".rb", "php": ".php", + "text": ".txt", "xml": ".xml", "toml": ".toml", "ini": ".ini", + } + db = SessionLocal() + try: + import io + import re + import zipfile + from fastapi import Response + docs = db.query(Document).filter(Document.id.in_(ids)).all() + buf = io.BytesIO() + used = set() + wrote = 0 + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for doc in docs: + try: + _verify_doc_owner(db, doc, user) + except HTTPException: + continue # skip docs the user doesn't own + ext = _ext.get(doc.language or "text", ".txt") + base = (doc.title or "document").strip() or "document" + base = re.sub(r"[^\w\-. ]+", "", base)[:60].strip() or doc.id + name = base if "." in base else base + ext + i = 1 + while name in used: + name = f"{base}-{i}" + ("" if "." in base else ext) + i += 1 + used.add(name) + zf.writestr(name, doc.current_content or "") + wrote += 1 + if not wrote: + raise HTTPException(404, "No documents found") + return Response( + content=buf.getvalue(), + media_type="application/zip", + headers={"Content-Disposition": 'attachment; filename="documents.zip"'}, + ) + finally: + db.close() + + # ---- PUT /api/document/{doc_id} — user manual edit ---- + # Coalesce window: if the last user version was saved within this many + # seconds, update it in-place (user is still actively editing). + # Once the gap exceeds this, the next save creates a new version. + VERSION_COALESCE_SECONDS = 60 + + @router.put("/api/document/{doc_id}") + async def update_document(request: Request, doc_id: str, req: DocumentUpdate) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + incoming_content = req.content + from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document + is_email_doc = ( + (doc.language or "").lower() == "email" + or _looks_like_email_document(doc.current_content or "", doc.title or "") + or _looks_like_email_document(req.content or "", doc.title or "") + ) + if is_email_doc: + incoming_content = _coerce_email_document_content(doc.current_content or "", req.content) + doc.language = "email" + + # Skip if content is identical unless the caller explicitly wants + # a checkpoint version from the current editor state. + if doc.current_content == incoming_content and not req.force_version: + return _doc_to_dict(doc) + + _reserve_document_uploads(user, incoming_content) + _assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler) + + # Check if we can coalesce with the latest version + latest_ver = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id, + ).order_by(DocumentVersion.version_number.desc()).first() + + now = datetime.now(timezone.utc) + coalesced = False + if latest_ver and latest_ver.source == "user" and not req.force_version: + ver_time = latest_ver.created_at + if ver_time.tzinfo is None: + ver_time = ver_time.replace(tzinfo=timezone.utc) + age = (now - ver_time).total_seconds() + if age < VERSION_COALESCE_SECONDS: + # Update the existing version in-place + latest_ver.content = incoming_content + latest_ver.created_at = now + if req.summary: + latest_ver.summary = req.summary + coalesced = True + + if not coalesced: + new_ver = doc.version_count + 1 + ver = DocumentVersion( + id=str(uuid.uuid4()), + document_id=doc_id, + version_number=new_ver, + content=incoming_content, + summary=req.summary or "Manual edit", + source="user", + ) + doc.version_count = new_ver + db.add(ver) + + doc.current_content = incoming_content + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, f"Failed to update document: {e}") + finally: + db.close() + + # ---- PATCH /api/document/{doc_id} — metadata only ---- + @router.patch("/api/document/{doc_id}") + async def patch_document(request: Request, doc_id: str, req: DocumentPatch) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + if req.title is not None: + doc.title = req.title + if req.language is not None: + doc.language = req.language + if req.session_id is not None: + # Empty string = unlink from session + if req.session_id: + _get_session_or_404(db, req.session_id, user) + doc.session_id = req.session_id if req.session_id else None + if not req.session_id: + # Tab closed / doc detached from its session — drop the + # in-memory active-doc pointer so the last-resort injection + # path doesn't re-surface this doc in a later chat (#1160). + try: + from src.agent_tools.document_tools import clear_active_document + clear_active_document(doc_id) + except Exception as e: + logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e) + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, str(e)) + finally: + db.close() + + # ---- DELETE /api/document/{doc_id} — soft delete ---- + @router.delete("/api/document/{doc_id}") + async def delete_document(request: Request, doc_id: str) -> Dict[str, str]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + doc.is_active = False + # Closed/deleted — drop the in-memory active-doc pointer so it isn't + # re-injected into a later, unrelated chat (#1160). + try: + from src.agent_tools.document_tools import clear_active_document + clear_active_document(doc_id) + except Exception: + pass + db.commit() + return {"status": "deleted", "id": doc_id} + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, str(e)) + finally: + db.close() + + # ---- GET /api/document/{doc_id}/versions ---- + @router.get("/api/document/{doc_id}/versions") + async def list_versions(request: Request, doc_id: str) -> List[Dict[str, Any]]: + user = get_current_user(request) + db = SessionLocal() + try: + # Verify ownership before listing versions + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + versions = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id + ).order_by(DocumentVersion.version_number.desc()).all() + return [{ + "id": v.id, + "version_number": v.version_number, + "content": v.content, + "summary": v.summary, + "source": v.source, + "created_at": v.created_at.isoformat() if v.created_at else None, + } for v in versions] + finally: + db.close() + + # ---- GET /api/document/{doc_id}/version/{num} ---- + @router.get("/api/document/{doc_id}/version/{num}") + async def get_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + # Verify ownership + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + ver = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id, + DocumentVersion.version_number == num, + ).first() + if not ver: + raise HTTPException(404, "Version not found") + return _version_to_dict(ver) + finally: + db.close() + + # ---- POST /api/document/{doc_id}/restore/{num} ---- + @router.post("/api/document/{doc_id}/restore/{num}") + async def restore_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + old_ver = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id, + DocumentVersion.version_number == num, + ).first() + if not old_ver: + raise HTTPException(404, "Version not found") + + new_ver_num = doc.version_count + 1 + ver = DocumentVersion( + id=str(uuid.uuid4()), + document_id=doc_id, + version_number=new_ver_num, + content=old_ver.content, + summary=f"Restored from v{num}", + source="user", + ) + doc.current_content = old_ver.content + doc.version_count = new_ver_num + db.add(ver) + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, str(e)) + finally: + db.close() + + # ---- POST /api/documents/tidy — clean up broken/empty documents ---- + @router.post("/api/documents/tidy") + async def tidy_documents(request: Request) -> Dict[str, Any]: + """Fix empty titles and remove broken/empty documents (user's docs only).""" + user = get_current_user(request) + db = SessionLocal() + try: + q = ( + db.query(Document) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == True) + .filter((Document.archived == False) | (Document.archived.is_(None))) + ) + q = _owner_session_filter(q, user) + docs = q.all() + fixed_titles = 0 + deleted = 0 + + # Same junk-detection logic as the scheduled tidy_documents + # action (src/document_actions.py). Keep these two in sync. + import re as _re + from src.document_actions import _JUNK_TITLES + + to_delete = [] + now = datetime.now(timezone.utc) + for doc in docs: + created = doc.created_at + if created and created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + + # Skip freshly created documents to avoid deleting them while the user is actively editing + if created and (now - created).total_seconds() < 900: # 15 minutes + continue + + content = (doc.current_content or "").strip() + title_raw = (doc.title or "").strip() + title = title_raw.lower() + is_fresh_empty = ( + not content + and created is not None + and (now - created).total_seconds() < 1800 + ) + if is_fresh_empty: + continue + + # Strip markdown noise to get a "real" character count + stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE) + stripped = _re.sub(r"[*_`>\-=]+", "", stripped) + stripped = _re.sub(r"\s+", " ", stripped).strip() + real_len = len(stripped) + + # Detect email-scaffold stubs: "To: \nSubject: \n---\n" style + # bodies with nothing typed in. Stub = every meaningful line + # is a header label (To:/From:/Subject:/...) with no real + # value (blank, "empty", "(empty)", "-", "none", "n/a"). + _is_email_stub = False + _HEADER_RE = _re.compile(r"^(to|from|cc|bcc|subject|reply-to):\s*(.*)$", _re.I) + _PLACEHOLDER_VALS = {"", "empty", "(empty)", "-", "—", "none", "n/a", "na", "tbd"} + if title in ("new email", "new mail", "new message") or doc.language == "email": + body_lines = [ln.strip() for ln in content.split("\n") + if ln.strip() and ln.strip() != "---"] + def _is_filler(ln): + m = _HEADER_RE.match(ln) + if not m: + return False + val = (m.group(2) or "").strip().lower() + return val in _PLACEHOLDER_VALS + has_real_body = any(not _is_filler(ln) for ln in body_lines) + if body_lines and not has_real_body: + _is_email_stub = True + + # Hard-delete obviously empty / junk documents + if not content or content in ("", "# Untitled"): + to_delete.append(doc); deleted += 1; continue + if _is_email_stub: + to_delete.append(doc); deleted += 1; continue + if title in _JUNK_TITLES: + to_delete.append(doc); deleted += 1; continue + + # Fix empty or placeholder titles on survivors + if not title_raw or title_raw == "Untitled": + new_title = _derive_title(content) + if new_title and new_title != "Untitled": + doc.title = new_title + fixed_titles += 1 + + for doc in to_delete: + db.delete(doc) + + # Also clean up inactive empty docs from previous soft-deletes + inactive_q = ( + db.query(Document) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == False) + .filter((Document.current_content == None) | (Document.current_content == "")) + ) + inactive_q = _owner_session_filter(inactive_q, user) + inactive_docs = inactive_q.all() + for doc in inactive_docs: + db.delete(doc) + deleted += len(inactive_docs) + + db.commit() + return { + "fixed_titles": fixed_titles, + "deleted": deleted, + "message": f"Fixed {fixed_titles} title{'s' if fixed_titles != 1 else ''}, removed {deleted} empty document{'s' if deleted != 1 else ''}", + } + except Exception as e: + db.rollback() + logger.error(f"Document tidy failed: {e}") + raise HTTPException(500, f"Tidy failed: {e}") + finally: + db.close() + + # ---- POST /api/documents/ai-tidy — AI-powered cleanup of junk/test documents ---- + @router.post("/api/documents/ai-tidy") + async def ai_tidy_documents(request: Request) -> Dict[str, Any]: + """Use AI to judge if documents are junk/test/accidental, then delete them. + Caches verdicts so previously-reviewed docs are skipped.""" + from src.task_endpoint import resolve_task_endpoint + from src.endpoint_resolver import resolve_endpoint + from src.llm_core import llm_call_async + + user = get_current_user(request) + url, model, headers = resolve_task_endpoint(owner=user or None) + if not url or not model: + # Fall back to default endpoint + url, model, headers = resolve_endpoint("default", owner=user or None) + if not url or not model: + raise HTTPException(500, "No endpoint configured for AI tidy") + + db = SessionLocal() + try: + q = ( + db.query(Document) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == True) + .filter((Document.archived == False) | (Document.archived.is_(None))) + ) + q = _owner_session_filter(q, user) + docs = q.all() + + # Only review docs that haven't been reviewed yet + to_review = [d for d in docs if not d.tidy_verdict] + if not to_review: + return {"deleted": 0, "reviewed": 0, "message": "All documents already reviewed"} + + # Build a batch prompt — review up to 30 at a time + batch = to_review[:30] + doc_list = [] + for i, doc in enumerate(batch): + preview = (doc.current_content or "")[:300].strip() + doc_list.append(f"[{i}] title=\"{doc.title}\" lang={doc.language or 'text'} content_preview=\"{preview}\"") + + prompt = ( + "You are a document library cleaner. For each document below, decide if it is JUNK " + "(test, accidental, placeholder, empty-ish, tool-test, throwaway) or KEEP (real content worth saving).\n\n" + "Respond with ONLY a JSON array of verdicts, one per document, like: [\"junk\",\"keep\",\"junk\",...]\n" + "No explanation, no markdown, just the JSON array.\n\n" + + "\n".join(doc_list) + ) + + response = await llm_call_async( + url, model, + [{"role": "system", "content": "You classify documents as junk or keep. Respond only with a JSON array."}, + {"role": "user", "content": prompt}], + temperature=0.1, + max_tokens=200, + headers=headers, + timeout=30, + ) + + # Parse verdicts + import re + match = re.search(r'\[.*?\]', response, re.DOTALL) + if not match: + raise HTTPException(500, "AI returned invalid response") + + import json as _json + verdicts = _json.loads(match.group()) + + deleted = 0 + reviewed = 0 + for i, doc in enumerate(batch): + if i >= len(verdicts): + break + verdict = str(verdicts[i] or "").lower().strip() + if verdict == "junk": + doc.tidy_verdict = "junk" + db.delete(doc) + deleted += 1 + else: + doc.tidy_verdict = "keep" + reviewed += 1 + + db.commit() + return { + "deleted": deleted, + "reviewed": reviewed, + "remaining": len(to_review) - len(batch), + "message": f"Reviewed {reviewed}, removed {deleted} junk document{'s' if deleted != 1 else ''}", + } + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"AI tidy failed: {e}") + raise HTTPException(500, f"AI tidy failed: {e}") + finally: + db.close() + + # ---- POST /api/document/{doc_id}/export-pdf/preview ---- + @router.post("/api/document/{doc_id}/export-pdf/preview") + async def export_pdf_preview(doc_id: str, request: Request) -> Dict[str, Any]: + """Return the field-value mapping that would be written to the PDF. + + Frontend shows this in a confirmation modal so the user can spot/fix + any wrong values before triggering the actual download. + """ + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") + + fields = load_field_sidecar(pdf_path) + if not fields: + raise HTTPException(404, "Field schema sidecar missing for source PDF") + + values = parse_markdown_to_values(doc.current_content or "") + field_meta = {f["name"]: f for f in fields} + + preview = [] + for name, current in values.items(): + meta = field_meta.get(name) + if not meta: + continue + preview.append({ + "name": name, + "label": meta.get("label") or name, + "type": meta.get("type"), + "options": meta.get("options") or [], + "page": meta.get("page"), + "value": current, + }) + + unknown = [ + name for name in values + if name not in field_meta + ] + return { + "doc_id": doc_id, + "upload_id": upload_id, + "fields": preview, + "unknown_fields": unknown, + "total": len(fields), + "filled": sum(1 for p in preview if p["value"] not in ("", False, None)), + } + finally: + db.close() + + # ---- GET /api/document/{doc_id}/render-pages ---- + @router.get("/api/document/{doc_id}/render-pages") + async def render_pages(doc_id: str, request: Request) -> Dict[str, Any]: + """Return per-page metadata for the interactive PDF view. + + Each page entry has its rendered-image dimensions (matching what + /page/{n}.png returns at the same DPI) plus the list of form fields + on that page with their rects translated to image-pixel coordinates. + Frontend overlays HTML form controls at those positions. + """ + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found") + + fitz = _load_pdf_viewer_fitz() + schema = load_field_sidecar(pdf_path) or [] + values = parse_markdown_to_values(doc.current_content or "") + + # Group fields by page + by_page: Dict[int, list] = {} + for f in schema: + by_page.setdefault(f["page"], []).append(f) + + scale = _PDF_RENDER_SCALE + pdf_doc = fitz.open(pdf_path) + try: + pages_out = [] + for page_index in range(pdf_doc.page_count): + page = pdf_doc[page_index] + page_no = page_index + 1 + pw, ph = page.rect.width, page.rect.height + img_w = int(pw * scale) + img_h = int(ph * scale) + fields_out = [] + for f in by_page.get(page_no, []): + x0, y0, x1, y1 = f["rect"] + fields_out.append({ + "name": f["name"], + "type": f["type"], + "label": f.get("label") or "", + "options": f.get("options") or [], + "value": values.get(f["name"], f.get("value", "")), + "rect_px": [ + int(x0 * scale), int(y0 * scale), + int(x1 * scale), int(y1 * scale), + ], + }) + pages_out.append({ + "page": page_no, + "width": img_w, + "height": img_h, + "fields": fields_out, + }) + return {"doc_id": doc_id, "scale": scale, "pages": pages_out} + finally: + pdf_doc.close() + finally: + db.close() + + # ---- GET /api/document/{doc_id}/page/{n}.png ---- + @router.get("/api/document/{doc_id}/page/{page_no}.png") + async def render_page_png(doc_id: str, page_no: int, request: Request): + """Render one page of the source PDF as a PNG (no values stamped — the + frontend overlays HTML form inputs on top).""" + from fastapi.responses import Response + from src.pdf_form_doc import find_source_upload_id + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, "Source PDF not found") + finally: + db.close() + + fitz = _load_pdf_viewer_fitz() + pdf_doc = fitz.open(pdf_path) + try: + if page_no < 1 or page_no > pdf_doc.page_count: + raise HTTPException(404, "Page out of range") + page = pdf_doc[page_no - 1] + mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) + pix = page.get_pixmap(matrix=mat, alpha=False) + png_bytes = pix.tobytes("png") + return Response( + content=png_bytes, + media_type="image/png", + headers={"Cache-Control": "public, max-age=3600"}, + ) + finally: + pdf_doc.close() + + # ---- POST /api/document/{doc_id}/ai-fill-annotations ---- + @router.post("/api/document/{doc_id}/ai-fill-annotations") + async def ai_fill_annotations(doc_id: str, request: Request) -> Dict[str, Any]: + """Ask a vision-capable LLM to locate fillable areas on a flat PDF and + propose annotation values for each, given a free-form user instruction. + + Returns a list of annotations: [{page, x, y, w, h, value}] where x/y/w/h + are page-percentages (0–100) — same coordinate system as the freeform + annotations the frontend already renders. + """ + import base64 + import json + import fitz + from src.pdf_form_doc import find_source_upload_id + from src.document_processor import _resolve_vl_model, _load_vl_settings + from src.llm_core import llm_call_async + + body = await request.json() if request.headers.get("content-type", "").startswith("application/json") else {} + instruction = (body or {}).get("instruction", "").strip() + if not instruction: + raise HTTPException(400, "instruction is required") + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, "Source PDF not found") + finally: + db.close() + + # Resolve VL model (admin-configured or auto-detected vision-capable) + settings = _load_vl_settings() + vl_model = settings.get("vision_model", "") + try: + url, model_id, headers = _resolve_vl_model(vl_model, owner=user) + except Exception as e: + raise HTTPException(503, f"No vision model available: {e}") + + system_prompt = ( + "You analyze rendered PDF page images and propose values to fill in. " + "For each blank line, box, underscore, or labeled space on the page that " + "should be filled given the user's instruction, output one annotation. " + "Coordinates are percentages (0-100) of the page width/height with the " + "origin at top-left. Width/height should match the visible blank box. " + "Return ONLY a JSON array, no prose, no markdown fences. Each entry: " + '{"x": number, "y": number, "w": number, "h": number, "value": string}. ' + "If a region should not be filled, omit it. If nothing should be filled, " + "return []." + ) + + all_annotations = [] + pdf_doc = fitz.open(pdf_path) + try: + for page_index in range(pdf_doc.page_count): + page = pdf_doc[page_index] + mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) + pix = page.get_pixmap(matrix=mat, alpha=False) + png_bytes = pix.tobytes("png") + b64 = base64.b64encode(png_bytes).decode("ascii") + + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + f"User instruction:\n{instruction}\n\n" + f"This is page {page_index + 1} of {pdf_doc.page_count}. " + "Return JSON array of annotations to add to this page." + ), + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + }, + ], + }, + ] + try: + raw = await llm_call_async( + url, model_id, messages, + temperature=0.1, max_tokens=2000, headers=headers, + ) + except Exception as e: + logger.error(f"VL call failed on page {page_index + 1}: {e}") + continue + + raw = (raw or "").strip() + if raw.startswith("```"): + raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + try: + parsed = json.loads(raw) + except Exception: + logger.warning(f"AI fill: page {page_index + 1} returned non-JSON: {raw[:200]}") + continue + if not isinstance(parsed, list): + continue + for item in parsed: + if not isinstance(item, dict): + continue + try: + x = float(item.get("x", 0)) + y = float(item.get("y", 0)) + w = float(item.get("w", 0)) + h = float(item.get("h", 0)) + value = str(item.get("value", "") or "") + except Exception: + continue + # Clamp + reject zero-size entries + if w <= 0.5 or h <= 0.3: + continue + x = max(0.0, min(99.0, x)) + y = max(0.0, min(99.0, y)) + w = max(0.5, min(100.0 - x, w)) + h = max(0.3, min(100.0 - y, h)) + if not value.strip(): + continue + all_annotations.append({ + "page": page_index + 1, + "x": round(x, 2), + "y": round(y, 2), + "w": round(w, 2), + "h": round(h, 2), + "value": value, + }) + finally: + pdf_doc.close() + + return {"annotations": all_annotations} + + # ---- GET /api/document/{doc_id}/render-pdf ---- + @router.get("/api/document/{doc_id}/render-pdf") + async def render_pdf(doc_id: str, request: Request): + """Inline PDF preview filled with the current markdown values. + + Same plumbing as the export route, but no signature stamping and + served inline (Content-Disposition: inline) so the browser can + embed it in an iframe. Cache-busted by the caller via query string. + """ + import base64 + import os + import tempfile + from fastapi.responses import FileResponse + from starlette.background import BackgroundTask + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, parse_markdown_annotations + from src.pdf_forms import fill_fields, stamp_annotations + from core.database import Signature + + # Track temp files for this request so they get unlinked AFTER + # the response is fully sent (BackgroundTask runs post-send). + _to_unlink: list[str] = [] + def _cleanup_temps(): + for _p in _to_unlink: + try: + os.unlink(_p) + except FileNotFoundError: + pass + except Exception as _e: + logger.warning(f"Could not unlink temp PDF {_p}: {_e}") + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found") + + # Fail fast with a clear 503 if the optional PyMuPDF dependency + # is missing — fill_fields/stamp_annotations will otherwise + # raise RuntimeError deep inside and bubble out as a 500. + # Mirrors the convention in _load_pdf_viewer_fitz above. + _load_pdf_viewer_fitz() + + values = parse_markdown_to_values(doc.current_content or "") + out_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(out_path) + try: + fill_fields(pdf_path, out_path, values) + except Exception as e: + logger.error(f"render_pdf fill_fields failed for {doc_id}: {e}") + _cleanup_temps() + raise HTTPException(500, f"PDF render failed: {e}") + + annotations = parse_markdown_annotations(doc.current_content or "") + if annotations: + ann_sig_ids = [ + a["value"][len("signature:"):].strip() + for a in annotations + if a.get("kind") == "signature" + and isinstance(a.get("value"), str) + and a["value"].startswith("signature:") + ] + ann_signature_pngs: dict[str, bytes] = {} + if ann_sig_ids: + # SECURITY: filter by owner so a caller can't reference + # someone else's signature ID from doc markdown and have + # it stamped/exported. + _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) + if user: + _sig_q = _sig_q.filter(Signature.owner == user) + sig_rows = _sig_q.all() + for s in sig_rows: + try: + ann_signature_pngs[s.id] = base64.b64decode(s.data_png) + except Exception as e: + logger.warning(f"Bad annotation signature data for {s.id}: {e}") + annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(annotated_path) + try: + stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) + out_path = annotated_path + except Exception as e: + logger.error(f"stamp_annotations (render) failed for {doc_id}: {e}") + + return FileResponse( + out_path, + media_type="application/pdf", + headers={"Content-Disposition": "inline"}, + background=BackgroundTask(_cleanup_temps), + ) + finally: + db.close() + + # ---- GET /api/document/{doc_id}/export-pdf ---- + @router.get("/api/document/{doc_id}/export-pdf") + async def export_pdf(doc_id: str, request: Request): + """Stream the filled PDF for download. + + Reads field values and signature selections from the markdown — there + is no separate confirmation step. Signature fields contain their + chosen signature ID encoded as `signature:<id>` in the value. + """ + import base64 + import os + import tempfile + from fastapi.responses import FileResponse + from starlette.background import BackgroundTask + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar, parse_markdown_annotations + from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations + from core.database import Signature + + _to_unlink: list[str] = [] + def _cleanup_temps(): + for _p in _to_unlink: + try: + os.unlink(_p) + except FileNotFoundError: + pass + except Exception as _e: + logger.warning(f"Could not unlink temp PDF {_p}: {_e}") + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") + + schema = load_field_sidecar(pdf_path) or [] + sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} + + all_values = parse_markdown_to_values(doc.current_content or "") + # Split: signature fields go to stamps, everything else to fill_fields + text_values: dict = {} + sig_ids: dict[str, str] = {} + for name, raw in all_values.items(): + if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): + sig_ids[name] = raw[len("signature:"):].strip() + elif name not in sig_field_names: + text_values[name] = raw + + stamps: dict = {} + if sig_ids: + # SECURITY: filter by owner — same reason as render_pdf. + _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) + if user: + _sig_q2 = _sig_q2.filter(Signature.owner == user) + rows = _sig_q2.all() + by_id = {s.id: s for s in rows} + for field_name, sid in sig_ids.items(): + s = by_id.get(sid) + if not s: + continue + try: + stamps[field_name] = base64.b64decode(s.data_png) + except Exception as e: + logger.warning(f"Bad signature data for {sid}: {e}") + + filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(filled_path) + try: + fill_fields(pdf_path, filled_path, text_values) + except Exception as e: + logger.error(f"fill_fields failed for doc {doc_id}: {e}") + _cleanup_temps() + raise HTTPException(500, f"PDF fill failed: {e}") + + out_path = filled_path + if stamps: + stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(stamped_path) + try: + stamp_signatures(filled_path, stamped_path, stamps) + out_path = stamped_path + except Exception as e: + logger.error(f"stamp_signatures failed for doc {doc_id}: {e}") + + # Burn freeform annotations (Text/Check/Sign drops) on top. + annotations = parse_markdown_annotations(doc.current_content or "") + if annotations: + # Resolve any signature annotations to their PNG bytes. + ann_sig_ids = [ + a["value"][len("signature:"):].strip() + for a in annotations + if a.get("kind") == "signature" + and isinstance(a.get("value"), str) + and a["value"].startswith("signature:") + ] + ann_signature_pngs: dict[str, bytes] = {} + if ann_sig_ids: + # SECURITY: filter by owner so a caller can't reference + # someone else's signature ID from doc markdown and have + # it stamped/exported. + _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) + if user: + _sig_q = _sig_q.filter(Signature.owner == user) + sig_rows = _sig_q.all() + for s in sig_rows: + try: + ann_signature_pngs[s.id] = base64.b64decode(s.data_png) + except Exception as e: + logger.warning(f"Bad annotation signature data for {s.id}: {e}") + annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(annotated_path) + try: + stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) + out_path = annotated_path + except Exception as e: + logger.error(f"stamp_annotations failed for doc {doc_id}: {e}") + + download_name = _slug(doc.title or "form") + "_annotated.pdf" + return FileResponse( + out_path, + media_type="application/pdf", + filename=download_name, + background=BackgroundTask(_cleanup_temps), + ) + finally: + db.close() + + # ---- POST /api/document/{doc_id}/prepare-signed-reply ---- + @router.post("/api/document/{doc_id}/prepare-signed-reply") + async def prepare_signed_reply(doc_id: str, request: Request): + """Bake the current PDF state (form fields + signature stamps + + annotations) into a flattened PDF, drop it in COMPOSE_UPLOADS_DIR + and return the reply context (To/Subject/threading headers) so the + frontend can open a reply draft with this attachment pre-loaded. + + Requires the document to have source_email_* metadata (set when the + doc was created via /api/email/attachment-as-doc). Otherwise 400. + """ + import base64 + import tempfile + import shutil + import uuid as _uuid + import email as _email_mod + from src.pdf_form_doc import ( + find_source_upload_id, parse_markdown_to_values, + load_field_sidecar, parse_markdown_annotations, + ) + from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations + from core.database import Signature + # COMPOSE_UPLOADS_DIR lives in email_routes — re-derive here so we + # don't import from a routes file (cycle-prone). Same env override + # as email_routes (ODYSSEUS_MAIL_ATTACHMENTS_DIR). + from pathlib import Path as _Path + _COMPOSE_DIR = _Path(MAIL_ATTACHMENTS_DIR) / "_compose" + _COMPOSE_DIR.mkdir(parents=True, exist_ok=True) + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + if not (doc.source_email_uid and doc.source_email_folder): + raise HTTPException(400, "Document has no source email — cannot reply") + + # 1) Build the flattened PDF (same pipeline as export_pdf) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found") + + schema = load_field_sidecar(pdf_path) or [] + sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} + all_values = parse_markdown_to_values(doc.current_content or "") + text_values: dict = {} + sig_ids: dict[str, str] = {} + for name, raw in all_values.items(): + if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): + sig_ids[name] = raw[len("signature:"):].strip() + elif name not in sig_field_names: + text_values[name] = raw + + stamps: dict = {} + if sig_ids: + # SECURITY: filter by owner — same reason as render_pdf. + _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) + if user: + _sig_q2 = _sig_q2.filter(Signature.owner == user) + rows = _sig_q2.all() + by_id = {s.id: s for s in rows} + for fname, sid in sig_ids.items(): + s = by_id.get(sid) + if not s: + continue + try: + stamps[fname] = base64.b64decode(s.data_png) + except Exception: + pass + + import os + _to_unlink: list[str] = [] + filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(filled_path) + fill_fields(pdf_path, filled_path, text_values) + out_path = filled_path + if stamps: + stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(stamped_path) + try: + stamp_signatures(filled_path, stamped_path, stamps) + out_path = stamped_path + except Exception as e: + logger.warning(f"stamp_signatures failed for {doc_id}: {e}") + + annotations = parse_markdown_annotations(doc.current_content or "") + if annotations: + ann_sig_ids = [ + a["value"][len("signature:"):].strip() + for a in annotations + if a.get("kind") == "signature" + and isinstance(a.get("value"), str) + and a["value"].startswith("signature:") + ] + ann_signature_pngs: dict[str, bytes] = {} + if ann_sig_ids: + # SECURITY: filter by owner so a caller can't reference + # someone else's signature ID from doc markdown and have + # it stamped/exported. + _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) + if user: + _sig_q = _sig_q.filter(Signature.owner == user) + sig_rows = _sig_q.all() + for s in sig_rows: + try: + ann_signature_pngs[s.id] = base64.b64decode(s.data_png) + except Exception: + pass + annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(annotated_path) + try: + stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) + out_path = annotated_path + except Exception as e: + logger.warning(f"stamp_annotations failed for {doc_id}: {e}") + + # 2) Move/copy into COMPOSE_UPLOADS_DIR with the token format + # `<uuid>_<original_name>` that /api/email/send expects. + filename = _slug(doc.title or "signed") + "_signed.pdf" + token = f"{_uuid.uuid4().hex}_{filename}" + dest = _COMPOSE_DIR / token + shutil.copyfile(out_path, str(dest)) + # Unlink the intermediate temp PDFs now that they've been + # copied into COMPOSE_UPLOADS_DIR. + for _p in _to_unlink: + try: + os.unlink(_p) + except FileNotFoundError: + pass + except Exception as _e: + logger.warning(f"Could not unlink temp PDF {_p}: {_e}") + + # 3) Fetch the source email's headers so we can build a clean reply + # context (To/Subject/In-Reply-To/References). + try: + from routes.email_routes import _imap, _decode_header + from routes.email_helpers import _q + except Exception: + _imap = None + _decode_header = lambda x: x or "" + _q = lambda x: x or "" + + to_addr = "" + from_name = "" + subject = "" + in_reply_to = doc.source_email_message_id or "" + references = in_reply_to + if _imap: + try: + with _imap(doc.source_email_account_id or None) as conn: + conn.select(_q(doc.source_email_folder), readonly=True) + status, data = conn.fetch(doc.source_email_uid.encode(), "(RFC822.HEADER)") + if status == "OK" and data and data[0]: + raw_hdr = data[0][1] + m = _email_mod.message_from_bytes(raw_hdr) + sender = _decode_header(m.get("From", "")) + from_name, to_addr = _email_mod.utils.parseaddr(sender) + if not to_addr: + to_addr = sender + subject = _decode_header(m.get("Subject", "") or "") + if subject and not subject.lower().startswith("re:"): + subject = "Re: " + subject + msg_refs = (m.get("References") or "").strip() + msg_in_reply = (m.get("Message-ID") or "").strip() or in_reply_to + in_reply_to = msg_in_reply + references = (msg_refs + " " + msg_in_reply).strip() if msg_refs else msg_in_reply + except Exception as e: + logger.warning(f"prepare-signed-reply header fetch failed: {e}") + + return { + "ok": True, + "attachment": { + "token": token, + "filename": filename, + "size": dest.stat().st_size, + }, + "reply": { + "to": to_addr, + "to_name": from_name, + "subject": subject, + "in_reply_to": in_reply_to, + "references": references, + "account_id": doc.source_email_account_id or None, + "source_uid": doc.source_email_uid, + "source_folder": doc.source_email_folder, + "source_message_id": doc.source_email_message_id, + }, + } + finally: + db.close() + + return router diff --git a/routes/document_helpers.py b/routes/document_helpers.py index a0c2d08eb..c1f68ca51 100644 --- a/routes/document_helpers.py +++ b/routes/document_helpers.py @@ -1,243 +1,14 @@ -"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py.""" +"""Backward-compat shim — canonical location is routes/document/document_helpers.py. -"""Document routes — CRUD for living documents with version history.""" +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.document_helpers``, ``from routes.document_helpers import +X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import +pattern used by test_security_regressions.py all operate on the *same* object. +Keeps existing import paths working after slice 2m (#4082/#4071). +""" -import logging -import os -import re -from typing import Any, Dict, Optional +import sys as _sys -from fastapi import HTTPException, Request -from pydantic import BaseModel +from routes.document import document_helpers as _canonical # noqa: F401 -from core.database import Document, DocumentVersion -from core.database import Session as DbSession -from src.auth_helpers import _auth_disabled -from src.upload_handler import UploadHandler - -logger = logging.getLogger(__name__) - - -# ---- Request schemas ---- - -class DocumentCreate(BaseModel): - session_id: Optional[str] = None - title: str = "Untitled" - language: Optional[str] = None - content: str = "" - -class DocumentUpdate(BaseModel): - content: str - summary: Optional[str] = None - force_version: bool = False - -class DocumentPatch(BaseModel): - title: Optional[str] = None - language: Optional[str] = None - session_id: Optional[str] = None # link/unlink document to a session - - -# ---- Helpers ---- - -def _doc_to_dict(doc: Document) -> Dict[str, Any]: - return { - "id": doc.id, - "session_id": doc.session_id, - "title": doc.title, - "language": doc.language, - "current_content": doc.current_content, - "version_count": doc.version_count, - "is_active": doc.is_active, - "archived": bool(getattr(doc, "archived", False)), - "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None, - "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None, - # Source-email provenance (set when doc was created from an email - # attachment) — drives the "Send signed reply" menu item. - "source_email_uid": getattr(doc, "source_email_uid", None), - "source_email_folder": getattr(doc, "source_email_folder", None), - "source_email_account_id": getattr(doc, "source_email_account_id", None), - "source_email_message_id": getattr(doc, "source_email_message_id", None), - } - -def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]: - return { - "id": v.id, - "document_id": v.document_id, - "version_number": v.version_number, - "content": v.content, - "summary": v.summary, - "source": v.source, - "created_at": v.created_at.isoformat() if v.created_at else None, - } - - -def _verify_doc_owner(db, doc: Document, user: str): - """Verify `user` owns this document. Raise 404 if not. - - Documents now carry their own `owner` column, so a doc whose session - was deleted (session_id → NULL) can still prove ownership and stay - openable / cloneable. We trust that column first and only fall back to - the session join for any not-yet-backfilled legacy row. - """ - if user is None: - if _auth_disabled(): - return # Single-user / no-auth mode: allow access - raise HTTPException(403, "Authentication required") - if doc.owner is not None: - if doc.owner != user: - raise HTTPException(404, "Document not found") - return - # Legacy fallback: derive ownership from the linked session. - if not doc.session_id: - raise HTTPException(404, "Document not found") - session = db.query(DbSession).filter(DbSession.id == doc.session_id).first() - if not session or session.owner != user: - raise HTTPException(404, "Document not found") - - -def _owner_session_filter(q, user): - """Restrict a documents query to those owned by `user`. - - Documents now carry their own `owner` column (backfilled at boot from - the linked session, or assigned to the admin user for legacy/orphaned - docs). We filter on that directly rather than on a session join, so a - document whose session was deleted (session_id → NULL) still shows up - for its owner instead of silently vanishing from the Library + search. - - The owner backfill runs in init_db before the app serves requests, so - by the time this filter is live there are no NULL-owner rows to leak; - we therefore match the owner strictly for authenticated callers.""" - if not user: - if user == "" or _auth_disabled(): - return q - return q.filter(False) - return q.filter(Document.owner == user) - - - -def _slug(name: str) -> str: - """Filesystem-friendly version of a document title. - - Whitespace becomes underscores; other unsafe punctuation is dropped. - Preserves letters, digits, dot, hyphen, underscore. Idempotent. - """ - import re as _re - s = (name or "").strip() - # Drop the trailing extension if the title happens to include one - s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE) - s = _re.sub(r'\s+', '_', s) - s = _re.sub(r'[^A-Za-z0-9._-]', '', s) - s = _re.sub(r'_+', '_', s).strip('_') - return s or "form" - - -# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units). -_PDF_RENDER_SCALE = 2.0 - - -def _upload_path_inside(upload_dir: str, path: str) -> bool: - base = os.path.realpath(upload_dir) - p = os.path.realpath(path) - try: - return os.path.commonpath([base, p]) == base - except Exception: - return False - - -def _resolve_user_upload_path( - upload_handler: Any, - upload_id: str, - owner: Optional[str], - auth_manager=None, -) -> Optional[str]: - """Resolve an upload id to a filesystem path the caller may read.""" - if upload_handler is None: - return None - resolved = upload_handler.resolve_upload( - upload_id, - owner=owner, - auth_manager=auth_manager, - ) - if not isinstance(resolved, dict) or not resolved: - return None - path = resolved.get("path") - upload_dir = getattr(upload_handler, "upload_dir", None) - if path and upload_dir and not _upload_path_inside(upload_dir, path): - logger.warning("Upload path outside upload directory: %s", path) - return None - return path - - -def _locate_upload( - upload_dir: str, - file_id: str, - owner: Optional[str] = None, - auth_manager=None, - upload_handler: Any = None, -): - """Find an upload by its filename ID via UploadHandler.resolve_upload.""" - if upload_handler is None: - from src.upload_handler import UploadHandler - - base_dir = os.path.dirname(os.path.abspath(upload_dir)) - upload_handler = UploadHandler(base_dir, upload_dir) - return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager) - - -def _assert_pdf_marker_upload_owned( - request: Request, - content: str, - user: Optional[str], - upload_handler: Any, -) -> None: - """Reject document content whose pdf_source marker points at another user's upload.""" - if upload_handler is None: - return - from src.pdf_form_doc import find_source_upload_id - - upload_id = find_source_upload_id(content or "") - if not upload_id: - return - auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) - if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager): - raise HTTPException( - 400, - "Document PDF marker references an upload you do not own", - ) - - -def _derive_title(content: str) -> str: - """Derive a title from document content.""" - import re - if not isinstance(content, str): - return "Untitled" - text = content.strip() - if not text: - return "Untitled" - - # Markdown header - md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE) - if md: - title = md.group(1).strip() - if len(title) > 50: - title = title[:48] + "…" - return title - - # HTML heading - html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE) - if html: - title = html.group(1).strip() - if len(title) > 50: - title = title[:48] + "…" - return title - - # First non-empty line (if short enough) - for line in text.split('\n'): - line = line.strip() - if line and 2 <= len(line) <= 60: - title = re.sub(r'[:#*`]+$', '', line).strip() - if title and len(title) > 50: - title = title[:48] + "…" - return title or "Untitled" - - return "Untitled" +_sys.modules[__name__] = _canonical diff --git a/routes/document_routes.py b/routes/document_routes.py index dae8b09fa..dd13e3c60 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -1,1810 +1,17 @@ -"""Document routes — CRUD for living documents with version history.""" +"""Backward-compat shim — canonical location is routes/document/document_routes.py. -import uuid -import logging -from datetime import datetime, timezone -from typing import Dict, Any, List, Optional +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.document_routes``, ``from routes.document_routes import +X``, ``importlib.import_module("routes.document_routes")``, and the +``import ... as droutes`` + ``droutes.SessionLocal = ...`` / +``monkeypatch.setattr(droutes, ...)`` pattern used by multiple tests all +operate on the *same* object the application actually uses. Keeps existing +import paths working after slice 2m (#4082/#4071). Source-introspection tests +read the canonical file by path. +""" -from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form +import sys as _sys -from sqlalchemy import case, func, or_ -from core.database import SessionLocal, Document, DocumentVersion -from core.database import Session as DbSession -from src.auth_helpers import get_current_user, _auth_disabled -from src.constants import MAIL_ATTACHMENTS_DIR -from src.upload_handler import reserve_upload_references +from routes.document import document_routes as _canonical # noqa: F401 -logger = logging.getLogger(__name__) - - -def _get_session_or_404(db, session_id: str, user: Optional[str]): - session = db.query(DbSession).filter(DbSession.id == session_id).first() - if not session: - raise HTTPException(404, "Session not found") - if user and session.owner != user: - raise HTTPException(404, "Session not found") - return session - - -def _aggregate_language_facets(lang_rows): - """Sum document counts per display language for the library facet. - - NULL-language and explicit "text" rows share the "text" bucket (the - language filter treats them as one), so they must be ADDED. The old dict - comprehension keyed both to "text", silently overwriting one group and - undercounting the facet versus what the filter actually returns. - """ - out = {} - for lang, cnt in lang_rows: - key = lang or "text" - out[key] = out.get(key, 0) + cnt - return out - - -def _library_language_for_document(doc: Document) -> str: - """Return the display language used by the document library. - - PDF documents are stored as markdown wrappers so the editor can preserve - extracted text, form fields, and annotations. The library should still - identify them as PDFs instead of exposing that internal wrapper format. - """ - from src.pdf_form_doc import find_source_upload_id - - if find_source_upload_id(doc.current_content or ""): - return "pdf" - return doc.language or "text" - - -def _email_source_key(content: str) -> tuple[str, str]: - """Return the source email identity embedded in an email draft document.""" - import re - - text = content or "" - uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text) - folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text) - uid = (uid_m.group(1).strip() if uid_m else "") - folder = (folder_m.group(1).strip() if folder_m else "INBOX") - return uid, folder - - -from routes.document_helpers import ( - DocumentCreate, DocumentUpdate, DocumentPatch, - _doc_to_dict, _version_to_dict, - _verify_doc_owner, _owner_session_filter, - _slug, _resolve_user_upload_path, _assert_pdf_marker_upload_owned, _derive_title, - _PDF_RENDER_SCALE, -) - - -def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: - router = APIRouter(tags=["documents"]) - - def _reserve_document_uploads(user: Optional[str], content: str) -> None: - missing_id = reserve_upload_references(upload_handler, user, content) - if missing_id: - raise HTTPException( - 409, - f"Referenced upload is no longer available: {missing_id}", - ) - - def _locate_current_user_upload(request: Request, upload_id: str, user: Optional[str]): - if upload_handler is None: - return None - auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) - return _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager) - - def _load_pdf_viewer_fitz(): - from src.pdf_runtime import load_pymupdf_for_pdf_viewer - - try: - return load_pymupdf_for_pdf_viewer() - except RuntimeError as exc: - raise HTTPException(503, str(exc)) from exc - - # ---- POST /api/document ---- - @router.post("/api/document") - async def create_document(request: Request, req: DocumentCreate) -> Dict[str, Any]: - from src.auth_helpers import require_privilege - user = require_privilege(request, "can_use_documents") - db = SessionLocal() - try: - # session_id is optional: a doc can be a session-less "library" doc - # (e.g. files imported from the library) — session_id is nullable and - # the doc is owner-stamped, so it lives in the library on its own. - session = None - if req.session_id: - # Match the lenient ownership model the rest of the app uses - # (see _owner_filter): only block when an AUTHENTICATED user is - # writing into a DIFFERENT user's session. In single-user / - # unconfigured / localhost-bypass mode, falsey users preserve - # the existing lenient path. - session = _get_session_or_404(db, req.session_id, user) - - # If no language was supplied (e.g. cloning a doc whose language - # was never set), detect it from the content rather than storing - # NULL — which made the editor fall back to plain text. Defaults - # to markdown for prose. - language = req.language - if not language: - from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content - language = _sniff_doc_language(req.content) - else: - from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content - if _looks_like_email_document(req.content, req.title): - language = "email" - - _reserve_document_uploads(user, req.content) - _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler) - - # Reply drafts are keyed to the source email. If a UI/tool path tries - # to create a second draft for the same email in the same chat, - # update the existing draft instead so quoted thread history stays - # attached to the visible document. - if language == "email" and req.session_id: - source_uid, source_folder = _email_source_key(req.content) - if source_uid: - candidates = ( - db.query(Document) - .filter(Document.session_id == req.session_id) - .filter(Document.is_active == True) - .filter(Document.language == "email") - .order_by(Document.updated_at.desc()) - .limit(25) - .all() - ) - for existing in candidates: - old_uid, old_folder = _email_source_key(existing.current_content or "") - if old_uid != source_uid or old_folder != source_folder: - continue - merged = _coerce_email_document_content(existing.current_content or "", req.content) - if existing.current_content != merged: - new_ver = (existing.version_count or 1) + 1 - existing.current_content = merged - existing.title = req.title or existing.title - existing.version_count = new_ver - db.add(DocumentVersion( - id=str(uuid.uuid4()), - document_id=existing.id, - version_number=new_ver, - content=merged, - summary="Updated existing email draft", - source="user", - )) - db.commit() - db.refresh(existing) - return _doc_to_dict(existing) - - doc_id = str(uuid.uuid4()) - ver_id = str(uuid.uuid4()) - - doc = Document( - id=doc_id, - session_id=req.session_id, - title=req.title, - language=language, - current_content=req.content, - version_count=1, - is_active=True, - # Stamp ownership directly so the doc survives its session - # being deleted. Fall back to the session's owner when the - # request is unauthenticated (single-user / localhost bypass). - owner=user or (session.owner if session else None), - ) - ver = DocumentVersion( - id=ver_id, - document_id=doc_id, - version_number=1, - content=req.content, - summary="Initial version", - source="user", - ) - db.add(doc) - db.add(ver) - db.commit() - db.refresh(doc) - try: - from src.event_bus import fire_event - fire_event("document_created", doc.owner) - except Exception: - logger.debug("document_created event dispatch failed", exc_info=True) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - logger.error(f"Failed to create document: {e}") - raise HTTPException(500, f"Failed to create document: {e}") - finally: - db.close() - - # ---- POST /api/documents/import-pdf ---- - @router.post("/api/documents/import-pdf") - async def import_pdf( - request: Request, - file: UploadFile = File(...), - session_id: Optional[str] = Form(None), - ) -> Dict[str, Any]: - """Upload a PDF and create the matching Document. - - Detects AcroForm fields — if any, creates a form-backed markdown doc - (clickable inputs in the PDF view). Otherwise creates a plain PDF doc - with a `pdf_source` marker so the viewer renders the pages without - overlays. - """ - from src.pdf_forms import has_form_fields, extract_fields - from src.pdf_form_doc import ( - save_field_sidecar, - create_form_markdown_document, - create_plain_pdf_document, - ) - from src.document_processor import _process_pdf, strip_pdf_content_marker - import os - - from src.auth_helpers import require_privilege - user = require_privilege(request, "can_use_documents") - - # session_id is optional — a library import isn't tied to a chat. When - # given, validate it; otherwise the PDF becomes a session-less library - # doc (the doc creators below already handle a missing session). - if session_id: - db = SessionLocal() - try: - _get_session_or_404(db, session_id, user) - finally: - db.close() - - if upload_handler is None: - raise HTTPException(500, "Upload handler not configured") - - client_ip = request.client.host if request.client else "unknown" - try: - meta = upload_handler.save_upload(file, client_ip, owner=user) - except HTTPException: - raise - except Exception as e: - logger.error(f"PDF import save_upload failed: {e}") - raise HTTPException(500, f"Upload failed: {e}") - - upload_id = meta["id"] - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(500, "Saved PDF could not be located") - - title = os.path.splitext(meta.get("original_name") or meta.get("name") or upload_id)[0] - try: - body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user)) - except Exception: - body_text = None - - is_form = False - try: - is_form = has_form_fields(pdf_path) - except Exception as e: - logger.warning(f"has_form_fields failed for {pdf_path}: {e}") - - if is_form: - fields = extract_fields(pdf_path) - save_field_sidecar(pdf_path, fields) - doc_id = create_form_markdown_document( - session_id=session_id, - fields=fields, - upload_id=upload_id, - title=title, - intro_text=body_text, - ) - else: - doc_id = create_plain_pdf_document( - session_id=session_id, - upload_id=upload_id, - title=title, - body_text=body_text, - ) - - if not doc_id: - raise HTTPException(500, "Failed to create document for PDF") - - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(500, "Created document not found") - # The PDF doc creators stamp owner from the session only; a - # session-less library import leaves owner NULL, which the Library's - # owner filter then hides. Stamp the requesting user so it shows. - if not doc.owner and user: - doc.owner = user - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - finally: - db.close() - - # ---- GET /api/documents/library ---- - @router.get("/api/documents/library") - async def documents_library( - request: Request, - search: Optional[str] = Query(None), - language: Optional[str] = Query(None), - sort: str = Query("recent"), - offset: int = Query(0, ge=0), - limit: int = Query(20, ge=1, le=50), - archived: bool = Query(False), - ) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - from sqlalchemy import or_ - pdf_marker_cond = or_( - Document.current_content.like('%<!-- pdf_source upload_id="%'), - Document.current_content.like('%<!-- pdf_form_source upload_id="%'), - ) - library_language_expr = case( - (pdf_marker_cond, "pdf"), - (Document.language.is_(None), "text"), - else_=Document.language, - ) - # Archived view shows ONLY archived docs; the default view excludes - # them (NULL = legacy rows that predate the column = not archived). - _arch_cond = (Document.archived == True) if archived else or_( - Document.archived == False, Document.archived.is_(None)) - # Language facet counts (owner-filtered). PDF documents are stored - # as markdown wrappers, so group by the library display language - # instead of the raw stored language. - lang_q = ( - db.query(library_language_expr, func.count(Document.id)) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == True).filter(_arch_cond) - ) - lang_q = _owner_session_filter(lang_q, user) - lang_rows = lang_q.group_by(library_language_expr).all() - languages = _aggregate_language_facets(lang_rows) - - # Session count (owner-filtered) - sc_q = ( - db.query(func.count(func.distinct(Document.session_id))) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == True).filter(_arch_cond) - ) - sc_q = _owner_session_filter(sc_q, user) - session_count = sc_q.scalar() - - # Base query - q = ( - db.query(Document, DbSession.name) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == True).filter(_arch_cond) - ) - q = _owner_session_filter(q, user) - - # Search filter — split on whitespace and require EACH term to - # match (title OR content). A single `%foo bar%` LIKE only matched - # the exact adjacent phrase, so any multi-word query with a space - # silently returned nothing. Per-term AND makes "machine learning" - # match docs containing both words regardless of position/order. - if search: - for tok in search.split(): - term = f"%{tok}%" - q = q.filter( - Document.title.ilike(term) | Document.current_content.ilike(term) - ) - - # Language filter. "pdf" is a display language derived from the - # source marker; "markdown" excludes those wrappers. - if language: - if language == "text": - q = q.filter((Document.language == None) | (Document.language == "text")) - elif language == "pdf": - q = q.filter(pdf_marker_cond) - else: - q = q.filter(Document.language == language) - if language == "markdown": - q = q.filter(~pdf_marker_cond) - - # Total before pagination - total = q.count() - - # Sorting - if sort == "oldest": - q = q.order_by(Document.created_at.asc()) - elif sort == "edits": - q = q.order_by(Document.version_count.desc()) - elif sort == "alpha": - q = q.order_by(Document.title.asc()) - else: # recent - q = q.order_by(Document.updated_at.desc()) - - rows = q.offset(offset).limit(limit).all() - - documents = [] - for doc, session_name in rows: - documents.append({ - "id": doc.id, - "session_id": doc.session_id, - "session_name": session_name, - "title": doc.title, - "language": _library_language_for_document(doc), - "preview": (doc.current_content or "")[:500], - "version_count": doc.version_count, - "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None, - "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None, - }) - - return { - "documents": documents, - "total": total, - "languages": languages, - "session_count": session_count, - } - except Exception as e: - logger.error(f"Failed to fetch document library: {e}") - raise HTTPException(500, f"Failed to fetch document library: {e}") - finally: - db.close() - - # ---- GET /api/documents/{session_id} ---- - @router.get("/api/documents/{session_id}") - async def list_documents(request: Request, session_id: str) -> List[Dict[str, Any]]: - user = get_current_user(request) - db = SessionLocal() - try: - if not user: - if not _auth_disabled(): - raise HTTPException(403, "Authentication required") - # v2 review HIGH-9: raise 403 explicitly when the caller - # can't see this session, instead of returning [] which the - # UI treats identically to "no docs" and silently masks - # auth failures. - _get_session_or_404(db, session_id, user) - q = db.query(Document).filter( - Document.session_id == session_id - ) - if user: - q = q.filter(or_(Document.owner == user, Document.owner.is_(None))) - docs = q.order_by(Document.created_at.desc()).all() - return [_doc_to_dict(d) for d in docs] - finally: - db.close() - - # ---- GET /api/document/{doc_id} ---- - @router.get("/api/document/{doc_id}") - async def get_document(request: Request, doc_id: str) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - return _doc_to_dict(doc) - finally: - db.close() - - # ---- POST /api/document/{doc_id}/archive — soft-archive / restore ---- - @router.post("/api/document/{doc_id}/archive") - async def archive_document(request: Request, doc_id: str, archived: bool = Query(True)) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - doc.archived = bool(archived) - db.commit() - return {"ok": True, "id": doc_id, "archived": doc.archived} - finally: - db.close() - - # ---- POST /api/document/{doc_id}/extract-pdf-text ---- - @router.post("/api/document/{doc_id}/extract-pdf-text") - async def extract_pdf_text(request: Request, doc_id: str) -> Dict[str, Any]: - """Re-run pypdf+VL text extraction against the PDF linked to this doc - and merge the result into the doc's markdown content. Idempotent — the - existing body (everything below the title heading) is replaced. - - Lets the AI see PDF contents for old docs that were imported before - text extraction was wired, plus for scanned/image-only PDFs where the - VL model picks up text the basic pypdf path missed.""" - import re - from src.document_processor import _process_pdf, strip_pdf_content_marker - from src.pdf_form_doc import find_source_upload_id - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - content = doc.current_content or "" - upload_id = find_source_upload_id(content) - if not upload_id: - raise HTTPException(400, "Document is not a PDF — no pdf_source marker found") - - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, "Source PDF could not be located") - - try: - body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user)) - except Exception as e: - logger.error(f"extract_pdf_text failed for {pdf_path}: {e}") - raise HTTPException(500, f"Extraction failed: {e}") - - if not body_text: - return {"ok": True, "id": doc_id, "extracted": False, "reason": "No readable content"} - - # Preserve everything up through the title (front-matter marker + - # first H1) and replace the rest with the freshly extracted text. - head_re = re.compile(r'^(<!--[^>]+-->\s*\n+#[^\n]*\n+)', re.MULTILINE) - head_match = head_re.match(content) - head = head_match.group(1) if head_match else (content.splitlines()[0] + "\n\n# " + (doc.title or "PDF") + "\n\n") - doc.current_content = head + body_text.strip() + "\n" - doc.version_count = (doc.version_count or 1) + 1 - db.add(DocumentVersion( - id=str(__import__("uuid").uuid4()), - document_id=doc_id, - version_number=doc.version_count, - content=doc.current_content, - summary="PDF text re-extracted (OCR)", - source="ocr", - )) - db.commit() - return {"ok": True, "id": doc_id, "extracted": True, "chars": len(body_text)} - finally: - db.close() - - # ---- POST /api/documents/export-zip — bundle selected docs into a .zip ---- - @router.post("/api/documents/export-zip") - async def documents_export_zip(request: Request): - """Zip the selected documents (each as a text file with the right - extension) — mirrors the gallery's bulk download-zip so multi-export - is one file instead of a blocked flood of individual downloads.""" - user = get_current_user(request) - try: - data = await request.json() - except Exception as e: - logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e) - data = {} - ids = data.get("ids") or [] - if not ids: - raise HTTPException(400, "No documents specified") - _ext = { - "javascript": ".js", "python": ".py", "html": ".html", "css": ".css", - "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh", - "sql": ".sql", "rust": ".rs", "go": ".go", "java": ".java", "c": ".c", - "cpp": ".cpp", "typescript": ".ts", "ruby": ".rb", "php": ".php", - "text": ".txt", "xml": ".xml", "toml": ".toml", "ini": ".ini", - } - db = SessionLocal() - try: - import io - import re - import zipfile - from fastapi import Response - docs = db.query(Document).filter(Document.id.in_(ids)).all() - buf = io.BytesIO() - used = set() - wrote = 0 - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - for doc in docs: - try: - _verify_doc_owner(db, doc, user) - except HTTPException: - continue # skip docs the user doesn't own - ext = _ext.get(doc.language or "text", ".txt") - base = (doc.title or "document").strip() or "document" - base = re.sub(r"[^\w\-. ]+", "", base)[:60].strip() or doc.id - name = base if "." in base else base + ext - i = 1 - while name in used: - name = f"{base}-{i}" + ("" if "." in base else ext) - i += 1 - used.add(name) - zf.writestr(name, doc.current_content or "") - wrote += 1 - if not wrote: - raise HTTPException(404, "No documents found") - return Response( - content=buf.getvalue(), - media_type="application/zip", - headers={"Content-Disposition": 'attachment; filename="documents.zip"'}, - ) - finally: - db.close() - - # ---- PUT /api/document/{doc_id} — user manual edit ---- - # Coalesce window: if the last user version was saved within this many - # seconds, update it in-place (user is still actively editing). - # Once the gap exceeds this, the next save creates a new version. - VERSION_COALESCE_SECONDS = 60 - - @router.put("/api/document/{doc_id}") - async def update_document(request: Request, doc_id: str, req: DocumentUpdate) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - incoming_content = req.content - from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document - is_email_doc = ( - (doc.language or "").lower() == "email" - or _looks_like_email_document(doc.current_content or "", doc.title or "") - or _looks_like_email_document(req.content or "", doc.title or "") - ) - if is_email_doc: - incoming_content = _coerce_email_document_content(doc.current_content or "", req.content) - doc.language = "email" - - # Skip if content is identical unless the caller explicitly wants - # a checkpoint version from the current editor state. - if doc.current_content == incoming_content and not req.force_version: - return _doc_to_dict(doc) - - _reserve_document_uploads(user, incoming_content) - _assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler) - - # Check if we can coalesce with the latest version - latest_ver = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id, - ).order_by(DocumentVersion.version_number.desc()).first() - - now = datetime.now(timezone.utc) - coalesced = False - if latest_ver and latest_ver.source == "user" and not req.force_version: - ver_time = latest_ver.created_at - if ver_time.tzinfo is None: - ver_time = ver_time.replace(tzinfo=timezone.utc) - age = (now - ver_time).total_seconds() - if age < VERSION_COALESCE_SECONDS: - # Update the existing version in-place - latest_ver.content = incoming_content - latest_ver.created_at = now - if req.summary: - latest_ver.summary = req.summary - coalesced = True - - if not coalesced: - new_ver = doc.version_count + 1 - ver = DocumentVersion( - id=str(uuid.uuid4()), - document_id=doc_id, - version_number=new_ver, - content=incoming_content, - summary=req.summary or "Manual edit", - source="user", - ) - doc.version_count = new_ver - db.add(ver) - - doc.current_content = incoming_content - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, f"Failed to update document: {e}") - finally: - db.close() - - # ---- PATCH /api/document/{doc_id} — metadata only ---- - @router.patch("/api/document/{doc_id}") - async def patch_document(request: Request, doc_id: str, req: DocumentPatch) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - if req.title is not None: - doc.title = req.title - if req.language is not None: - doc.language = req.language - if req.session_id is not None: - # Empty string = unlink from session - if req.session_id: - _get_session_or_404(db, req.session_id, user) - doc.session_id = req.session_id if req.session_id else None - if not req.session_id: - # Tab closed / doc detached from its session — drop the - # in-memory active-doc pointer so the last-resort injection - # path doesn't re-surface this doc in a later chat (#1160). - try: - from src.agent_tools.document_tools import clear_active_document - clear_active_document(doc_id) - except Exception as e: - logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e) - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, str(e)) - finally: - db.close() - - # ---- DELETE /api/document/{doc_id} — soft delete ---- - @router.delete("/api/document/{doc_id}") - async def delete_document(request: Request, doc_id: str) -> Dict[str, str]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - doc.is_active = False - # Closed/deleted — drop the in-memory active-doc pointer so it isn't - # re-injected into a later, unrelated chat (#1160). - try: - from src.agent_tools.document_tools import clear_active_document - clear_active_document(doc_id) - except Exception: - pass - db.commit() - return {"status": "deleted", "id": doc_id} - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, str(e)) - finally: - db.close() - - # ---- GET /api/document/{doc_id}/versions ---- - @router.get("/api/document/{doc_id}/versions") - async def list_versions(request: Request, doc_id: str) -> List[Dict[str, Any]]: - user = get_current_user(request) - db = SessionLocal() - try: - # Verify ownership before listing versions - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - versions = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id - ).order_by(DocumentVersion.version_number.desc()).all() - return [{ - "id": v.id, - "version_number": v.version_number, - "content": v.content, - "summary": v.summary, - "source": v.source, - "created_at": v.created_at.isoformat() if v.created_at else None, - } for v in versions] - finally: - db.close() - - # ---- GET /api/document/{doc_id}/version/{num} ---- - @router.get("/api/document/{doc_id}/version/{num}") - async def get_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - # Verify ownership - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - ver = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id, - DocumentVersion.version_number == num, - ).first() - if not ver: - raise HTTPException(404, "Version not found") - return _version_to_dict(ver) - finally: - db.close() - - # ---- POST /api/document/{doc_id}/restore/{num} ---- - @router.post("/api/document/{doc_id}/restore/{num}") - async def restore_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - old_ver = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id, - DocumentVersion.version_number == num, - ).first() - if not old_ver: - raise HTTPException(404, "Version not found") - - new_ver_num = doc.version_count + 1 - ver = DocumentVersion( - id=str(uuid.uuid4()), - document_id=doc_id, - version_number=new_ver_num, - content=old_ver.content, - summary=f"Restored from v{num}", - source="user", - ) - doc.current_content = old_ver.content - doc.version_count = new_ver_num - db.add(ver) - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, str(e)) - finally: - db.close() - - # ---- POST /api/documents/tidy — clean up broken/empty documents ---- - @router.post("/api/documents/tidy") - async def tidy_documents(request: Request) -> Dict[str, Any]: - """Fix empty titles and remove broken/empty documents (user's docs only).""" - user = get_current_user(request) - db = SessionLocal() - try: - q = ( - db.query(Document) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == True) - .filter((Document.archived == False) | (Document.archived.is_(None))) - ) - q = _owner_session_filter(q, user) - docs = q.all() - fixed_titles = 0 - deleted = 0 - - # Same junk-detection logic as the scheduled tidy_documents - # action (src/document_actions.py). Keep these two in sync. - import re as _re - from src.document_actions import _JUNK_TITLES - - to_delete = [] - now = datetime.now(timezone.utc) - for doc in docs: - created = doc.created_at - if created and created.tzinfo is None: - created = created.replace(tzinfo=timezone.utc) - - # Skip freshly created documents to avoid deleting them while the user is actively editing - if created and (now - created).total_seconds() < 900: # 15 minutes - continue - - content = (doc.current_content or "").strip() - title_raw = (doc.title or "").strip() - title = title_raw.lower() - is_fresh_empty = ( - not content - and created is not None - and (now - created).total_seconds() < 1800 - ) - if is_fresh_empty: - continue - - # Strip markdown noise to get a "real" character count - stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE) - stripped = _re.sub(r"[*_`>\-=]+", "", stripped) - stripped = _re.sub(r"\s+", " ", stripped).strip() - real_len = len(stripped) - - # Detect email-scaffold stubs: "To: \nSubject: \n---\n" style - # bodies with nothing typed in. Stub = every meaningful line - # is a header label (To:/From:/Subject:/...) with no real - # value (blank, "empty", "(empty)", "-", "none", "n/a"). - _is_email_stub = False - _HEADER_RE = _re.compile(r"^(to|from|cc|bcc|subject|reply-to):\s*(.*)$", _re.I) - _PLACEHOLDER_VALS = {"", "empty", "(empty)", "-", "—", "none", "n/a", "na", "tbd"} - if title in ("new email", "new mail", "new message") or doc.language == "email": - body_lines = [ln.strip() for ln in content.split("\n") - if ln.strip() and ln.strip() != "---"] - def _is_filler(ln): - m = _HEADER_RE.match(ln) - if not m: - return False - val = (m.group(2) or "").strip().lower() - return val in _PLACEHOLDER_VALS - has_real_body = any(not _is_filler(ln) for ln in body_lines) - if body_lines and not has_real_body: - _is_email_stub = True - - # Hard-delete obviously empty / junk documents - if not content or content in ("", "# Untitled"): - to_delete.append(doc); deleted += 1; continue - if _is_email_stub: - to_delete.append(doc); deleted += 1; continue - if title in _JUNK_TITLES: - to_delete.append(doc); deleted += 1; continue - - # Fix empty or placeholder titles on survivors - if not title_raw or title_raw == "Untitled": - new_title = _derive_title(content) - if new_title and new_title != "Untitled": - doc.title = new_title - fixed_titles += 1 - - for doc in to_delete: - db.delete(doc) - - # Also clean up inactive empty docs from previous soft-deletes - inactive_q = ( - db.query(Document) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == False) - .filter((Document.current_content == None) | (Document.current_content == "")) - ) - inactive_q = _owner_session_filter(inactive_q, user) - inactive_docs = inactive_q.all() - for doc in inactive_docs: - db.delete(doc) - deleted += len(inactive_docs) - - db.commit() - return { - "fixed_titles": fixed_titles, - "deleted": deleted, - "message": f"Fixed {fixed_titles} title{'s' if fixed_titles != 1 else ''}, removed {deleted} empty document{'s' if deleted != 1 else ''}", - } - except Exception as e: - db.rollback() - logger.error(f"Document tidy failed: {e}") - raise HTTPException(500, f"Tidy failed: {e}") - finally: - db.close() - - # ---- POST /api/documents/ai-tidy — AI-powered cleanup of junk/test documents ---- - @router.post("/api/documents/ai-tidy") - async def ai_tidy_documents(request: Request) -> Dict[str, Any]: - """Use AI to judge if documents are junk/test/accidental, then delete them. - Caches verdicts so previously-reviewed docs are skipped.""" - from src.task_endpoint import resolve_task_endpoint - from src.endpoint_resolver import resolve_endpoint - from src.llm_core import llm_call_async - - user = get_current_user(request) - url, model, headers = resolve_task_endpoint(owner=user or None) - if not url or not model: - # Fall back to default endpoint - url, model, headers = resolve_endpoint("default", owner=user or None) - if not url or not model: - raise HTTPException(500, "No endpoint configured for AI tidy") - - db = SessionLocal() - try: - q = ( - db.query(Document) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == True) - .filter((Document.archived == False) | (Document.archived.is_(None))) - ) - q = _owner_session_filter(q, user) - docs = q.all() - - # Only review docs that haven't been reviewed yet - to_review = [d for d in docs if not d.tidy_verdict] - if not to_review: - return {"deleted": 0, "reviewed": 0, "message": "All documents already reviewed"} - - # Build a batch prompt — review up to 30 at a time - batch = to_review[:30] - doc_list = [] - for i, doc in enumerate(batch): - preview = (doc.current_content or "")[:300].strip() - doc_list.append(f"[{i}] title=\"{doc.title}\" lang={doc.language or 'text'} content_preview=\"{preview}\"") - - prompt = ( - "You are a document library cleaner. For each document below, decide if it is JUNK " - "(test, accidental, placeholder, empty-ish, tool-test, throwaway) or KEEP (real content worth saving).\n\n" - "Respond with ONLY a JSON array of verdicts, one per document, like: [\"junk\",\"keep\",\"junk\",...]\n" - "No explanation, no markdown, just the JSON array.\n\n" - + "\n".join(doc_list) - ) - - response = await llm_call_async( - url, model, - [{"role": "system", "content": "You classify documents as junk or keep. Respond only with a JSON array."}, - {"role": "user", "content": prompt}], - temperature=0.1, - max_tokens=200, - headers=headers, - timeout=30, - ) - - # Parse verdicts - import re - match = re.search(r'\[.*?\]', response, re.DOTALL) - if not match: - raise HTTPException(500, "AI returned invalid response") - - import json as _json - verdicts = _json.loads(match.group()) - - deleted = 0 - reviewed = 0 - for i, doc in enumerate(batch): - if i >= len(verdicts): - break - verdict = str(verdicts[i] or "").lower().strip() - if verdict == "junk": - doc.tidy_verdict = "junk" - db.delete(doc) - deleted += 1 - else: - doc.tidy_verdict = "keep" - reviewed += 1 - - db.commit() - return { - "deleted": deleted, - "reviewed": reviewed, - "remaining": len(to_review) - len(batch), - "message": f"Reviewed {reviewed}, removed {deleted} junk document{'s' if deleted != 1 else ''}", - } - except HTTPException: - raise - except Exception as e: - db.rollback() - logger.error(f"AI tidy failed: {e}") - raise HTTPException(500, f"AI tidy failed: {e}") - finally: - db.close() - - # ---- POST /api/document/{doc_id}/export-pdf/preview ---- - @router.post("/api/document/{doc_id}/export-pdf/preview") - async def export_pdf_preview(doc_id: str, request: Request) -> Dict[str, Any]: - """Return the field-value mapping that would be written to the PDF. - - Frontend shows this in a confirmation modal so the user can spot/fix - any wrong values before triggering the actual download. - """ - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") - - fields = load_field_sidecar(pdf_path) - if not fields: - raise HTTPException(404, "Field schema sidecar missing for source PDF") - - values = parse_markdown_to_values(doc.current_content or "") - field_meta = {f["name"]: f for f in fields} - - preview = [] - for name, current in values.items(): - meta = field_meta.get(name) - if not meta: - continue - preview.append({ - "name": name, - "label": meta.get("label") or name, - "type": meta.get("type"), - "options": meta.get("options") or [], - "page": meta.get("page"), - "value": current, - }) - - unknown = [ - name for name in values - if name not in field_meta - ] - return { - "doc_id": doc_id, - "upload_id": upload_id, - "fields": preview, - "unknown_fields": unknown, - "total": len(fields), - "filled": sum(1 for p in preview if p["value"] not in ("", False, None)), - } - finally: - db.close() - - # ---- GET /api/document/{doc_id}/render-pages ---- - @router.get("/api/document/{doc_id}/render-pages") - async def render_pages(doc_id: str, request: Request) -> Dict[str, Any]: - """Return per-page metadata for the interactive PDF view. - - Each page entry has its rendered-image dimensions (matching what - /page/{n}.png returns at the same DPI) plus the list of form fields - on that page with their rects translated to image-pixel coordinates. - Frontend overlays HTML form controls at those positions. - """ - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found") - - fitz = _load_pdf_viewer_fitz() - schema = load_field_sidecar(pdf_path) or [] - values = parse_markdown_to_values(doc.current_content or "") - - # Group fields by page - by_page: Dict[int, list] = {} - for f in schema: - by_page.setdefault(f["page"], []).append(f) - - scale = _PDF_RENDER_SCALE - pdf_doc = fitz.open(pdf_path) - try: - pages_out = [] - for page_index in range(pdf_doc.page_count): - page = pdf_doc[page_index] - page_no = page_index + 1 - pw, ph = page.rect.width, page.rect.height - img_w = int(pw * scale) - img_h = int(ph * scale) - fields_out = [] - for f in by_page.get(page_no, []): - x0, y0, x1, y1 = f["rect"] - fields_out.append({ - "name": f["name"], - "type": f["type"], - "label": f.get("label") or "", - "options": f.get("options") or [], - "value": values.get(f["name"], f.get("value", "")), - "rect_px": [ - int(x0 * scale), int(y0 * scale), - int(x1 * scale), int(y1 * scale), - ], - }) - pages_out.append({ - "page": page_no, - "width": img_w, - "height": img_h, - "fields": fields_out, - }) - return {"doc_id": doc_id, "scale": scale, "pages": pages_out} - finally: - pdf_doc.close() - finally: - db.close() - - # ---- GET /api/document/{doc_id}/page/{n}.png ---- - @router.get("/api/document/{doc_id}/page/{page_no}.png") - async def render_page_png(doc_id: str, page_no: int, request: Request): - """Render one page of the source PDF as a PNG (no values stamped — the - frontend overlays HTML form inputs on top).""" - from fastapi.responses import Response - from src.pdf_form_doc import find_source_upload_id - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, "Source PDF not found") - finally: - db.close() - - fitz = _load_pdf_viewer_fitz() - pdf_doc = fitz.open(pdf_path) - try: - if page_no < 1 or page_no > pdf_doc.page_count: - raise HTTPException(404, "Page out of range") - page = pdf_doc[page_no - 1] - mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) - pix = page.get_pixmap(matrix=mat, alpha=False) - png_bytes = pix.tobytes("png") - return Response( - content=png_bytes, - media_type="image/png", - headers={"Cache-Control": "public, max-age=3600"}, - ) - finally: - pdf_doc.close() - - # ---- POST /api/document/{doc_id}/ai-fill-annotations ---- - @router.post("/api/document/{doc_id}/ai-fill-annotations") - async def ai_fill_annotations(doc_id: str, request: Request) -> Dict[str, Any]: - """Ask a vision-capable LLM to locate fillable areas on a flat PDF and - propose annotation values for each, given a free-form user instruction. - - Returns a list of annotations: [{page, x, y, w, h, value}] where x/y/w/h - are page-percentages (0–100) — same coordinate system as the freeform - annotations the frontend already renders. - """ - import base64 - import json - import fitz - from src.pdf_form_doc import find_source_upload_id - from src.document_processor import _resolve_vl_model, _load_vl_settings - from src.llm_core import llm_call_async - - body = await request.json() if request.headers.get("content-type", "").startswith("application/json") else {} - instruction = (body or {}).get("instruction", "").strip() - if not instruction: - raise HTTPException(400, "instruction is required") - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, "Source PDF not found") - finally: - db.close() - - # Resolve VL model (admin-configured or auto-detected vision-capable) - settings = _load_vl_settings() - vl_model = settings.get("vision_model", "") - try: - url, model_id, headers = _resolve_vl_model(vl_model, owner=user) - except Exception as e: - raise HTTPException(503, f"No vision model available: {e}") - - system_prompt = ( - "You analyze rendered PDF page images and propose values to fill in. " - "For each blank line, box, underscore, or labeled space on the page that " - "should be filled given the user's instruction, output one annotation. " - "Coordinates are percentages (0-100) of the page width/height with the " - "origin at top-left. Width/height should match the visible blank box. " - "Return ONLY a JSON array, no prose, no markdown fences. Each entry: " - '{"x": number, "y": number, "w": number, "h": number, "value": string}. ' - "If a region should not be filled, omit it. If nothing should be filled, " - "return []." - ) - - all_annotations = [] - pdf_doc = fitz.open(pdf_path) - try: - for page_index in range(pdf_doc.page_count): - page = pdf_doc[page_index] - mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) - pix = page.get_pixmap(matrix=mat, alpha=False) - png_bytes = pix.tobytes("png") - b64 = base64.b64encode(png_bytes).decode("ascii") - - messages = [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": [ - { - "type": "text", - "text": ( - f"User instruction:\n{instruction}\n\n" - f"This is page {page_index + 1} of {pdf_doc.page_count}. " - "Return JSON array of annotations to add to this page." - ), - }, - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{b64}"}, - }, - ], - }, - ] - try: - raw = await llm_call_async( - url, model_id, messages, - temperature=0.1, max_tokens=2000, headers=headers, - ) - except Exception as e: - logger.error(f"VL call failed on page {page_index + 1}: {e}") - continue - - raw = (raw or "").strip() - if raw.startswith("```"): - raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip() - try: - parsed = json.loads(raw) - except Exception: - logger.warning(f"AI fill: page {page_index + 1} returned non-JSON: {raw[:200]}") - continue - if not isinstance(parsed, list): - continue - for item in parsed: - if not isinstance(item, dict): - continue - try: - x = float(item.get("x", 0)) - y = float(item.get("y", 0)) - w = float(item.get("w", 0)) - h = float(item.get("h", 0)) - value = str(item.get("value", "") or "") - except Exception: - continue - # Clamp + reject zero-size entries - if w <= 0.5 or h <= 0.3: - continue - x = max(0.0, min(99.0, x)) - y = max(0.0, min(99.0, y)) - w = max(0.5, min(100.0 - x, w)) - h = max(0.3, min(100.0 - y, h)) - if not value.strip(): - continue - all_annotations.append({ - "page": page_index + 1, - "x": round(x, 2), - "y": round(y, 2), - "w": round(w, 2), - "h": round(h, 2), - "value": value, - }) - finally: - pdf_doc.close() - - return {"annotations": all_annotations} - - # ---- GET /api/document/{doc_id}/render-pdf ---- - @router.get("/api/document/{doc_id}/render-pdf") - async def render_pdf(doc_id: str, request: Request): - """Inline PDF preview filled with the current markdown values. - - Same plumbing as the export route, but no signature stamping and - served inline (Content-Disposition: inline) so the browser can - embed it in an iframe. Cache-busted by the caller via query string. - """ - import base64 - import os - import tempfile - from fastapi.responses import FileResponse - from starlette.background import BackgroundTask - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, parse_markdown_annotations - from src.pdf_forms import fill_fields, stamp_annotations - from core.database import Signature - - # Track temp files for this request so they get unlinked AFTER - # the response is fully sent (BackgroundTask runs post-send). - _to_unlink: list[str] = [] - def _cleanup_temps(): - for _p in _to_unlink: - try: - os.unlink(_p) - except FileNotFoundError: - pass - except Exception as _e: - logger.warning(f"Could not unlink temp PDF {_p}: {_e}") - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found") - - # Fail fast with a clear 503 if the optional PyMuPDF dependency - # is missing — fill_fields/stamp_annotations will otherwise - # raise RuntimeError deep inside and bubble out as a 500. - # Mirrors the convention in _load_pdf_viewer_fitz above. - _load_pdf_viewer_fitz() - - values = parse_markdown_to_values(doc.current_content or "") - out_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(out_path) - try: - fill_fields(pdf_path, out_path, values) - except Exception as e: - logger.error(f"render_pdf fill_fields failed for {doc_id}: {e}") - _cleanup_temps() - raise HTTPException(500, f"PDF render failed: {e}") - - annotations = parse_markdown_annotations(doc.current_content or "") - if annotations: - ann_sig_ids = [ - a["value"][len("signature:"):].strip() - for a in annotations - if a.get("kind") == "signature" - and isinstance(a.get("value"), str) - and a["value"].startswith("signature:") - ] - ann_signature_pngs: dict[str, bytes] = {} - if ann_sig_ids: - # SECURITY: filter by owner so a caller can't reference - # someone else's signature ID from doc markdown and have - # it stamped/exported. - _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) - if user: - _sig_q = _sig_q.filter(Signature.owner == user) - sig_rows = _sig_q.all() - for s in sig_rows: - try: - ann_signature_pngs[s.id] = base64.b64decode(s.data_png) - except Exception as e: - logger.warning(f"Bad annotation signature data for {s.id}: {e}") - annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(annotated_path) - try: - stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) - out_path = annotated_path - except Exception as e: - logger.error(f"stamp_annotations (render) failed for {doc_id}: {e}") - - return FileResponse( - out_path, - media_type="application/pdf", - headers={"Content-Disposition": "inline"}, - background=BackgroundTask(_cleanup_temps), - ) - finally: - db.close() - - # ---- GET /api/document/{doc_id}/export-pdf ---- - @router.get("/api/document/{doc_id}/export-pdf") - async def export_pdf(doc_id: str, request: Request): - """Stream the filled PDF for download. - - Reads field values and signature selections from the markdown — there - is no separate confirmation step. Signature fields contain their - chosen signature ID encoded as `signature:<id>` in the value. - """ - import base64 - import os - import tempfile - from fastapi.responses import FileResponse - from starlette.background import BackgroundTask - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar, parse_markdown_annotations - from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations - from core.database import Signature - - _to_unlink: list[str] = [] - def _cleanup_temps(): - for _p in _to_unlink: - try: - os.unlink(_p) - except FileNotFoundError: - pass - except Exception as _e: - logger.warning(f"Could not unlink temp PDF {_p}: {_e}") - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") - - schema = load_field_sidecar(pdf_path) or [] - sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} - - all_values = parse_markdown_to_values(doc.current_content or "") - # Split: signature fields go to stamps, everything else to fill_fields - text_values: dict = {} - sig_ids: dict[str, str] = {} - for name, raw in all_values.items(): - if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): - sig_ids[name] = raw[len("signature:"):].strip() - elif name not in sig_field_names: - text_values[name] = raw - - stamps: dict = {} - if sig_ids: - # SECURITY: filter by owner — same reason as render_pdf. - _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) - if user: - _sig_q2 = _sig_q2.filter(Signature.owner == user) - rows = _sig_q2.all() - by_id = {s.id: s for s in rows} - for field_name, sid in sig_ids.items(): - s = by_id.get(sid) - if not s: - continue - try: - stamps[field_name] = base64.b64decode(s.data_png) - except Exception as e: - logger.warning(f"Bad signature data for {sid}: {e}") - - filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(filled_path) - try: - fill_fields(pdf_path, filled_path, text_values) - except Exception as e: - logger.error(f"fill_fields failed for doc {doc_id}: {e}") - _cleanup_temps() - raise HTTPException(500, f"PDF fill failed: {e}") - - out_path = filled_path - if stamps: - stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(stamped_path) - try: - stamp_signatures(filled_path, stamped_path, stamps) - out_path = stamped_path - except Exception as e: - logger.error(f"stamp_signatures failed for doc {doc_id}: {e}") - - # Burn freeform annotations (Text/Check/Sign drops) on top. - annotations = parse_markdown_annotations(doc.current_content or "") - if annotations: - # Resolve any signature annotations to their PNG bytes. - ann_sig_ids = [ - a["value"][len("signature:"):].strip() - for a in annotations - if a.get("kind") == "signature" - and isinstance(a.get("value"), str) - and a["value"].startswith("signature:") - ] - ann_signature_pngs: dict[str, bytes] = {} - if ann_sig_ids: - # SECURITY: filter by owner so a caller can't reference - # someone else's signature ID from doc markdown and have - # it stamped/exported. - _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) - if user: - _sig_q = _sig_q.filter(Signature.owner == user) - sig_rows = _sig_q.all() - for s in sig_rows: - try: - ann_signature_pngs[s.id] = base64.b64decode(s.data_png) - except Exception as e: - logger.warning(f"Bad annotation signature data for {s.id}: {e}") - annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(annotated_path) - try: - stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) - out_path = annotated_path - except Exception as e: - logger.error(f"stamp_annotations failed for doc {doc_id}: {e}") - - download_name = _slug(doc.title or "form") + "_annotated.pdf" - return FileResponse( - out_path, - media_type="application/pdf", - filename=download_name, - background=BackgroundTask(_cleanup_temps), - ) - finally: - db.close() - - # ---- POST /api/document/{doc_id}/prepare-signed-reply ---- - @router.post("/api/document/{doc_id}/prepare-signed-reply") - async def prepare_signed_reply(doc_id: str, request: Request): - """Bake the current PDF state (form fields + signature stamps + - annotations) into a flattened PDF, drop it in COMPOSE_UPLOADS_DIR - and return the reply context (To/Subject/threading headers) so the - frontend can open a reply draft with this attachment pre-loaded. - - Requires the document to have source_email_* metadata (set when the - doc was created via /api/email/attachment-as-doc). Otherwise 400. - """ - import base64 - import tempfile - import shutil - import uuid as _uuid - import email as _email_mod - from src.pdf_form_doc import ( - find_source_upload_id, parse_markdown_to_values, - load_field_sidecar, parse_markdown_annotations, - ) - from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations - from core.database import Signature - # COMPOSE_UPLOADS_DIR lives in email_routes — re-derive here so we - # don't import from a routes file (cycle-prone). Same env override - # as email_routes (ODYSSEUS_MAIL_ATTACHMENTS_DIR). - from pathlib import Path as _Path - _COMPOSE_DIR = _Path(MAIL_ATTACHMENTS_DIR) / "_compose" - _COMPOSE_DIR.mkdir(parents=True, exist_ok=True) - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - if not (doc.source_email_uid and doc.source_email_folder): - raise HTTPException(400, "Document has no source email — cannot reply") - - # 1) Build the flattened PDF (same pipeline as export_pdf) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found") - - schema = load_field_sidecar(pdf_path) or [] - sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} - all_values = parse_markdown_to_values(doc.current_content or "") - text_values: dict = {} - sig_ids: dict[str, str] = {} - for name, raw in all_values.items(): - if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): - sig_ids[name] = raw[len("signature:"):].strip() - elif name not in sig_field_names: - text_values[name] = raw - - stamps: dict = {} - if sig_ids: - # SECURITY: filter by owner — same reason as render_pdf. - _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) - if user: - _sig_q2 = _sig_q2.filter(Signature.owner == user) - rows = _sig_q2.all() - by_id = {s.id: s for s in rows} - for fname, sid in sig_ids.items(): - s = by_id.get(sid) - if not s: - continue - try: - stamps[fname] = base64.b64decode(s.data_png) - except Exception: - pass - - import os - _to_unlink: list[str] = [] - filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(filled_path) - fill_fields(pdf_path, filled_path, text_values) - out_path = filled_path - if stamps: - stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(stamped_path) - try: - stamp_signatures(filled_path, stamped_path, stamps) - out_path = stamped_path - except Exception as e: - logger.warning(f"stamp_signatures failed for {doc_id}: {e}") - - annotations = parse_markdown_annotations(doc.current_content or "") - if annotations: - ann_sig_ids = [ - a["value"][len("signature:"):].strip() - for a in annotations - if a.get("kind") == "signature" - and isinstance(a.get("value"), str) - and a["value"].startswith("signature:") - ] - ann_signature_pngs: dict[str, bytes] = {} - if ann_sig_ids: - # SECURITY: filter by owner so a caller can't reference - # someone else's signature ID from doc markdown and have - # it stamped/exported. - _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) - if user: - _sig_q = _sig_q.filter(Signature.owner == user) - sig_rows = _sig_q.all() - for s in sig_rows: - try: - ann_signature_pngs[s.id] = base64.b64decode(s.data_png) - except Exception: - pass - annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(annotated_path) - try: - stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) - out_path = annotated_path - except Exception as e: - logger.warning(f"stamp_annotations failed for {doc_id}: {e}") - - # 2) Move/copy into COMPOSE_UPLOADS_DIR with the token format - # `<uuid>_<original_name>` that /api/email/send expects. - filename = _slug(doc.title or "signed") + "_signed.pdf" - token = f"{_uuid.uuid4().hex}_{filename}" - dest = _COMPOSE_DIR / token - shutil.copyfile(out_path, str(dest)) - # Unlink the intermediate temp PDFs now that they've been - # copied into COMPOSE_UPLOADS_DIR. - for _p in _to_unlink: - try: - os.unlink(_p) - except FileNotFoundError: - pass - except Exception as _e: - logger.warning(f"Could not unlink temp PDF {_p}: {_e}") - - # 3) Fetch the source email's headers so we can build a clean reply - # context (To/Subject/In-Reply-To/References). - try: - from routes.email_routes import _imap, _decode_header - from routes.email_helpers import _q - except Exception: - _imap = None - _decode_header = lambda x: x or "" - _q = lambda x: x or "" - - to_addr = "" - from_name = "" - subject = "" - in_reply_to = doc.source_email_message_id or "" - references = in_reply_to - if _imap: - try: - with _imap(doc.source_email_account_id or None) as conn: - conn.select(_q(doc.source_email_folder), readonly=True) - status, data = conn.fetch(doc.source_email_uid.encode(), "(RFC822.HEADER)") - if status == "OK" and data and data[0]: - raw_hdr = data[0][1] - m = _email_mod.message_from_bytes(raw_hdr) - sender = _decode_header(m.get("From", "")) - from_name, to_addr = _email_mod.utils.parseaddr(sender) - if not to_addr: - to_addr = sender - subject = _decode_header(m.get("Subject", "") or "") - if subject and not subject.lower().startswith("re:"): - subject = "Re: " + subject - msg_refs = (m.get("References") or "").strip() - msg_in_reply = (m.get("Message-ID") or "").strip() or in_reply_to - in_reply_to = msg_in_reply - references = (msg_refs + " " + msg_in_reply).strip() if msg_refs else msg_in_reply - except Exception as e: - logger.warning(f"prepare-signed-reply header fetch failed: {e}") - - return { - "ok": True, - "attachment": { - "token": token, - "filename": filename, - "size": dest.stat().st_size, - }, - "reply": { - "to": to_addr, - "to_name": from_name, - "subject": subject, - "in_reply_to": in_reply_to, - "references": references, - "account_id": doc.source_email_account_id or None, - "source_uid": doc.source_email_uid, - "source_folder": doc.source_email_folder, - "source_message_id": doc.source_email_message_id, - }, - } - finally: - db.close() - - return router +_sys.modules[__name__] = _canonical diff --git a/tests/test_document_routes_shim.py b/tests/test_document_routes_shim.py new file mode 100644 index 000000000..68d049a62 --- /dev/null +++ b/tests/test_document_routes_shim.py @@ -0,0 +1,29 @@ +"""Regression test for the document route shim (slice 2m, #4082/#4071). + +The backward-compat shims at ``routes/document_routes.py`` and +``routes/document_helpers.py`` use ``sys.modules`` replacement so the legacy +import paths and the canonical ``routes.document.*`` paths resolve to the +*same* module objects. This is required because multiple tests do +``import routes.document_routes as droutes`` followed by +``droutes.SessionLocal = ...`` / ``monkeypatch.setattr(droutes, ...)`` and +``sys.modules.pop("routes.document_helpers")`` + re-import — for those to +take effect at runtime, the legacy and canonical module objects must be +identical. +""" + +import importlib + +import routes.document_routes as _shim_routes # noqa: F401 +import routes.document_helpers as _shim_helpers # noqa: F401 + + +def test_legacy_and_canonical_routes_are_same_object(): + legacy = importlib.import_module("routes.document_routes") + canonical = importlib.import_module("routes.document.document_routes") + assert legacy is canonical + + +def test_legacy_and_canonical_helpers_are_same_object(): + legacy = importlib.import_module("routes.document_helpers") + canonical = importlib.import_module("routes.document.document_helpers") + assert legacy is canonical diff --git a/tests/test_imap_mailbox_quoting.py b/tests/test_imap_mailbox_quoting.py index 7c5bb1645..636270a56 100644 --- a/tests/test_imap_mailbox_quoting.py +++ b/tests/test_imap_mailbox_quoting.py @@ -87,7 +87,7 @@ def test_known_imap_mailbox_call_sites_are_quoted(): assert "conn.select(sent_name" not in pollers assert "imap.append(sent_folder" not in pollers - document_routes = Path("routes/document_routes.py").read_text() + document_routes = Path("routes/document/document_routes.py").read_text() assert "conn.select(doc.source_email_folder" not in document_routes diff --git a/tests/test_model_helper_owner_scope.py b/tests/test_model_helper_owner_scope.py index dafbad594..f48a1f7e2 100644 --- a/tests/test_model_helper_owner_scope.py +++ b/tests/test_model_helper_owner_scope.py @@ -14,7 +14,7 @@ def _function_source(path: str, name: str) -> str: def test_document_ai_tidy_resolves_with_owner_scope(): - body = _function_source("routes/document_routes.py", "ai_tidy_documents") + body = _function_source("routes/document/document_routes.py", "ai_tidy_documents") assert "resolve_task_endpoint(owner=user or None)" in body assert 'resolve_endpoint("default", owner=user or None)' in body diff --git a/tests/test_vision_owner_scope.py b/tests/test_vision_owner_scope.py index f0d3a184d..29de101a3 100644 --- a/tests/test_vision_owner_scope.py +++ b/tests/test_vision_owner_scope.py @@ -88,7 +88,7 @@ def test_request_vision_call_sites_pass_owner(): chat_source = (ROOT / "src" / "chat_handler.py").read_text() processor_source = (ROOT / "src" / "document_processor.py").read_text() upload_source = (ROOT / "routes" / "upload_routes.py").read_text() - document_source = (ROOT / "routes" / "document_routes.py").read_text() + document_source = (ROOT / "routes" / "document" / "document_routes.py").read_text() gallery_source = (ROOT / "routes" / "gallery" / "gallery_routes.py").read_text() memory_source = (ROOT / "routes" / "memory" / "memory_routes.py").read_text() From 9d686180dd20e6ef842f0c23f9ef2f0ce39cee4f Mon Sep 17 00:00:00 2001 From: Ashvin <76151462+ashvinctrl@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:47:41 +0530 Subject: [PATCH 33/43] fix(integrations): pin api_call to the SSRF-validated IP (#5727) * fix(integrations): pin api_call to the SSRF-validated IP execute_api_call runs check_outbound_url on the target, but that guard only resolves the host to answer (ok, reason) and hands back no address. The request right after it opened a plain httpx.AsyncClient, which resolves the host again at connect time. A base_url host on a low TTL can pass the guard as a public IP and then flip to 169.254.169.254 for the connect, so the call lands on cloud metadata with the integration's stored auth headers attached. Resolve once, remember the IPs the guard actually validated, and pin the client's socket to that set through a small AnyIO-backed transport. SNI and the Host header still come from the URL, so TLS and vhost routing are unchanged; connect-time fallback stays inside the approved address set over one shared deadline. This is the same pinning the webhook sender and web-fetch paths already do -- api_call was the last outbound path that skipped it. Fixes #5513 * fix(integrations): de-duplicate the pinned IP list _default_resolver calls getaddrinfo(host, None) with no socktype filter, so glibc returns one record per socktype and a single-homed host comes back three times over. _validated_ips kept every entry, so the transport pinned the same address repeatedly and the connect fallback could spend its shared deadline retrying one dead address instead of moving on to a genuinely different one. Windows getaddrinfo collapses those duplicate records, which is why the ip-literal pin test only failed on CI and not locally. --- src/integrations.py | 175 ++++++++++++- tests/test_integration_api_call_ssrf.py | 240 ++++++++++++++++++ .../test_integrations_api_call_truncation.py | 14 +- 3 files changed, 420 insertions(+), 9 deletions(-) diff --git a/src/integrations.py b/src/integrations.py index aa6c4982e..52dd4b2d1 100644 --- a/src/integrations.py +++ b/src/integrations.py @@ -1,11 +1,14 @@ +import ipaddress import json import os +import time import uuid import logging import re from typing import Dict, List, Optional, Any from urllib.parse import urljoin, urlparse, urlunparse +import httpcore import httpx from fastapi import HTTPException @@ -354,6 +357,152 @@ def _find_integration(identifier: str) -> Optional[Dict[str, Any]]: return None +# httpcore raises its own exception hierarchy; map the ones a simple request can +# surface back to their httpx equivalents so the caller's `except httpx.*` blocks +# below behave exactly as they did with the default transport. +_HTTPCORE_TO_HTTPX_EXC = { + httpcore.ConnectError: httpx.ConnectError, + httpcore.ConnectTimeout: httpx.ConnectTimeout, + httpcore.NetworkError: httpx.NetworkError, + httpcore.PoolTimeout: httpx.PoolTimeout, + httpcore.ProtocolError: httpx.ProtocolError, + httpcore.ReadError: httpx.ReadError, + httpcore.ReadTimeout: httpx.ReadTimeout, + httpcore.RemoteProtocolError: httpx.RemoteProtocolError, + httpcore.TimeoutException: httpx.TimeoutException, + httpcore.WriteError: httpx.WriteError, + httpcore.WriteTimeout: httpx.WriteTimeout, +} + + +class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend): + """Network backend that connects only to the pre-validated IPs, in order. + + Every address here came out of the single SSRF resolution, so moving to the + next one after a connect failure is not re-resolution — it's ordinary + multi-address fallback restricted to the set the guard already approved. + httpcore takes TLS SNI and the ``Host`` header from the request URL rather + than the connect host, so pinning the socket destination leaves certificate + validation and vhost routing pointed at the original hostname. + """ + + def __init__(self, ips: List[ipaddress._BaseAddress]): + self._ips = [str(ip) for ip in ips] + self._real = httpcore.AnyIOBackend() + + async def connect_tcp(self, host, port, timeout=None, local_address=None, + socket_options=None): + # One shared connect budget: each attempt gets the time left until the + # original deadline, so N dead addresses can't stretch the connect + # phase to N * timeout. + deadline = None if timeout is None else time.monotonic() + timeout + last_exc: Optional[Exception] = None + for ip in self._ips: + remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) + try: + return await self._real.connect_tcp( + ip, port, remaining, local_address, socket_options + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_exc = exc + if deadline is not None and time.monotonic() >= deadline: + break + raise last_exc + + async def connect_unix_socket(self, path, timeout=None, socket_options=None): + return await self._real.connect_unix_socket(path, timeout, socket_options) + + async def sleep(self, seconds: float) -> None: + return await self._real.sleep(seconds) + + +class _PinnedAsyncTransport(httpx.AsyncBaseTransport): + """httpx transport that pins the TCP connect to the pre-resolved IP(s). + + Kept local, mirroring the per-module pinned transports web fetch and + webhook delivery already carry, rather than coupling api_call to the + webhook subsystem. The request URL passes through unchanged, so SNI and the + ``Host`` header stay the original hostname; only the socket destination is + pinned, which is what closes the rebinding window. + """ + + def __init__(self, ips: List[ipaddress._BaseAddress]): + self._pinned_ips = list(ips) + self._pool = httpcore.AsyncConnectionPool( + # Reuse the CA trust the default httpx client would build (certifi + # plus SSL_CERT_FILE / SSL_CERT_DIR when trust_env is set) so + # swapping in this transport doesn't quietly change which chains + # verify. ssl.create_default_context() would use system roots. + ssl_context=httpx.create_ssl_context(), + http1=True, + http2=False, + network_backend=_PinnedAsyncBackend(ips), + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + core_req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + try: + core_resp = await self._pool.handle_async_request(core_req) + content = b"".join([chunk async for chunk in core_resp.aiter_stream()]) + await core_resp.aclose() + except Exception as exc: + mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc)) + if mapped is not None: + raise mapped(str(exc)) from exc + raise + return httpx.Response( + status_code=core_resp.status, + headers=core_resp.headers, + content=content, + extensions=core_resp.extensions, + ) + + async def aclose(self) -> None: + await self._pool.aclose() + + +def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]: + """Return every entry that parses as an IP address, de-duplicated, order + preserved. + + check_outbound_url only reports ok when *all* of these classify as safe, so + the whole list is guard-approved and any of them is a legitimate connect + target. Skipping unparseable entries mirrors how the guard walks the same + resolver output. + + De-duplication matters because the resolver is getaddrinfo(host, None) with + no socktype filter, so glibc reports the same address once per socktype + (SOCK_STREAM/SOCK_DGRAM/SOCK_RAW) — a single-homed host comes back three + times. Without this, the connect fallback would spend the shared deadline + retrying one dead address instead of moving on to a genuinely different one. + """ + ips: List[ipaddress._BaseAddress] = [] + seen = set() + for raw in raw_ips: + if not isinstance(raw, str): + continue + try: + ip = ipaddress.ip_address(raw.split("%")[0]) # strip IPv6 zone id + except ValueError: + continue + if ip in seen: + continue + seen.add(ip) + ips.append(ip) + return ips + + async def execute_api_call( integration_id: str, method: str, @@ -409,13 +558,31 @@ async def execute_api_call( # loopback for locked-down deployments. Private stays allowed by default # because LAN integrations (Home Assistant, Miniflux, ntfy) are the # primary use case. - from src.url_safety import check_outbound_url + from src.url_safety import check_outbound_url, _default_resolver block_private = os.getenv( "INTEGRATION_API_BLOCK_PRIVATE_IPS", "false" ).lower() == "true" - ok, reason = check_outbound_url(url, block_private=block_private) + # Resolve the host exactly once and remember the IPs the guard validated so + # the request below can be pinned to them. check_outbound_url only reports + # (ok, reason); a plain httpx client re-resolves the host at connect time, + # which reopens a DNS-rebinding TOCTOU — a base_url host that answers with a + # public IP for the guard and then flips to 169.254.169.254 for the connect + # would reach cloud metadata with the integration's auth headers attached. + resolved_ips: List[str] = [] + + def _recording_resolver(host: str) -> List[str]: + ips = _default_resolver(host) + resolved_ips[:] = ips + return ips + + ok, reason = check_outbound_url( + url, block_private=block_private, resolver=_recording_resolver + ) if not ok: return {"error": f"URL rejected: {reason}", "exit_code": 1} + pinned_ips = _validated_ips(resolved_ips) + if not pinned_ips: + return {"error": "URL rejected: host did not resolve to a usable address", "exit_code": 1} method = method.upper() @@ -455,7 +622,9 @@ async def execute_api_call( auth = httpx.BasicAuth(parts[0], parts[1]) try: - async with httpx.AsyncClient(timeout=30.0) as client: + async with httpx.AsyncClient( + timeout=30.0, transport=_PinnedAsyncTransport(pinned_ips) + ) as client: response = await client.request( method, url, diff --git a/tests/test_integration_api_call_ssrf.py b/tests/test_integration_api_call_ssrf.py index 53dc671c5..f23cc40de 100644 --- a/tests/test_integration_api_call_ssrf.py +++ b/tests/test_integration_api_call_ssrf.py @@ -9,8 +9,13 @@ link-local/metadata is always rejected; RFC-1918/loopback only when INTEGRATION_API_BLOCK_PRIVATE_IPS=true (LAN integrations are the primary use case, so private stays allowed by default). """ +import asyncio +import ipaddress +import ssl from unittest.mock import AsyncMock, MagicMock, patch +import httpcore +import httpx import pytest from src import integrations @@ -97,3 +102,238 @@ async def test_private_base_url_allowed_by_default_blocked_with_knob(monkeypatch assert result["exit_code"] == 1 assert "rejected" in result["error"].lower() client.request.assert_not_called() + + +async def _call_capturing_transport(base_url, path="/items"): + """Drive execute_api_call and return (result, transport) where transport is + the object passed to httpx.AsyncClient(transport=...).""" + resp = MagicMock() + resp.status_code = 200 + resp.headers = {"content-type": "application/json"} + resp.json.return_value = {"ok": True} + resp.text = '{"ok": true}' + + client = AsyncMock() + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=None) + client.request = AsyncMock(return_value=resp) + + captured = {} + + def _fake_async_client(*args, **kwargs): + captured.update(kwargs) + return client + + with ( + patch.object(integrations, "_find_integration", + return_value=_integration(base_url)), + patch("httpx.AsyncClient", side_effect=_fake_async_client), + ): + result = await integrations.execute_api_call("test_integ", "GET", path) + return result, captured.get("transport"), client + + +@pytest.mark.asyncio +async def test_connection_is_pinned_to_the_validated_ip(monkeypatch): + """DNS-rebinding defense: the guard resolves the host once to a benign + public IP, and the request must be pinned to *that* IP so a host that + rebinds to the metadata range at connect time can't be reached with the + integration's auth headers. Static resolution passing the guard is not + enough — a plain client would re-resolve at connect.""" + monkeypatch.setattr("src.url_safety._default_resolver", + lambda host: ["93.184.216.34"]) + result, transport, client = await _call_capturing_transport( + "http://rebinding.attacker.example") + + assert result.get("exit_code") == 0 + client.request.assert_called_once() + assert isinstance(transport, integrations._PinnedAsyncTransport) + assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"] + + +@pytest.mark.asyncio +async def test_pin_carries_the_whole_validated_ip_set(monkeypatch): + """When a host resolves to several records the transport keeps all of them + (check_outbound_url validated every one), in resolver order, so it can fall + back past a dead first address instead of failing the whole call.""" + monkeypatch.setattr("src.url_safety._default_resolver", + lambda host: ["93.184.216.34", "198.51.100.7"]) + result, transport, _ = await _call_capturing_transport("http://multi.example") + + assert result.get("exit_code") == 0 + assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34", "198.51.100.7"] + + +class _FakeStream: + """Stand-in for the connected socket the real backend returns.""" + + +class _RecordingBackend: + """Fake httpcore backend: connect_tcp fails for the addresses in `dead` + and succeeds for the rest, recording the order it was asked to connect.""" + + def __init__(self, dead): + self.dead = set(dead) + self.attempts = [] + + async def connect_tcp(self, host, port, timeout=None, local_address=None, + socket_options=None): + self.attempts.append((host, timeout)) + if host in self.dead: + raise httpcore.ConnectError(f"connection refused: {host}") + return _FakeStream() + + +def _pinned_backend(ips, dead): + """A _PinnedAsyncBackend whose underlying connect is the recording fake.""" + backend = integrations._PinnedAsyncBackend(ips) + backend._real = _RecordingBackend(dead) + return backend + + +@pytest.mark.asyncio +async def test_connect_falls_back_from_dead_first_to_live_second(): + """first-dead / second-live: the pinned backend must try the next validated + address when the first refuses, rather than surfacing the failure. It also + ignores the `host` httpcore passes (the original hostname) and connects to + the pinned IPs, which is what keeps TLS SNI / Host on the real hostname.""" + ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")] + backend = _pinned_backend(ips, dead={"203.0.113.10"}) + + stream = await backend.connect_tcp("original.hostname.example", 443, timeout=5.0) + + assert isinstance(stream, _FakeStream) + # Tried the dead address first, then the live one — never the hostname. + assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"] + # Fallback shared one budget: the second attempt got the time left, not a fresh 5s. + assert backend._real.attempts[1][1] <= 5.0 + + +@pytest.mark.asyncio +async def test_connect_raises_when_every_validated_address_is_dead(): + ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")] + backend = _pinned_backend(ips, dead={"203.0.113.10", "198.51.100.7"}) + + with pytest.raises(httpcore.ConnectError): + await backend.connect_tcp("original.hostname.example", 443, timeout=5.0) + assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"] + + +@pytest.mark.asyncio +async def test_pinned_transport_reuses_httpx_ca_trust(monkeypatch): + """TLS trust must come from the same builder the default httpx client uses + (certifi + SSL_CERT_FILE / SSL_CERT_DIR via trust_env), not from + ssl.create_default_context()'s system roots — otherwise chains that verified + under the old default client can silently stop verifying.""" + sentinel = ssl.create_default_context() + calls = [] + + def _fake_create(*args, **kwargs): + calls.append(kwargs) + return sentinel + + monkeypatch.setattr(httpx, "create_ssl_context", _fake_create) + transport = integrations._PinnedAsyncTransport([ipaddress.ip_address("93.184.216.34")]) + try: + assert calls, "transport did not build its context via httpx.create_ssl_context" + assert transport._pool._ssl_context is sentinel + finally: + await transport.aclose() + + +@pytest.mark.asyncio +async def test_real_socket_falls_back_from_dead_first_to_live_second(): + """End-to-end over real loopback sockets: pin [127.0.0.2 (nothing + listening), 127.0.0.1 (live)], and the request must succeed by falling back + to the second address while the Host header stays the original hostname — + i.e. only the socket destination moved, vhost/SNI routing did not.""" + captured = {} + + async def handle(reader, writer): + request = await reader.read(4096) + for line in request.split(b"\r\n"): + if line.lower().startswith(b"host:"): + captured["host"] = line.split(b":", 1)[1].strip().decode() + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nhi") + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + async with server: + await server.start_serving() + transport = integrations._PinnedAsyncTransport( + [ipaddress.ip_address("127.0.0.2"), ipaddress.ip_address("127.0.0.1")] + ) + try: + async with httpx.AsyncClient(transport=transport) as client: + resp = await client.get(f"http://pinned.example:{port}/health") + finally: + await transport.aclose() + + assert resp.status_code == 200 + assert resp.text == "hi" + assert captured.get("host") == f"pinned.example:{port}" + + +@pytest.mark.asyncio +async def test_ip_literal_base_url_still_pins_and_is_not_rejected(): + """A base_url that is already an IP has nothing to rebind, but it must not + trip the "did not resolve" guard either. + + check_outbound_url resolves even a literal (getaddrinfo returns the address + itself), so the captured list is populated and the pin is a no-op rather + than a rejection. Uses the real resolver on purpose — no monkeypatch — so + this would catch the fail-closed branch firing on a literal. + """ + result, transport, client = await _call_capturing_transport( + "http://93.184.216.34") + + assert result.get("exit_code") == 0 + assert isinstance(transport, integrations._PinnedAsyncTransport) + assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"] + + +@pytest.mark.asyncio +async def test_ipv6_base_url_pins_every_validated_address(monkeypatch): + """IPv6 goes down the same path as v4. + + Resolution is stubbed rather than using a literal so this doesn't depend on + the runner having IPv6 configured. + """ + v6 = "2606:2800:220:1:248:1893:25c8:1946" + monkeypatch.setattr("src.url_safety._default_resolver", lambda host: [v6]) + result, transport, client = await _call_capturing_transport("http://v6.example") + + assert result.get("exit_code") == 0 + assert isinstance(transport, integrations._PinnedAsyncTransport) + assert [str(ip) for ip in transport._pinned_ips] == [v6] + + +def test_validated_ips_strips_zone_id_and_drops_junk(): + """getaddrinfo can hand back a scoped v6 address like 'fe80::1%eth0'.""" + got = integrations._validated_ips( + ["93.184.216.34", "fe80::1%eth0", "not-an-ip", None, "2001:db8::5"] + ) + assert [str(ip) for ip in got] == ["93.184.216.34", "fe80::1", "2001:db8::5"] + + +def test_validated_ips_deduplicates_repeated_addresses(): + """The resolver is getaddrinfo(host, None) with no socktype filter, so glibc + returns one record per socktype and a single-homed host arrives three times + over. Duplicates must collapse (first-seen order kept) or the connect + fallback wastes its shared deadline retrying one dead address.""" + got = integrations._validated_ips( + ["93.184.216.34", "93.184.216.34", "93.184.216.34"] + ) + assert [str(ip) for ip in got] == ["93.184.216.34"] + + # Order is first-seen, and distinct addresses all survive. + got = integrations._validated_ips( + ["198.51.100.7", "93.184.216.34", "198.51.100.7", "2001:db8::5"] + ) + assert [str(ip) for ip in got] == ["198.51.100.7", "93.184.216.34", "2001:db8::5"] + + # A zone-id variant is the same address once stripped, so it collapses too. + got = integrations._validated_ips(["fe80::1%eth0", "fe80::1%eth1", "fe80::1"]) + assert [str(ip) for ip in got] == ["fe80::1"] diff --git a/tests/test_integrations_api_call_truncation.py b/tests/test_integrations_api_call_truncation.py index bf1ec7d05..a0ad61b4a 100644 --- a/tests/test_integrations_api_call_truncation.py +++ b/tests/test_integrations_api_call_truncation.py @@ -83,9 +83,10 @@ async def _call(json_data, status=200): with ( patch.object(integrations, "_find_integration", return_value=DUMMY_INTEGRATION), patch("httpx.AsyncClient", return_value=mock_client), - # api.example.com doesn't resolve; the SSRF guard would fail closed. - # These tests are about truncation, so stub the guard open. - patch("src.url_safety.check_outbound_url", return_value=(True, "ok")), + # api.example.com doesn't resolve. Point the resolver at a public + # address instead of stubbing the guard open, so the real check (and + # the connect-IP pinning that reads its result) still runs. + patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]), ): return await integrations.execute_api_call("test_integ", "GET", "/items") @@ -101,9 +102,10 @@ async def _call_with_integration(integration, path="/items"): with ( patch.object(integrations, "_find_integration", return_value=integration), patch("httpx.AsyncClient", return_value=mock_client), - # api.example.com doesn't resolve; the SSRF guard would fail closed. - # These tests are about URL joining, so stub the guard open. - patch("src.url_safety.check_outbound_url", return_value=(True, "ok")), + # api.example.com doesn't resolve. Point the resolver at a public + # address instead of stubbing the guard open, so the real check (and + # the connect-IP pinning that reads its result) still runs. + patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]), ): result = await integrations.execute_api_call("test_integ", "GET", path) return result, mock_client From 20e7fc0164286e1521569d9edc17a4ae4d0d2e22 Mon Sep 17 00:00:00 2001 From: adabarbulescu <94562950+adabarbulescu@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:17:45 +0300 Subject: [PATCH 34/43] fix(skills): require manage_skills action (#5856) --- src/tools/system.py | 6 ++++-- tests/test_manage_skills_action_required.py | 24 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tests/test_manage_skills_action_required.py diff --git a/src/tools/system.py b/src/tools/system.py index 813d57df2..c2eb9ceab 100644 --- a/src/tools/system.py +++ b/src/tools/system.py @@ -46,7 +46,9 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict: except ValueError: return {"error": "Invalid JSON arguments", "exit_code": 1} - action = (args.get("action") or "").lower() + action = (args.get("action") or "").strip().lower() + if not action: + return {"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)", "exit_code": 1} from services.memory.skills import SkillsManager from services.memory.skill_format import Skill, slugify from src.constants import DATA_DIR @@ -55,7 +57,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict: # Accept legacy `skill_id` as an alias for `name`. name = (args.get("name") or args.get("skill_id") or "").strip() - if action in ("list", "index", ""): + if action in ("list", "index"): all_skills = sm.load(owner=owner) if not all_skills: return {"results": "No skills yet. Create one with action='add'."} diff --git a/tests/test_manage_skills_action_required.py b/tests/test_manage_skills_action_required.py new file mode 100644 index 000000000..4efae8026 --- /dev/null +++ b/tests/test_manage_skills_action_required.py @@ -0,0 +1,24 @@ +import json + +import pytest + +from src.tools.system import do_manage_skills + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload", + [ + {}, + {"action": ""}, + {"action": " "}, + {"name": "demo", "description": "x", "procedure": ["step"]}, + ], +) +async def test_manage_skills_requires_action(payload): + result = await do_manage_skills(json.dumps(payload), owner="test") + + assert result == { + "error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)", + "exit_code": 1, + } From c8a012d4d2db27142196a9a7b290687323979b5b Mon Sep 17 00:00:00 2001 From: Ashvin <76151462+ashvinctrl@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:03:50 +0530 Subject: [PATCH 35/43] fix(memory): don't let an unreadable store get overwritten with an empty one (#5831) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(memory): don't let an unreadable store get overwritten with an empty one load_all() answered a failed read the same way it answered an empty store: with []. Every mutation path is a read-modify-write (load the whole file, change it, save it back), so a failed read became load_all() -> [] -> [].append(new) -> save([new]) and save() is atomic, so the replacement stuck. The case that actually destroys data is a store that is READABLE but not parseable - a truncated file, or one holding {} instead of []. Nothing obstructs the write, so adding a memory returns HTTP 200 and every memory already stored is gone. Verified end-to-end against a running instance: on the current code a truncated memory.json plus one add leaves the file holding only the new entry. Truncation is reachable - core/database.py rewrites memory.json during migration with a plain open(.., "w") + json.dump, which is not atomic. A live exclusive lock is not the dangerous case: it blocks the read and the os.replace alike, so the save fails too and the store survives. That path currently 500s and loses nothing. _read_entries() now returns [] only when the file genuinely does not exist and raises MemoryStoreUnreadable for every other failure, including a store that parses but is not a JSON array. load_all() keeps the old lenient behaviour so display, search and context injection still degrade quietly instead of breaking chat. The read-modify-write callers switch to load_all_for_update(), which propagates the error: the memory routes turn it into a 503 and change nothing, backup import refuses rather than saving only the incoming rows, and auto-extraction and the audit merge skip the write. The audit merge mattered most - it rebuilds the whole file from one owner's slice plus everyone else's rows, so an empty read there dropped every other tenant's memories. The corrupt-JSON path still gets its one shot at the legacy memory.txt migration before raising, so that recovery is unchanged. The two updated fakes gained load_all_for_update because the real class has it; MagicMock would otherwise hand the import path a Mock instead of the seeded list. Fixes #5673 * fix(memory): fail closed on the remaining read-modify-write add paths The strict loader landed with the routes, the backup import and the extractor converted, but three read-modify-write sinks still called load_all(), which degrades an unreadable store to []. Two of them are the paths users actually reach, so the data loss in #5673 stayed reproducible: - src/ai_interaction.py do_manage_memory, action "add" — reached from ordinary chat via src/tool_execution.py:793 -> dispatch_ai_tool. "Remember that I prefer X" against an unreadable store wrote a one-entry file over it and reported success. - mcp_servers/memory_server.py, action "add" — the same shape through _scope_entries(), registered as a built-in in src/builtin_mcp.py. - src/memory_provider.py NativeMemoryProvider.remember and .delete — wired into app state in src/app_initializer.py but not consumed outside tests yet, converted here so the pattern is uniform before it goes live. The MCP server takes _scope_entries(for_update=True) so list keeps the lenient read. The edit and delete branches on both tool paths were already fail-closed by accident — an empty view matches nothing and returns before the save — so they are left alone. The three new tests drive the real entry points rather than replaying the shape, and use a truncated store, which is the case that reads back fine so nothing stops the save. Each asserts memory.json is byte-identical afterwards; all three fail on the previous commit with the store overwritten. --- mcp_servers/memory_server.py | 26 +- routes/backup_routes.py | 11 +- routes/memory/memory_routes.py | 26 +- services/memory/__init__.py | 3 +- services/memory/memory.py | 14 +- services/memory/memory_extractor.py | 23 +- src/ai_interaction.py | 11 +- src/memory.py | 90 ++++++- src/memory_provider.py | 11 +- tests/test_backup_import_cross_user_dedup.py | 3 + ...st_memory_extractor_vector_cross_tenant.py | 6 + tests/test_memory_store_unreadable_no_wipe.py | 255 ++++++++++++++++++ 12 files changed, 451 insertions(+), 28 deletions(-) create mode 100644 tests/test_memory_store_unreadable_no_wipe.py diff --git a/mcp_servers/memory_server.py b/mcp_servers/memory_server.py index fafbcfc2b..fd574fd1f 100644 --- a/mcp_servers/memory_server.py +++ b/mcp_servers/memory_server.py @@ -17,6 +17,8 @@ from mcp.types import Tool, TextContent sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from src.memory import MemoryStoreUnreadable + server = Server("memory") # Late-initialized managers (set during first tool call) @@ -29,6 +31,10 @@ _OWNER_SCOPE_ERROR = ( "Error: Memory MCP owner is not configured for an owner-scoped memory store. " "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool." ) +_UNREADABLE_STORE_ERROR = ( + "Error: Memory store is temporarily unreadable — nothing was saved. " + "Repair or restore memory.json, then retry." +) def _configured_owner() -> str | None: @@ -51,9 +57,21 @@ def _owner_scoped_store(entries: list[dict]) -> bool: return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict)) -def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]: - """Return configured owner, all entries, visible entries, and optional error.""" - entries = _memory_manager.load_all() +def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]: + """Return configured owner, all entries, visible entries, and optional error. + + ``for_update=True`` is for read-modify-write callers. They save the ``all + entries`` list back, so an unreadable store must be reported as an error + instead of degrading to ``[]`` — otherwise the save writes their one new + entry over the whole store (issue #5673). + """ + if for_update: + try: + entries = _memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})" + else: + entries = _memory_manager.load_all() owner = _configured_owner() if owner is None and _owner_scoped_store(entries): return None, entries, [], _OWNER_SCOPE_ERROR @@ -161,7 +179,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: category = arguments.get("category", "fact") if not text: return _text_result("Error: Memory text cannot be empty") - owner, memories, _visible, scope_error = _scope_entries() + owner, memories, _visible, scope_error = _scope_entries(for_update=True) if scope_error: return _text_result(scope_error) entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner) diff --git a/routes/backup_routes.py b/routes/backup_routes.py index 313369370..4ecf4f165 100644 --- a/routes/backup_routes.py +++ b/routes/backup_routes.py @@ -6,6 +6,7 @@ from datetime import datetime from fastapi import APIRouter, HTTPException, Request, Response from core.middleware import require_admin +from services.memory import MemoryStoreUnreadable from src.auth_helpers import get_current_user from src.settings import load_settings, save_settings, load_features, save_features @@ -76,7 +77,15 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo # ── Memories ── if "memories" in body and isinstance(body["memories"], list): - existing = memory_manager.load_all() + # Strict load: importing on top of an unreadable store would write + # only the incoming rows back and drop everything already saved. + try: + existing = memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Refusing to import memories: %s", e) + raise HTTPException( + 503, "Memory store is temporarily unreadable — nothing was imported." + ) # Dedup against THIS user's own memories only. Using every tenant's # rows (load_all) meant a memory whose text matched any other # user's was silently skipped, so the importing user lost their own diff --git a/routes/memory/memory_routes.py b/routes/memory/memory_routes.py index d290046ec..c4232bec4 100644 --- a/routes/memory/memory_routes.py +++ b/routes/memory/memory_routes.py @@ -21,7 +21,7 @@ def _strip_list_prefix(text: str) -> str: return text return _LIST_PREFIX_RE.sub("", text, count=1).strip() -from services.memory import MemoryManager +from services.memory import MemoryManager, MemoryStoreUnreadable from core.session_manager import SessionManager from src.request_models import MemoryAddRequest from core.database import SessionLocal @@ -35,6 +35,22 @@ from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES logger = logging.getLogger(__name__) +def _load_for_update(memory_manager) -> List[Dict[str, Any]]: + """Load the whole store for a read-modify-write cycle. + + A transient read failure must not look like an empty store: the caller + would append to ``[]`` and save that back, atomically destroying every + existing memory (issue #5673). Surface it as a 503 and change nothing. + """ + try: + return memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Refusing to rewrite the memory store: %s", e) + raise HTTPException( + 503, "Memory store is temporarily unreadable — no changes were made." + ) + + def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None): """Set up memory-related routes.""" router = APIRouter(prefix="/api/memory", tags=["memory"]) @@ -116,7 +132,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user) if memory_data.session_id: new_entry["session_id"] = memory_data.session_id - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) all_mem.append(new_entry) memory_manager.save(all_mem) # Sync vector index @@ -487,7 +503,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)): """Pin or unpin a memory. Pinned memories are always included in context.""" user = _owner(request) - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) for i, memory in enumerate(all_mem): if memory["id"] == memory_id: _verify_memory_owner(memory, user) @@ -512,7 +528,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)): """Update an existing memory item with new text and optional category.""" user = _owner(request) - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) for i, memory in enumerate(all_mem): if memory["id"] == memory_id: _verify_memory_owner(memory, user) @@ -534,7 +550,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM def delete_memory(request: Request, memory_id: str): """Delete a memory item by its ID.""" user = _owner(request) - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) # Find and verify ownership before deleting target = next((m for m in all_mem if m["id"] == memory_id), None) diff --git a/services/memory/__init__.py b/services/memory/__init__.py index 53fc80bd8..31fa1d5fa 100644 --- a/services/memory/__init__.py +++ b/services/memory/__init__.py @@ -2,7 +2,7 @@ """Memory service — persistent memory storage and retrieval.""" from .service import MemoryService, Memory, MemorySearchResult -from .memory import MemoryManager +from .memory import MemoryManager, MemoryStoreUnreadable from .memory_vector import MemoryVectorStore __all__ = [ @@ -10,5 +10,6 @@ __all__ = [ "Memory", "MemorySearchResult", "MemoryManager", + "MemoryStoreUnreadable", "MemoryVectorStore", ] diff --git a/services/memory/memory.py b/services/memory/memory.py index 031c13ac4..b9aaaa2a8 100644 --- a/services/memory/memory.py +++ b/services/memory/memory.py @@ -5,6 +5,16 @@ application runtime instantiates ``src.memory.MemoryManager``, so keeping a parallel implementation here risks silent drift between import paths. """ -from src.memory import MemoryManager, get_text_similarity, tokenize +from src.memory import ( + MemoryManager, + MemoryStoreUnreadable, + get_text_similarity, + tokenize, +) -__all__ = ["MemoryManager", "get_text_similarity", "tokenize"] +__all__ = [ + "MemoryManager", + "MemoryStoreUnreadable", + "get_text_similarity", + "tokenize", +] diff --git a/services/memory/memory_extractor.py b/services/memory/memory_extractor.py index e5f609250..11539263b 100644 --- a/services/memory/memory_extractor.py +++ b/services/memory/memory_extractor.py @@ -17,6 +17,8 @@ import os import re from typing import Optional +from src.memory import MemoryStoreUnreadable + logger = logging.getLogger(__name__) @@ -387,7 +389,13 @@ async def extract_and_store( # Get owner from session _owner = getattr(session, 'owner', None) - existing = memory_manager.load_all() + # Strict load: this is a read-modify-write. Degrading to [] here would + # save only the newly extracted facts and drop the entire store. + try: + existing = memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Skipping auto memory extraction, store unreadable: %s", e) + return added = 0 for fact in facts: @@ -626,7 +634,18 @@ async def audit_memories( # Merge audited entries back with other users' entries if owner: - all_entries = memory_manager.load_all() + # Strict load: the merge below reconstructs the whole file. If this + # degraded to [] we would save only this owner's audited slice and + # destroy every other tenant's memories. + try: + all_entries = memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Aborting memory audit save, store unreadable: %s", e) + return { + "before": before_count, + "after": before_count, + "error": "store_unreadable", + } audited_ids = {e["id"] for e in final_entries} other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)] # Also keep legacy entries that weren't part of this audit diff --git a/src/ai_interaction.py b/src/ai_interaction.py index 9ee97368f..e777ca32a 100644 --- a/src/ai_interaction.py +++ b/src/ai_interaction.py @@ -22,6 +22,7 @@ import time from typing import Any, Awaitable, Callable, Dict, Optional, Tuple from src.constants import GENERATED_IMAGES_DIR +from src.memory import MemoryStoreUnreadable logger = logging.getLogger(__name__) @@ -384,7 +385,15 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner return {"error": "Memory text cannot be empty"} entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner) - memories = _memory_manager.load_all() + # Strict load: this is a read-modify-write, and it is the path an + # ordinary "remember that I prefer X" takes. Degrading to [] here would + # save just this one entry over a store we only failed to read, + # atomically destroying every memory in it (issue #5673). + try: + memories = _memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Refusing to add memory, store unreadable: %s", e) + return {"error": "Memory store is temporarily unreadable — nothing was saved."} memories.append(entry) _memory_manager.save(memories) diff --git a/src/memory.py b/src/memory.py index 1d8cdbc1e..92efbf5b2 100644 --- a/src/memory.py +++ b/src/memory.py @@ -10,6 +10,18 @@ from datetime import datetime logger = logging.getLogger(__name__) + +class MemoryStoreUnreadable(RuntimeError): + """memory.json exists on disk but could not be read or parsed. + + "The contents are unknown" is categorically different from "there are no + memories". A read-modify-write caller that conflates the two appends to an + empty view and then persists it, destroying the whole store — the writes + are atomic, so the loss is durable. Raised by + :meth:`MemoryManager.load_all_for_update` so those callers fail closed. + """ + + def tokenize(text: str) -> List[str]: """Simple tokenizer that splits on whitespace and removes punctuation.""" return [word.strip('.,!?";') for word in text.split()] @@ -110,21 +122,69 @@ class MemoryManager: with open(self.memory_file, 'w', encoding='utf-8') as f: json.dump([], f, ensure_ascii=False, indent=2) - def load_all(self) -> List[Dict]: - """Load all memory entries from JSON file (unfiltered).""" + def _read_entries(self) -> List[Dict]: + """Parse the store, or raise :class:`MemoryStoreUnreadable`. + + Returns ``[]`` only when the file genuinely does not exist. Every other + failure mode raises, so callers can tell "no memories" apart from + "couldn't read the memories". + """ if not os.path.exists(self.memory_file): return [] try: with open(self.memory_file, "r", encoding="utf-8") as f: data = json.load(f) - if isinstance(data, list): - return self._validate_entries(data) - except (json.JSONDecodeError, PermissionError) as e: - logger.error("Error loading memory.json: %s", e) - return self._migrate_from_legacy() + except OSError as e: + # PermissionError is an OSError (a scanner holding the file, a + # permissions problem, bad media). + raise MemoryStoreUnreadable( + f"cannot read {self.memory_file}: {e}" + ) from e + except json.JSONDecodeError as e: + # This is the branch that actually destroyed stores: the file reads + # back fine, so nothing stops the save that follows. A truncated + # memory.json is reachable because core/database.py rewrites it with + # a plain open(..,"w") + json.dump during migration. + # + # Preserved behaviour: a corrupt store still gets one shot at the + # pre-JSON memory.txt migration. Only raise when that finds nothing, + # so we never report "empty" for a store we simply failed to parse. + legacy = self._migrate_from_legacy() + if legacy: + return legacy + raise MemoryStoreUnreadable( + f"{self.memory_file} is not valid JSON: {e}" + ) from e - return [] + if not isinstance(data, list): + raise MemoryStoreUnreadable( + f"{self.memory_file} is not a JSON array (got {type(data).__name__})" + ) + return self._validate_entries(data) + + def load_all(self) -> List[Dict]: + """Load all memory entries from JSON file (unfiltered). + + Lenient by design: this feeds display, search, and context-injection + paths, so an unreadable store degrades to an empty list rather than + breaking chat. Never build a value from this that you intend to save + back — use :meth:`load_all_for_update` for that. + """ + try: + return self._read_entries() + except MemoryStoreUnreadable as e: + logger.error("Error loading memory.json: %s", e) + return [] + + def load_all_for_update(self) -> List[Dict]: + """Load for a read-modify-write cycle. + + Propagates :class:`MemoryStoreUnreadable` instead of degrading to ``[]`` + so a caller can never append to an empty view and persist it over a + store that was only temporarily unreadable (issue #5673). + """ + return self._read_entries() def load(self, owner: str = None) -> List[Dict]: """Load memory entries, optionally filtered by owner.""" @@ -135,7 +195,12 @@ class MemoryManager: def claim_ownerless(self, owner: str): """Assign all ownerless memory entries to the given owner.""" - entries = self.load_all() + try: + entries = self.load_all_for_update() + except MemoryStoreUnreadable as e: + # Skip the sweep rather than rewrite the store from an unknown view. + logger.error("Skipping ownerless claim, memory store unreadable: %s", e) + return changed = False claimed = 0 for entry in entries: @@ -235,7 +300,12 @@ class MemoryManager: if not ids: return id_set = set(ids) - entries = self.load_all() + try: + entries = self.load_all_for_update() + except MemoryStoreUnreadable as e: + # Best-effort counter; never worth rewriting the store blind. + logger.error("Skipping uses bump, memory store unreadable: %s", e) + return changed = False for e in entries: if e.get("id") in id_set: diff --git a/src/memory_provider.py b/src/memory_provider.py index 925c59192..8974a6e84 100644 --- a/src/memory_provider.py +++ b/src/memory_provider.py @@ -157,7 +157,11 @@ class NativeMemoryProvider(MemoryProvider): if metadata: entry["metadata"] = dict(metadata) - memories = self.memory_manager.load_all() + # Strict load: read-modify-write. `load_all` degrades an unreadable + # store to [], which would save this single entry over everything + # already stored (issue #5673). The provider API has no error channel, + # so MemoryStoreUnreadable propagates to the caller. + memories = self.memory_manager.load_all_for_update() memories.append(entry) self.memory_manager.save(memories) @@ -223,7 +227,10 @@ class NativeMemoryProvider(MemoryProvider): ] async def delete(self, memory_id: str, *, owner: Optional[str] = None) -> bool: - memories = self.memory_manager.load_all() + # Strict load for the same reason: `remaining` is derived from this + # list and saved back, so it must never be built from a store we + # failed to read. + memories = self.memory_manager.load_all_for_update() remaining = [] deleted_id = None diff --git a/tests/test_backup_import_cross_user_dedup.py b/tests/test_backup_import_cross_user_dedup.py index 2df5936ef..135be78ee 100644 --- a/tests/test_backup_import_cross_user_dedup.py +++ b/tests/test_backup_import_cross_user_dedup.py @@ -27,6 +27,9 @@ def _setup(monkeypatch, store, user="alice"): mem = MagicMock() mem.load_all.return_value = list(store) + # import_data reads through the strict loader so a store it cannot read is + # never overwritten (#5673); the double has to offer the same entry point. + mem.load_all_for_update.return_value = list(store) saved = {} mem.save.side_effect = lambda entries: saved.__setitem__("entries", entries) diff --git a/tests/test_memory_extractor_vector_cross_tenant.py b/tests/test_memory_extractor_vector_cross_tenant.py index 49702c17f..06ca31667 100644 --- a/tests/test_memory_extractor_vector_cross_tenant.py +++ b/tests/test_memory_extractor_vector_cross_tenant.py @@ -67,6 +67,12 @@ class FakeMemoryManager: def load_all(self): return list(self.rows) + def load_all_for_update(self): + # Mirrors the real MemoryManager: extraction is a read-modify-write and + # goes through the strict loader (#5673). A healthy store behaves the + # same as load_all. + return list(self.rows) + def load(self, owner=None): return [r for r in self.rows if r.get("owner") == owner] diff --git a/tests/test_memory_store_unreadable_no_wipe.py b/tests/test_memory_store_unreadable_no_wipe.py new file mode 100644 index 000000000..4b9076065 --- /dev/null +++ b/tests/test_memory_store_unreadable_no_wipe.py @@ -0,0 +1,255 @@ +"""A memory store that cannot be READ must never be overwritten (issue #5673). + +`MemoryManager.save` is atomic, and the add/import/extract paths are all +read-modify-write: load the whole store, append, save it back. `load_all` +used to answer a *failed read* with `[]` — indistinguishable from "no +memories" — so a failed read turned into + + load_all() -> [] -> [].append(new) -> save([new]) + +which atomically replaced the entire store with one entry. + +The trigger that actually bites is a store that is **readable but not +parseable** — a truncated file, or one holding `{}` instead of `[]`. Nothing +obstructs the write, so the request succeeds with HTTP 200 and every existing +memory is destroyed silently. Truncation is reachable: `core/database.py` +rewrites memory.json during migration with a plain `open(..., "w")` + +`json.dump`, which is not atomic. + +A live exclusive lock is NOT the dangerous case: it blocks the read and the +`os.replace` alike, so the save fails too and the store survives (verified +end-to-end — clean dev returns 500 there and loses nothing). + +`load_all_for_update` is the strict loader those callers now use: it raises +`MemoryStoreUnreadable` rather than reporting an empty store. +""" + +import asyncio +import builtins +import json +import os + +import pytest + +from src.memory import MemoryManager, MemoryStoreUnreadable + +_SEED = [ + {"id": "m1", "text": "user prefers dark mode", "owner": "alice"}, + {"id": "m2", "text": "user lives in Berlin", "owner": "alice"}, + {"id": "m3", "text": "bob's cat is called Mila", "owner": "bob"}, +] + + +def _seeded(tmp_path): + m = MemoryManager(str(tmp_path)) + m.save([dict(e) for e in _SEED]) + return m + + +def _break_reads_of(monkeypatch, target, exc): + """Make open() raise `exc` for `target` only, leaving every other path alone.""" + real_open = builtins.open + + def fake_open(file, mode="r", *args, **kwargs): + if os.path.abspath(str(file)) == os.path.abspath(target) and "r" in mode: + raise exc + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", fake_open) + + +# ── the strict loader signals, rather than reporting "empty" ────────────── + +def test_strict_load_raises_on_permission_error(tmp_path, monkeypatch): + m = _seeded(tmp_path) + _break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked")) + with pytest.raises(MemoryStoreUnreadable): + m.load_all_for_update() + + +def test_strict_load_raises_on_corrupt_json(tmp_path): + m = _seeded(tmp_path) + with open(m.memory_file, "w", encoding="utf-8") as f: + f.write('[{"id": "m1", "text": "truncated mid-writ') + with pytest.raises(MemoryStoreUnreadable): + m.load_all_for_update() + + +def test_strict_load_raises_when_store_is_not_a_list(tmp_path): + # A file holding `{}` or `null` is not an empty store, it is a broken one. + m = _seeded(tmp_path) + with open(m.memory_file, "w", encoding="utf-8") as f: + json.dump({}, f) + with pytest.raises(MemoryStoreUnreadable): + m.load_all_for_update() + + +def test_strict_load_returns_entries_when_healthy(tmp_path): + m = _seeded(tmp_path) + assert {e["id"] for e in m.load_all_for_update()} == {"m1", "m2", "m3"} + + +def test_strict_load_returns_empty_when_file_genuinely_absent(tmp_path): + m = _seeded(tmp_path) + os.remove(m.memory_file) + # Absent is the one case that legitimately means "no memories yet". + assert m.load_all_for_update() == [] + + +# ── read paths stay lenient, so an unreadable store can't break chat ────── + +def test_read_path_still_degrades_to_empty(tmp_path, monkeypatch): + m = _seeded(tmp_path) + _break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked")) + # Context injection / search must not raise; they just see nothing. + assert m.load_all() == [] + assert m.load(owner="alice") == [] + + +# ── the actual #5673 regression: the store survives ─────────────────────── + +def test_add_cycle_under_transient_read_error_does_not_wipe(tmp_path, monkeypatch): + """Mirrors routes/memory/memory_routes.py api_add_memory exactly.""" + m = _seeded(tmp_path) + new_entry = m.add_entry("a brand new fact", owner="alice") + + with monkeypatch.context() as mp: + _break_reads_of(mp, m.memory_file, PermissionError(13, "locked")) + with pytest.raises(MemoryStoreUnreadable): + all_mem = m.load_all_for_update() + all_mem.append(new_entry) + m.save(all_mem) + + # Reads work again; every original memory is still there and the file was + # never replaced by the single new entry. + assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"} + + +def test_audit_merge_cannot_drop_other_tenants(tmp_path, monkeypatch): + """The audit path rebuilds the whole file from load_all + one owner's slice. + + Reading [] there would save only the audited owner's entries and destroy + every other tenant's memories, so it has to fail closed too. + """ + m = _seeded(tmp_path) + alice_slice = [e for e in _SEED if e["owner"] == "alice"] + + with monkeypatch.context() as mp: + _break_reads_of(mp, m.memory_file, PermissionError(13, "locked")) + with pytest.raises(MemoryStoreUnreadable): + all_entries = m.load_all_for_update() + others = [e for e in all_entries if e.get("owner") != "alice"] + m.save(alice_slice + others) + + assert any(e["id"] == "m3" for e in m.load_all()), "bob's memory was destroyed" + + +def test_uses_bump_skips_write_when_unreadable(tmp_path, monkeypatch): + m = _seeded(tmp_path) + with monkeypatch.context() as mp: + _break_reads_of(mp, m.memory_file, PermissionError(13, "locked")) + m.increment_uses(["m1"]) # must not raise, must not write + assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"} + + +def test_claim_ownerless_skips_write_when_unreadable(tmp_path, monkeypatch): + m = _seeded(tmp_path) + with monkeypatch.context() as mp: + _break_reads_of(mp, m.memory_file, PermissionError(13, "locked")) + m.claim_ownerless("alice") + assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"} + + +# ── the add sinks users actually reach ──────────────────────────────────── +# +# The tests above replay the read-modify-write shape. These drive the real +# entry points end to end, because those are what #5673 reports: "remember +# that I prefer X" in ordinary chat (src/ai_interaction.py do_manage_memory, +# routed from src/tool_execution.py) and the built-in memory MCP server +# (mcp_servers/memory_server.py, registered in src/builtin_mcp.py). +# +# They use a truncated store rather than a read error on purpose: it reads +# fine, so nothing stops the save, which is the case that silently destroyed +# stores. The assertion is that the file is left byte-identical — still broken, +# but still holding the user's memories, so it can be repaired by hand. + + +def _truncated_store(tmp_path): + """Seed a store that reads back fine but no longer parses.""" + m = _seeded(tmp_path) + good = json.dumps([dict(e) for e in _SEED], indent=2) + with open(m.memory_file, "w", encoding="utf-8") as f: + f.write(good[:good.rindex("]")]) # drop the closing bracket only + with open(m.memory_file, "rb") as f: + return m, f.read() + + +def _on_disk(manager) -> bytes: + with open(manager.memory_file, "rb") as f: + return f.read() + + +def test_agent_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch): + """src/ai_interaction.py do_manage_memory, action "add".""" + from src import ai_interaction + + manager, before = _truncated_store(tmp_path) + monkeypatch.setattr(ai_interaction, "_memory_manager", manager) + monkeypatch.setattr(ai_interaction, "_memory_vector", None) + + result = asyncio.run(ai_interaction.do_manage_memory("add\nuser prefers tabs")) + + assert _on_disk(manager) == before, "the unreadable store was overwritten" + assert b"m3" in _on_disk(manager) + assert "error" in result, "the add reported success over an unreadable store" + + +def test_mcp_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch): + """mcp_servers/memory_server.py, action "add".""" + import mcp_servers.memory_server as memory_server + + manager, before = _truncated_store(tmp_path) + monkeypatch.setattr(memory_server, "_memory_manager", manager) + monkeypatch.setattr(memory_server, "_memory_vector", None) + monkeypatch.setattr(memory_server, "_initialized", True) + for key in memory_server._OWNER_ENV_KEYS: + monkeypatch.delenv(key, raising=False) + + result = asyncio.run(memory_server.call_tool( + "manage_memory", {"action": "add", "text": "user prefers tabs"} + )) + + assert _on_disk(manager) == before, "the unreadable store was overwritten" + assert b"m3" in _on_disk(manager) + assert result[0].text.startswith("Error:") + + +def test_native_provider_remember_does_not_overwrite_unreadable_store(tmp_path): + """src/memory_provider.py NativeMemoryProvider.remember. + + Registered into app state in src/app_initializer.py but not yet consumed + outside tests, so this is the pattern held in place before it goes live. + """ + from src.memory_provider import NativeMemoryProvider + + manager, before = _truncated_store(tmp_path) + provider = NativeMemoryProvider(manager) + + with pytest.raises(MemoryStoreUnreadable): + asyncio.run(provider.remember("user prefers tabs", owner="alice")) + + assert _on_disk(manager) == before + + +# ── the legacy memory.txt migration is preserved ────────────────────────── + +def test_corrupt_store_still_migrates_from_legacy_txt(tmp_path): + m = _seeded(tmp_path) + with open(m.memory_file, "w", encoding="utf-8") as f: + f.write("{ not json") + legacy = os.path.join(str(tmp_path), "memory.txt") + with open(legacy, "w", encoding="utf-8") as f: + f.write("recovered fact one\nrecovered fact two\n") + + entries = m.load_all_for_update() + assert [e["text"] for e in entries] == ["recovered fact one", "recovered fact two"] From 5ddef23d949b0ebb5ed90d6ae4904798d1619bca Mon Sep 17 00:00:00 2001 From: adabarbulescu <94562950+adabarbulescu@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:12:21 +0300 Subject: [PATCH 36/43] fix(welcome): rotate startup tips (#5871) --- static/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index 8257660fe..0136f2316 100644 --- a/static/index.html +++ b/static/index.html @@ -1005,7 +1005,7 @@ var tips = mobile ? phone : desktop; var el = document.getElementById('welcome-tip'); if (el) { - el.textContent = 'Pick a model if you want, or just type.'; + el.textContent = tips[Math.floor(Math.random() * tips.length)]; } fetch('/api/version').then(function(r){return r.json()}).then(function(d){ if (d.version) window._appVersion = d.version; From 36d409842177e18017dac2fa4bbc5266bb451ac7 Mon Sep 17 00:00:00 2001 From: Jakub Grula <ramsters110@gmail.com> Date: Fri, 7 Aug 2026 19:15:50 +0200 Subject: [PATCH 37/43] fix: Edit box formatting was removing triple tick boxes (#5737) --- static/js/chat.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/js/chat.js b/static/js/chat.js index ea2d8c1bb..ca583c5fc 100644 --- a/static/js/chat.js +++ b/static/js/chat.js @@ -4787,7 +4787,8 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr if (msgIndex < 0) return; const bodyEl = userMsgElement.querySelector('.body'); - const currentText = bodyEl ? bodyEl.textContent.trim().replace(/\s*\[\d+ attachment\(s\)\]$/, '') : ''; + let currentText = (userMsgElement.dataset.raw || (bodyEl ? bodyEl.textContent : '') || '').trim(); + currentText = currentText.replace(/\s*\[\d+ attachment\(s\)\]$/, ''); // Replace body with an editable textarea const editor = document.createElement('textarea'); From f1e96d102e5692fca3a91b1a40f46e22028e7ecb Mon Sep 17 00:00:00 2001 From: Husam <husamemad60@gmail.com> Date: Fri, 7 Aug 2026 20:33:14 +0300 Subject: [PATCH 38/43] fix(tool_parsing): require a pipe on the Qwen bare end marker (#5829) The `end` branch of _QWEN_BARE_MARKER_RE had both pipes optional (`\|?end\|?`), so it also matched a bare `end` between whitespace and replaced it with a space. Messages containing Ruby, Lua or shell code that closes a block with a lone `end` had those lines deleted, and ordinary prose lost the word too. Require at least one pipe so only real turn markers match; `|end`, `end|`, `|end|` and `/|end|` strip exactly as before. Applied to the duplicated pattern in static/js/chatRenderer.js as well. Fixes #5547 --- src/tool_parsing.py | 6 +- static/js/chatRenderer.js | 5 +- tests/test_tool_parsing_bare_end_marker.py | 96 ++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 tests/test_tool_parsing_bare_end_marker.py diff --git a/src/tool_parsing.py b/src/tool_parsing.py index 2885cc00f..98dc1b5f6 100644 --- a/src/tool_parsing.py +++ b/src/tool_parsing.py @@ -187,8 +187,12 @@ _FUNCTION_MODEL_NAME_RE = re.compile( _FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"<parameters>\s*", re.IGNORECASE) _FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"</parameters>", re.IGNORECASE) _QWEN_ROLE_MARKER_RE = re.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", re.IGNORECASE) +# At least one pipe is required around `end`. Both pipes used to be optional +# (`\|?end\|?`), which also matched a bare `end` on its own line and deleted it +# from ordinary prose and from Ruby/Lua/shell snippets that close blocks with +# one; see #5547. `|end`, `end|`, `|end|` and `/|end|` still strip as before. _QWEN_BARE_MARKER_RE = re.compile( - r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|" + r"(?:^|[\t\r\n ])(?:/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|" r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)", re.IGNORECASE, ) diff --git a/static/js/chatRenderer.js b/static/js/chatRenderer.js index 10709679d..1d6e2e4a9 100644 --- a/static/js/chatRenderer.js +++ b/static/js/chatRenderer.js @@ -478,7 +478,10 @@ const DSML_STRAY_RE = /<\s*\/?\s*[｜|]+\s*DSML\s*[｜|]+[^>]*>/gi; const DSML_INVOKE_RE = /<\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\b[^>]*>[\s\S]*?(?:<\s*\/\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\s*>|$)/gi; const RAW_OPENAI_TOOL_JSON_RE = /(?:\[\s*)?\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"[^"]*"\s*,\s*"type"\s*:\s*"function"\s*\}\s*\]?/gi; const QWEN_ROLE_MARKER_RE = /<\/?\|(?:assistant|assistan|user|system|tool)\|>?|<\/\|end\|>?/gi; -const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\|?end\|?|\/?\|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi; +// Keep in sync with _QWEN_BARE_MARKER_RE in src/tool_parsing.py. At least one +// pipe is required around `end`: with both optional (`\|?end\|?`) this also ate +// a bare `end` on its own line, breaking Ruby/Lua/shell snippets (#5547). +const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi; // Self-narration about tool results (model echoing stdout/exit_code) const TOOL_NARRATION_RE = /(?:The (?:result|output) shows?:?\s*)?-?\s*(?:stdout|stderr|exit_code):\s*.+/gi; diff --git a/tests/test_tool_parsing_bare_end_marker.py b/tests/test_tool_parsing_bare_end_marker.py new file mode 100644 index 000000000..6167c8dde --- /dev/null +++ b/tests/test_tool_parsing_bare_end_marker.py @@ -0,0 +1,96 @@ +"""Regression: the Qwen bare-marker scrub must not eat a lone `end` (#5547). + +`_QWEN_BARE_MARKER_RE` cleans Qwen turn markers that leak into content. Its +`end` branch was `\\|?end\\|?` — both pipes optional — so it also matched a bare +`end` surrounded by whitespace and replaced it with a space. Any message +containing Ruby, Lua or shell code that closes a block with a lone `end` had +those lines silently deleted, in the stored text and in the rendered message. + +Requiring at least one pipe keeps every real marker (`|end`, `end|`, `|end|`, +`/|end|`) stripping as before. The same pattern is duplicated in +static/js/chatRenderer.js, so the JS copy is checked here too — the two must +not drift. +""" +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +import src.agent_tools # noqa: F401 (break agent_tools<->tool_parsing import cycle) +from src.tool_parsing import strip_tool_blocks + +_REPO = Path(__file__).resolve().parent.parent +_CHAT_RENDERER = _REPO / "static" / "js" / "chatRenderer.js" + +# Inputs that must survive untouched, and the substring that proves they did. +KEPT = [ + ("loop do\n puts \"yo\"\nend\n", "\nend"), # the reported Ruby case + ("if x then\nend", "\nend"), + ("function f()\nend\n", "\nend"), + ("a end b", "a end b"), + ("append end", "append end"), + ("END", "END"), + ("\nEnd\n", "End"), +] + +# Real markers — at least one pipe, plus the role word — with the exact output +# they must still produce. Asserted as equality rather than "marker not in out" +# so narrowing the pattern can't pass by deleting more than it should. +STRIPPED = [ + ("a |end| b", "a b"), + ("a /|end| b", "a b"), + ("a |end b", "a b"), + ("a end| b", "a b"), + ("x assistant y", "x y"), +] + + +@pytest.mark.parametrize("text,kept", KEPT) +def test_bare_end_survives_stripping(text, kept): + assert kept in strip_tool_blocks(text) + + +@pytest.mark.parametrize("text,expected", STRIPPED) +def test_piped_end_markers_are_still_stripped(text, expected): + assert strip_tool_blocks(text) == expected + + +def test_bare_end_inside_a_fenced_block_survives(): + """The scrub runs over the whole message, fenced regions included.""" + out = strip_tool_blocks("Here:\n```ruby\nloop do\n puts 1\nend\n```\nDone.") + assert "\nend\n" in out + + +def _js_bare_marker_regex_source(): + src = _CHAT_RENDERER.read_text(encoding="utf-8") + m = re.search(r"^const QWEN_BARE_MARKER_RE = (/.*/[gimsuy]*);$", src, re.MULTILINE) + assert m, "QWEN_BARE_MARKER_RE literal not found in chatRenderer.js" + return m.group(1) + + +def test_js_copy_of_the_pattern_matches_the_python_one(): + """Guard the duplication: the JS branch must require a pipe too.""" + if shutil.which("node") is None: + pytest.skip("node binary not on PATH") + + cases = [text for text, _ in KEPT] + [text for text, _ in STRIPPED] + script = ( + "const RE = %s;\n" + "const cases = JSON.parse(process.argv[1]);\n" + "console.log(JSON.stringify(cases.map(c => c.replace(RE, ' '))));" + % _js_bare_marker_regex_source() + ) + result = subprocess.run( + ["node", "--input-type=module", "-e", script, json.dumps(cases)], + cwd=_REPO, capture_output=True, timeout=15, text=True, + ) + assert result.returncode == 0, f"node failed:\n{result.stderr}" + got = json.loads(result.stdout.splitlines()[-1]) + + for (text, kept), out in zip(KEPT, got): + assert kept in out, f"JS regex dropped {kept!r} from {text!r}" + for (text, expected), out in zip(STRIPPED, got[len(KEPT):]): + assert out == expected, f"JS regex: {text!r} -> {out!r}, expected {expected!r}" From 99566d28b53cdd53efaecff05a61ef3b7b5daa06 Mon Sep 17 00:00:00 2001 From: Husam <husamemad60@gmail.com> Date: Fri, 7 Aug 2026 20:34:50 +0300 Subject: [PATCH 39/43] fix(chat): stop ArrowUp from eating an unsent multi-line prompt (#5875) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit static/app.js carried a near-verbatim copy of the prompt-recall logic in static/js/composerArrowUpRecall.js, wired as a second capture-phase keydown listener on the same #message textarea. The copy omitted the draft guard the module has: it called preventDefault() and stopImmediatePropagation() unconditionally, then recalled history[0] over whatever the user had typed. Because it stopped immediate propagation, the copy won regardless of registration order — if it ran first the module never saw the event, and if it ran second the module had already declined to stop propagation on an unmatched draft. The guard at composerArrowUpRecall.js:109 was unreachable on the real page, so ArrowUp on a multi-line draft replaced it with the last sent prompt instead of moving the caret up a line. Delete the duplicate. The module keeps ownership of ArrowUp/ArrowDown recall, which is the behavior MODULE_SUMMARY.md documents ("on an empty composer") and the behavior tests/test_composer_arrow_up_recall_js.py already pins via test_non_empty_composer_does_not_recall and test_multiline_caret_navigation_preserved. Also correct a stale comment in the module that described the deleted behavior and contradicted the guard 35 lines above it, and add a regression test asserting app.js does not reintroduce a second handler. Fixes #5862 --- static/app.js | 83 ++--------------------- static/js/composerArrowUpRecall.js | 6 +- tests/test_composer_arrow_up_recall_js.py | 21 ++++++ 3 files changed, 28 insertions(+), 82 deletions(-) diff --git a/static/app.js b/static/app.js index 97f0ae77e..c9e3a567f 100644 --- a/static/app.js +++ b/static/app.js @@ -3908,85 +3908,10 @@ function startOdysseusApp() { const messageInput = el('message'); const modelPickerWrap = document.getElementById('model-picker-wrap'); - function _readComposerPromptHistory() { - const chatBox = document.getElementById('chat-history'); - if (!chatBox) return []; - return Array.from(chatBox.querySelectorAll('.msg-user')) - .reverse() - .map(msg => { - const body = msg.querySelector('.body'); - return msg.dataset?.raw || (body ? body.textContent : '') || ''; - }) - .filter(Boolean); - } - - if (messageInput && !messageInput._odysseusPromptRecallCapture) { - messageInput._odysseusPromptRecallCapture = true; - let recallHistory = []; - let recallIndex = -1; - let lastRecalled = ''; - const norm = (v) => String(v || '').replace(/\r\n/g, '\n').trimEnd(); - messageInput.addEventListener('input', () => { - if (norm(messageInput.value) === norm(lastRecalled)) return; - recallHistory = []; - recallIndex = -1; - lastRecalled = ''; - try { delete messageInput.dataset.odysseusRecallIndex; } catch {} - }, true); - messageInput.addEventListener('keydown', (e) => { - if (e.key !== 'ArrowUp' && e.key !== 'ArrowDown') return; - if (e.shiftKey || e.altKey || e.ctrlKey || e.metaKey || e.isComposing) return; - if (window._ghostAutocomplete?.isActive?.()) return; - const fresh = _readComposerPromptHistory(); - const history = fresh.length ? fresh : recallHistory; - if (!history.length) return; - const current = norm(messageInput.value); - let currentIndex = current ? history.findIndex(item => norm(item) === current) : -1; - if (current && currentIndex < 0 && current === norm(lastRecalled)) currentIndex = recallIndex; - if (current && currentIndex < 0) { - const markedIndex = Number(messageInput.dataset.odysseusRecallIndex); - if (Number.isInteger(markedIndex) && markedIndex >= 0 && markedIndex < history.length) { - currentIndex = markedIndex; - } - } - e.preventDefault(); - e.stopPropagation(); - e.stopImmediatePropagation(); - if (e.key === 'ArrowDown') { - if (currentIndex < 0) return; - const nextIndex = currentIndex - 1; - if (nextIndex < 0) { - recallHistory = history; - recallIndex = -1; - lastRecalled = ''; - try { delete messageInput.dataset.odysseusRecallIndex; } catch {} - messageInput.value = ''; - try { messageInput.selectionStart = messageInput.selectionEnd = 0; } catch {} - try { uiModule.autoResize(messageInput); } catch {} - return; - } - const recalled = history[nextIndex]; - recallHistory = history; - recallIndex = nextIndex; - lastRecalled = recalled; - try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {} - messageInput.value = recalled; - try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {} - try { uiModule.autoResize(messageInput); } catch {} - return; - } - const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0; - const recalled = history[nextIndex]; - if (!recalled) return; - recallHistory = history; - recallIndex = nextIndex; - lastRecalled = recalled; - try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {} - messageInput.value = recalled; - try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {} - try { uiModule.autoResize(messageInput); } catch {} - }, true); - } + // ArrowUp/ArrowDown prompt recall on #message lives in + // static/js/composerArrowUpRecall.js (wired from chat.js). Do not re-add a + // copy here: two capture-phase listeners on the same textarea meant the one + // without the draft guard won and ate unsent multi-line prompts (#5862). const _sendIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><path d="M12 19V5M5 12l7-7 7 7"/></svg>'; const _micIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"/><path d="M19 10v2a7 7 0 0 1-14 0v-2"/><line x1="12" y1="19" x2="12" y2="23"/><line x1="8" y1="23" x2="16" y2="23"/></svg>'; diff --git a/static/js/composerArrowUpRecall.js b/static/js/composerArrowUpRecall.js index e0b20d6b4..83141bfe9 100644 --- a/static/js/composerArrowUpRecall.js +++ b/static/js/composerArrowUpRecall.js @@ -143,9 +143,9 @@ export function wireArrowUpRecall(composer, getUserMessages, options = {}) { return; } - // ArrowUp owns prompt history in the chat composer. If the current text - // is not already a recalled prompt, start from newest instead of letting - // the browser move the caret inside the textarea. + // ArrowUp walks older prompts. An unmatched draft already returned above, + // so reaching here means the composer is empty or holds a recalled prompt + // — the caret-navigation case is never hijacked. const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0; const recalled = history[nextIndex]; if (!recalled) { diff --git a/tests/test_composer_arrow_up_recall_js.py b/tests/test_composer_arrow_up_recall_js.py index eadc3bc94..022fcbc02 100644 --- a/tests/test_composer_arrow_up_recall_js.py +++ b/tests/test_composer_arrow_up_recall_js.py @@ -306,3 +306,24 @@ def test_integration_recalls_from_chat_history_dom(): ) assert proc.returncode == 0, proc.stderr assert json.loads(proc.stdout.strip()) == {"value": "stored prompt", "prevented": True} + + +def test_prompt_recall_is_not_duplicated_in_app_js(): + """Only composerArrowUpRecall.js may own ArrowUp on #message (issue #5862). + + static/app.js once carried a near-verbatim copy of this recall logic, wired + as a second capture-phase listener on the same textarea. That copy lacked + the draft guard here, and because it called stopImmediatePropagation it won + regardless of registration order — so a typed multi-line prompt was replaced + by the last sent one instead of the caret moving up a line. + """ + app_js = (_REPO / "static" / "app.js").read_text(encoding="utf-8") + for marker in ( + "_odysseusPromptRecallCapture", + "_readComposerPromptHistory", + "odysseusRecallIndex", + ): + assert marker not in app_js, ( + f"static/app.js reintroduces prompt recall ({marker!r}); " + "it belongs to static/js/composerArrowUpRecall.js alone" + ) From f06a0a30a80e6739a7f2d7f9b23ed38a8ffb21fd Mon Sep 17 00:00:00 2001 From: Samy <12219635+touzenesmy@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:04:53 -0400 Subject: [PATCH 40/43] fix(session): restore session URL hash writes (removed in cf4e240a) (#5872) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix: restore session URL hash writes (removed in cf4e240a) Restores history.replaceState() calls in selectSession() and materializePendingSession() that were dropped during the July 23 merge. Without these, chat URLs never update the address bar hash, making sessions unshareable and causing bare-URL reloads to land on the welcome screen instead of restoring the last active chat. Root cause: selectSession() had its hash-write deliberately removed; materializePendingSession() lost its during a larger refactor that added the stale-response and incognito guards. Fixes #5870 (upstream) * fix: session URL hash lost when sending message mid-stream Two independent bugs caused the session hash to disappear from the URL: Bug 1 — ReferenceError in catch block silently killed error recovery In handleChatSubmit, two const variables (streamingTTS at line 1922 and abortCtrl at line 1741) were declared inside the try block but referenced in the catch block. Since const is block-scoped in JavaScript, they were undefined in catch, causing a ReferenceError that silently aborted the error handler. This prevented materializePendingSession() from ever being called, so no hash was written to the URL. Fix: Hoisted both as let declarations before the try { block. Bug 2 — Dual sessions.js ES module instances with mismatched state app.js imported sessions.js with a version query string (?v=20260722ctxheader4) while every other module imported ./sessions.js without one. The browser treated them as different URLs, creating two separate module instances with independent _pendingChat and currentSessionId state. createDirectChat() set pending on one instance while handleChatSubmit() checked hasPendingChat() on the other — so the pending session never materialized. Fix: Removed the version query string from the sessions.js import in app.js and from the modulepreload + script tags in index.html. All modules now share a single sessions.js instance. Bonus guard: _adoptOpenedSessionBeforeAutoCreate() now checks hasPendingChat() before adopting a stale DOM-active session, preventing the send path from landing in the wrong session when a New Chat is pending. --------- Co-authored-by: samy <samy@users.noreply.github.com> --- static/app.js | 4 ++-- static/index.html | 8 ++++---- static/js/chat.js | 9 +++++++-- static/js/sessions.js | 5 +++++ 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/static/app.js b/static/app.js index c9e3a567f..9d05d8991 100644 --- a/static/app.js +++ b/static/app.js @@ -10,14 +10,14 @@ import modelsModule from './js/models.js?v=20260715startupcalm2'; import ragModule from './js/rag.js'; import presetsModule from './js/presets.js'; import searchModule from './js/search.js'; -import chatModule from './js/chat.js?v=20260722ctxheader4'; +import chatModule from './js/chat.js?v=20260801fix1'; import compareModule from './js/compare/index.js?v=20260723compareicon2'; import documentModule from './js/document.js?v=20260722emailfastindex1'; import searchChatModule from './js/search-chat.js'; import { makeWindowDraggable } from './js/windowDrag.js'; import markdownModule from './js/markdown.js'; import chatRenderer from './js/chatRenderer.js?v=20260722emailfastindex1'; -import sessionModule from './js/sessions.js?v=20260722ctxheader4'; +import sessionModule from './js/sessions.js'; import memoryModule from './js/memory.js?v=20260722memoryloading1'; import voiceRecorderModule from './js/voiceRecorder.js'; import censorModule from './js/censor.js'; diff --git a/static/index.html b/static/index.html index 0136f2316..0fea836ba 100644 --- a/static/index.html +++ b/static/index.html @@ -250,9 +250,9 @@ </script> <link rel="stylesheet" href="/static/style.css?v=20260723tasksbulkfeedback1"> <link rel="modulepreload" href="/static/app.js?v=20260723tasksbulkfeedback1"> - <link rel="modulepreload" href="/static/js/chat.js?v=20260722ctxheader4"> + <link rel="modulepreload" href="/static/js/chat.js?v=20260801fix1"> <link rel="modulepreload" href="/static/js/ui.js"> - <link rel="modulepreload" href="/static/js/sessions.js?v=20260722ctxheader4"> + <link rel="modulepreload" href="/static/js/sessions.js"> <link rel="modulepreload" href="/static/js/markdown.js"> </head> <body> @@ -2504,7 +2504,7 @@ <script type="module" src="/static/js/ui.js"></script> <script type="module" src="/static/js/markdown.js"></script> <script type="module" src="/static/js/dragSort.js"></script> -<script type="module" src="/static/js/sessions.js?v=20260722ctxheader4"></script> +<script type="module" src="/static/js/sessions.js"></script> <script type="module" src="/static/js/memory.js?v=20260722memoryloading1"></script> <script type="module" src="/static/js/skills.js"></script> <script type="module" src="/static/js/tourHints.js"></script> @@ -2522,7 +2522,7 @@ <script type="module" src="/static/js/chatRenderer.js?v=20260722emailfastindex1"></script> <script type="module" src="/static/js/codeRunner.js"></script> <script type="module" src="/static/js/chatStream.js?v=20260722emailfastindex1"></script> - <script type="module" src="/static/js/chat.js?v=20260722ctxheader4"></script> + <script type="module" src="/static/js/chat.js?v=20260801fix1"></script> <script type="module" src="/static/js/cookbook.js"></script> <script src="/static/js/cookbookSchedule.js"></script> <script type="module" src="/static/js/search-chat.js"></script> diff --git a/static/js/chat.js b/static/js/chat.js index ca583c5fc..3c8bbe850 100644 --- a/static/js/chat.js +++ b/static/js/chat.js @@ -349,6 +349,9 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr async function _adoptOpenedSessionBeforeAutoCreate() { if (!sessionModule || !sessionModule.getCurrentSessionId || sessionModule.getCurrentSessionId()) return true; + // Don't adopt a stale session when the user explicitly started a New Chat + // (pending state set) — the send path must materialize the pending session. + if (sessionModule.hasPendingChat && sessionModule.hasPendingChat()) return false; const activeRowId = document.querySelector('.list-item.active-session[data-session-id], .session-item.active[data-session-id]')?.dataset?.sessionId || ''; const hashId = _hashSessionCandidate(); const lastSelectedId = String(window.__odysseusLastSelectedSessionId || '').trim(); @@ -1403,6 +1406,8 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr currentAccumulated = ''; currentHolder = null; + let abortCtrl = null; + let streamingTTS = false; try { // Re-enable auto-scroll when user sends a message uiModule.setAutoScroll(true); @@ -1716,7 +1721,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr } - const abortCtrl = new AbortController(); + abortCtrl = new AbortController(); abortCtrl._reason = ''; currentAbort = abortCtrl; @@ -1897,7 +1902,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr let isThinking = false; let thinkingStartTime = null; // Streaming TTS: synthesize sentence-by-sentence during streaming - const streamingTTS = !!(window.aiTTSManager && window.aiTTSManager.autoPlay && window.aiTTSManager.available); + streamingTTS = !!(window.aiTTSManager && window.aiTTSManager.autoPlay && window.aiTTSManager.available); if (streamingTTS) window.aiTTSManager.streamingStart(); // Multi-bubble agent tracking let roundHolder = holder; // Current AI text bubble (changes per round) diff --git a/static/js/sessions.js b/static/js/sessions.js index cf59d478c..edf83c8a4 100644 --- a/static/js/sessions.js +++ b/static/js/sessions.js @@ -1847,6 +1847,10 @@ export async function selectSession(id, { keepSidebar = false, showLoading = tru const _isTransientChat = !!_meta && (_meta.folder === 'Assistant' || _meta.folder === 'Tasks'); if (!_isTransientChat) { Storage.set('lastSessionId', id); + // Update URL hash without triggering hashchange handler + if (window.location.hash !== '#' + id) { + history.replaceState(null, '', '#' + id); + } } // Restore character preset for persistent chats try { @@ -2313,6 +2317,7 @@ export async function materializePendingSession() { currentSessionId = payload.id; if (!isIncognito) { Storage.set('lastSessionId', payload.id); + history.replaceState(null, '', '#' + payload.id); } // Reload the sidebar in the background. Awaiting this used to block the first From 378518f6dfb994481a8ec5fbb0032d4a8c23c4e1 Mon Sep 17 00:00:00 2001 From: Samy <12219635+touzenesmy@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:06:17 -0400 Subject: [PATCH 41/43] Fix #5870: stale skills panel data on tab reopen (#5876) Remove early-return guard in loadSkills() that skipped both API re-fetch and renderSkillsList() when the Skills tab was reopened after first load. The cascade entrance animation is already handled inside renderSkillsList() via _cascadeNext, so the guard was unnecessary and caused deleted/edited skills to remain visible until a full page reload. Co-authored-by: samy <samy@users.noreply.github.com> --- static/js/skills.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/static/js/skills.js b/static/js/skills.js index 84974d446..b45403570 100644 --- a/static/js/skills.js +++ b/static/js/skills.js @@ -83,11 +83,9 @@ export async function loadSkills(cascade = false) { // Play the domino-in entrance on this load (set when the tab is opened, // not for the silent re-loads after an edit/delete). if (cascade) _cascadeNext = true; - if (cascade && loaded && !_loadPromise && _playSkillsCascade()) { - _cascadeNext = false; - updateCount(); - return; - } + // Always re-fetch when the tab is explicitly opened — the cascade + // animation is handled inside renderSkillsList() via _cascadeNext. + // Skipping the fetch here caused stale data on panel close/reopen (#5870). if (_loadPromise) return _loadPromise; _loadPromise = (async () => { try { From e4fa4ae5dd1d709ce4168397bd1d200fec1b2494 Mon Sep 17 00:00:00 2001 From: Wes Huber <wesleybaxterhuber@gmail.com> Date: Fri, 7 Aug 2026 13:07:07 -0700 Subject: [PATCH 42/43] fix(brain): give the Add Memory form a submit button and reliable Enter handling (#5830) The Brain > Add tab rendered only a text input and category select with no submit control, and Enter submission relied on a deprecated keypress listener that is not guaranteed to fire, so the form could not be submitted at all (#5828). Add a labelled submit button styled like the neighbouring Skill Import button (theme-io-btn, inline SVG icon), switch the Enter handler to keydown with preventDefault, ignore IME composition, and pin both submit paths with a source-level regression test. Fixes #5828 Co-authored-by: Claude Fable 5 <noreply@anthropic.com> --- static/app.js | 12 ++++- static/index.html | 1 + tests/test_memory_add_submit_regression.py | 54 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/test_memory_add_submit_regression.py diff --git a/static/app.js b/static/app.js index 9d05d8991..2f1e8d4bf 100644 --- a/static/app.js +++ b/static/app.js @@ -1689,12 +1689,20 @@ function initializeEventListeners() { const newMemoryInput = el('new-memory-input'); if (newMemoryInput) { - newMemoryInput.addEventListener('keypress', (e) => { - if (e.key === 'Enter') { + // keydown, not the deprecated keypress: keypress is not guaranteed to + // fire for Enter everywhere, which left the Add Memory form with no + // working submit path (#5828). + newMemoryInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && !e.isComposing) { + e.preventDefault(); memoryModule.addNewMemory(); } }); } + const newMemoryAddBtn = el('new-memory-add-btn'); + if (newMemoryAddBtn) { + newMemoryAddBtn.addEventListener('click', () => memoryModule.addNewMemory()); + } // Voice recording is handled by the dual-purpose send/mic button (see below) diff --git a/static/index.html b/static/index.html index 0fea836ba..fea4e20ac 100644 --- a/static/index.html +++ b/static/index.html @@ -365,6 +365,7 @@ <span class="skill-rich-ph"><span class="k">Add a memory</span> — e.g. 'I prefer concise replies' <svg class="k" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-left:4px;" aria-hidden="true"><polyline points="9 10 4 15 9 20"/><path d="M20 4v7a4 4 0 0 1-4 4H4"/></svg></span> </div> <select id="new-memory-category" class="memory-edit-cat-select" aria-label="Memory category"></select> + <button type="button" id="new-memory-add-btn" class="theme-io-btn" title="Save this memory" style="flex:none;height:28px;font-size:12px;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:4px;" aria-hidden="true"><line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/></svg>Add</button> </div> </div> <div class="admin-card"> diff --git a/tests/test_memory_add_submit_regression.py b/tests/test_memory_add_submit_regression.py new file mode 100644 index 000000000..450d63003 --- /dev/null +++ b/tests/test_memory_add_submit_regression.py @@ -0,0 +1,54 @@ +"""The Brain > Add Memory form must be submittable (#5828). + +The form previously had no submit button and relied on a deprecated +``keypress`` listener for Enter, which is not guaranteed to fire on all +platforms — leaving the form with no working submit path. Pins: + +- a visible, keyboard-accessible submit button next to the category select; +- the button wired to ``memoryModule.addNewMemory()``; +- Enter handled via ``keydown`` with ``preventDefault()`` (and no lingering + ``keypress`` handler on the input). +""" +from pathlib import Path + +APP_JS = Path("static/app.js") +INDEX_HTML = Path("static/index.html") + + +def _add_memory_row(html): + start = html.index('id="new-memory-input"') + end = html.index("</div>", html.index('id="new-memory-add-btn"', start)) + return html[start:end] + + +def test_add_memory_form_renders_a_submit_button(): + html = INDEX_HTML.read_text() + row = _add_memory_row(html) + + assert 'id="new-memory-category"' in row, "button must sit in the same row as the form fields" + btn_start = row.index('id="new-memory-add-btn"') + btn_tag = row[row.rindex("<button", 0, btn_start):row.index(">", btn_start)] + assert 'type="button"' in btn_tag, "must not rely on implicit submit semantics" + + +def _new_memory_wiring_block(source): + start = source.index("const newMemoryInput = el('new-memory-input');") + end = source.index("// Voice recording", start) + return source[start:end] + + +def test_submit_button_is_wired_to_add_new_memory(): + block = _new_memory_wiring_block(APP_JS.read_text()) + + assert "el('new-memory-add-btn')" in block + assert "addEventListener('click', () => memoryModule.addNewMemory())" in block + + +def test_enter_uses_keydown_with_prevent_default(): + block = _new_memory_wiring_block(APP_JS.read_text()) + + assert "addEventListener('keydown'" in block + assert "addEventListener('keypress'" not in block, "keypress is deprecated and unreliable for Enter" + assert "e.preventDefault();" in block + assert "!e.isComposing" in block, "IME composition must not submit the form" + assert "memoryModule.addNewMemory();" in block From 42da399b4d9d814400b8e19365dffedad5b66ab3 Mon Sep 17 00:00:00 2001 From: Matyas Gosztonyi <matyas@fenyves.net> Date: Sat, 8 Aug 2026 23:06:41 +0200 Subject: [PATCH 43/43] fix(email): route summaries through shared LLM adapter (#5841) * fix(email): route summaries through shared llm adapter * chore(ci): refresh PR checks * fix(email): preserve scheduled summary safeguards --------- Co-authored-by: Matyas Fenyves <16389204+uhhgoat@users.noreply.github.com> --- routes/email_helpers.py | 120 +++++++ routes/email_pollers.py | 32 +- routes/email_routes.py | 94 +++--- static/js/emailLibrary.js | 7 +- static/js/emailLibrary/utils.js | 19 ++ tests/test_email_summary_error_ui_js.py | 52 +++ tests/test_email_summary_llm.py | 406 ++++++++++++++++++++++++ 7 files changed, 670 insertions(+), 60 deletions(-) create mode 100644 tests/test_email_summary_error_ui_js.py create mode 100644 tests/test_email_summary_llm.py diff --git a/routes/email_helpers.py b/routes/email_helpers.py index c8639e1c7..257f5f921 100644 --- a/routes/email_helpers.py +++ b/routes/email_helpers.py @@ -247,6 +247,7 @@ import re as _re_reply _REPLY_OPEN_RE = _re_reply.compile(r"<<<\s*(?:REPLY|SUMMARY|OUTPUT)\s*>>+", _re_reply.I) _REPLY_CLOSE_RE = _re_reply.compile(r"<<<\s*END\s*>>+", _re_reply.I) _REPLY_ROLE_MARKER_RE = _re_reply.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", _re_reply.I) +_SUMMARY_BULLET_RE = _re_reply.compile(r"^(?:[-*\u2022]\s+|\d+[.)]\s+)") def _extract_reply(text: str) -> str: @@ -277,6 +278,125 @@ def _extract_reply(text: str) -> str: return _strip_think(t).strip() +def _build_email_summary_messages(sender: str, subject: str, body_for_llm: str) -> list[dict[str, str]]: + return [ + { + "role": "system", + "content": ( + "You are an email summarizer. Format: 1-3 short bullet points " + "(use '- '). Cover: main point, action items, deadlines. If the " + "email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR " + "CONTENTS - pull invoice totals, deadlines, key clauses, concrete " + "numbers/dates from PDFs/docs into the bullets. Be terse.\n\n" + "OUTPUT FORMAT: Put ONLY the bullet points between these exact " + "markers, each on its own line:\n" + "<<<SUMMARY>>>\n" + "- ...\n" + "<<<END>>>\n" + "Any reasoning must come BEFORE <<<SUMMARY>>> (ideally inside " + "<think>...</think>). Only the text between the markers is kept." + ), + }, + { + "role": "user", + "content": ( + f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}" + "\n\n---\n\nSummarize the email. Output the bullets between " + "<<<SUMMARY>>> and <<<END>>>." + ), + }, + ] + + +async def _generate_email_summary( + url: str, + model: str, + sender: str, + subject: str, + body_for_llm: str, + *, + headers: dict | None = None, + max_tokens: int = 8192, + timeout: int = 180, +) -> str: + """Generate an interactive email summary through the shared LLM adapter.""" + from src.llm_core import llm_call_async + + raw = await llm_call_async( + url=url, + model=model, + messages=_build_email_summary_messages(sender, subject, body_for_llm), + temperature=0.3, + max_tokens=max_tokens, + headers=headers, + timeout=timeout, + workload="foreground", + ) + return _normalize_email_summary(raw) + + +async def _generate_scheduled_email_summary( + url: str, + model: str, + sender: str, + subject: str, + body_for_llm: str, + *, + headers: dict | None = None, + owner: str | None = None, + max_tokens: int = 8192, + timeout: int = 180, +) -> str: + """Generate a scheduled summary through the background task candidate chain.""" + from src.task_endpoint import task_llm_call_async + + raw = await task_llm_call_async( + messages=_build_email_summary_messages(sender, subject, body_for_llm), + fallback_url=url, + fallback_model=model, + fallback_headers=headers, + owner=owner, + temperature=0.3, + max_tokens=max_tokens, + timeout=timeout, + ) + return _normalize_email_summary(raw) + + +def _normalize_email_summary(raw) -> str: + """Extract a stable cache/UI summary from provider output.""" + raw_text = raw or "" + if _REPLY_OPEN_RE.search(raw_text): + summary = _extract_reply(raw_text) + if summary: + return summary + + cleaned = _strip_think(raw_text).strip() + bullets = [ + line.strip() + for line in cleaned.splitlines() + if _SUMMARY_BULLET_RE.match(line.strip()) + ] + if bullets: + return "\n".join(bullets) + return cleaned.strip() + + +EMAIL_SUMMARY_ERROR_CODE = "email_summary_unavailable" +EMAIL_SUMMARY_ERROR_MESSAGE = "Failed to summarize" + + +def _email_summary_failure_log_detail(exc: BaseException) -> str: + """Return useful provider-failure metadata without echoing exception text.""" + detail = f"type={type(exc).__name__}" + status = getattr(exc, "status_code", None) + if status is None: + status = getattr(getattr(exc, "response", None), "status_code", None) + if isinstance(status, int): + detail += f" status={status}" + return detail + + def _apply_email_style_mechanics(text: str) -> str: """Enforce deterministic writing-style mechanics that models often miss.""" if not text: diff --git a/routes/email_pollers.py b/routes/email_pollers.py index 5d96bd0f9..a2507989d 100644 --- a/routes/email_pollers.py +++ b/routes/email_pollers.py @@ -40,6 +40,7 @@ from routes.email_helpers import ( _pre_retrieve_context, _attach_compose_uploads, _cleanup_compose_uploads, _q, SCHEDULED_DB, _EMAIL_REPLY_SYS_PROMPT_BASE, _email_cache_owner_clause, + _generate_scheduled_email_summary, _email_summary_failure_log_detail, ) logger = logging.getLogger(__name__) @@ -653,6 +654,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None no_msgid = 0 examined = 0 _summaries_created = 0 + _summary_failed = 0 _events_created = 0 _replies_drafted = 0 _reply_failed = 0 @@ -785,16 +787,17 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None if need_sum: try: - summary = await task_llm_call_async( - messages=[ - {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."}, - {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."}, - ], - fallback_url=url, fallback_model=model, fallback_headers=headers, + summary = await _generate_scheduled_email_summary( + url=url, + model=model, + sender=sender, + subject=subject, + body_for_llm=body_for_llm, + headers=req_headers, owner=account_owner or None, - temperature=0.3, max_tokens=16384, timeout=240, + max_tokens=16384, + timeout=240, ) - summary = _extract_reply((summary or "").strip()) if summary: _c = _sql3.connect(SCHEDULED_DB) _c.execute(""" @@ -808,10 +811,19 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None _summaries_created += 1 _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") + else: + _summary_failed += 1 + _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) + _detail_lines.append(f"summary empty · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") except Exception as e: + _summary_failed += 1 _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") - logger.warning(f"Auto-summary {uid} failed: {e}") + logger.warning( + "Auto-summary uid=%s failed %s", + _uid_text, + _email_summary_failure_log_detail(e), + ) if need_reply: await _emit_progress(progress_cb, f"Drafting reply {processed + 1}/{_max_process} · checked {examined}/{len(uid_list)}") @@ -1320,6 +1332,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None parts.append(f"processed {processed} new") if auto_sum: parts.append(f"summarized {_summaries_created}") + if _summary_failed: + parts.append(f"{_summary_failed} summary failed") if auto_reply_draft: parts.append(f"drafted {_replies_drafted} repl" + ("y" if _replies_drafted == 1 else "ies")) if _reply_failed: diff --git a/routes/email_routes.py b/routes/email_routes.py index 3c8e407bd..76a744ce1 100644 --- a/routes/email_routes.py +++ b/routes/email_routes.py @@ -57,7 +57,8 @@ from routes.email_helpers import ( _extract_attachment_to_disk, _extract_html, _extract_text, _fetch_sender_thread_context, _pre_retrieve_context, _EMAIL_REPLY_SYS_PROMPT_BASE, _POOL_HOOKS, - _friendly_email_auth_error, + _friendly_email_auth_error, _email_summary_failure_log_detail, + _generate_email_summary, EMAIL_SUMMARY_ERROR_CODE, EMAIL_SUMMARY_ERROR_MESSAGE, SendEmailRequest, ExtractStyleRequest, ATTACHMENTS_DIR, COMPOSE_UPLOADS_DIR, SCHEDULED_DB, attachment_extract_dir, _email_cache_owner_clause, email_translation_body_hash, @@ -4766,8 +4767,6 @@ def setup_email_routes(): """Generate a quick AI summary of an email body.""" try: from src.endpoint_resolver import resolve_endpoint - from src.llm_core import _uses_max_completion_tokens, _restricts_temperature - import requests as _req body = data.get("body", "") subject = data.get("subject", "") @@ -4778,7 +4777,11 @@ def setup_email_routes(): if account_id: _assert_owns_account(account_id, owner) if not body: - return {"success": False, "error": "No body provided"} + return { + "success": False, + "error": "No body provided", + "error_code": "email_summary_missing_body", + } # If we know which UID this is, fetch the raw message and pull # attachment text so the summary can reference invoice totals, @@ -4807,53 +4810,43 @@ def setup_email_routes(): if not url: url, model, headers = resolve_endpoint("default", owner=owner) if not url or not model: - return {"success": False, "error": "No LLM endpoint configured"} + return { + "success": False, + "error": "No model configured for email summaries", + "error_code": "email_summary_not_configured", + } req_headers = {"Content-Type": "application/json"} if headers: req_headers.update(headers) - tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" - payload = { - "model": model, - "messages": [ - {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull invoice totals, deadlines, key clauses, concrete numbers/dates from PDFs/docs into the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."}, - {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."}, - ], - tok_key: 8192, - "temperature": 0.3, - "stream": False, - } - # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature. - if _restricts_temperature(model): - payload.pop("temperature", None) - resp = await asyncio.to_thread( - _req.post, url, json=payload, headers=req_headers, timeout=180 - ) - if not resp.ok: - return {"success": False, "error": f"LLM HTTP {resp.status_code}"} - rdata = resp.json() - msg = (rdata.get("choices") or [{}])[0].get("message", {}) - content = (msg.get("content") or "").strip() - content = _extract_reply(content) + try: + content = await _generate_email_summary( + url=url, + model=model, + sender=sender, + subject=subject, + body_for_llm=body_for_llm, + headers=req_headers, + max_tokens=8192, + timeout=180, + ) + except Exception as e: + logger.warning( + "Email summary LLM call failed %s", + _email_summary_failure_log_detail(e), + ) + return { + "success": False, + "error": EMAIL_SUMMARY_ERROR_MESSAGE, + "error_code": EMAIL_SUMMARY_ERROR_CODE, + } if not content: - # Model put everything in reasoning_content — extract bullet points - rc = (msg.get("reasoning_content") or "").strip() - # Find bullet-point style output (lines starting with -, •, *, or numbered) - bullet_lines = [] - for line in rc.split("\n"): - stripped = line.strip() - if re.match(r"^[-•*]\s+|^\d+[.)]\s+", stripped): - bullet_lines.append(stripped) - if bullet_lines: - content = "\n".join(bullet_lines) - else: - # Last resort: take the last paragraph - paragraphs = [p.strip() for p in rc.split("\n\n") if p.strip()] - content = paragraphs[-1] if paragraphs else rc[:500] - - if not content: - return {"success": False, "error": "Empty response from model"} + return { + "success": False, + "error": "The model returned an empty summary", + "error_code": "email_summary_empty", + } # Cache the summary if we have a message_id mid = data.get("message_id", "") @@ -4876,8 +4869,15 @@ def setup_email_routes(): return {"success": True, "summary": content, "model_used": model} except Exception as e: - logger.error(f"Failed to summarize: {e}") - return {"success": False, "error": "Mail operation failed"} + logger.error( + "Email summary route failed %s", + _email_summary_failure_log_detail(e), + ) + return { + "success": False, + "error": EMAIL_SUMMARY_ERROR_MESSAGE, + "error_code": EMAIL_SUMMARY_ERROR_CODE, + } @router.post("/translate") async def translate_email(data: dict, owner: str = Depends(require_owner)): diff --git a/static/js/emailLibrary.js b/static/js/emailLibrary.js index 6a0d3e294..32b906ddc 100644 --- a/static/js/emailLibrary.js +++ b/static/js/emailLibrary.js @@ -13,7 +13,7 @@ import { makeWindowDraggable } from './windowDrag.js'; import { _esc, _escLinkify, _extractName, _parseTurnMeta, _formatBubbleDate, _formatRecipients, _senderColor, _initials, - _sanitizeHtml, + _sanitizeHtml, _renderEmailSummaryError, _TALON_WROTE, _TALON_FROM, _TALON_SENT, _TALON_SUBJ, _TALON_TO, _TALON_ORIG_RE, _SIG_BLOAT_MIN_CHARS, } from './emailLibrary/utils.js'; @@ -7259,12 +7259,11 @@ async function _generateSummary(reader, data, btn) { if (label) label.textContent = 'Summary'; } } else { - content.innerHTML = `<span style="color:var(--red)">${_esc(result.error || 'Failed to summarize')}</span>`; - panel.remove(); + _renderEmailSummaryError(content, result); } } catch (e) { sp.destroy(); - panel.remove(); + _renderEmailSummaryError(content, null); if (uiModule) uiModule.showError?.('Failed to summarize'); } finally { if (btn) btn.disabled = false; diff --git a/static/js/emailLibrary/utils.js b/static/js/emailLibrary/utils.js index 82a5c86ec..f634c9949 100644 --- a/static/js/emailLibrary/utils.js +++ b/static/js/emailLibrary/utils.js @@ -30,6 +30,25 @@ export function _esc(text) { return div.innerHTML; } +const _EMAIL_SUMMARY_ERROR_MESSAGES = Object.freeze({ + email_summary_missing_body: 'No email body to summarize', + email_summary_not_configured: 'No model configured for email summaries', + email_summary_empty: 'The model returned an empty summary', + email_summary_unavailable: 'Failed to summarize', +}); + +export function _emailSummaryErrorMessage(result) { + const code = String(result?.error_code || ''); + return _EMAIL_SUMMARY_ERROR_MESSAGES[code] || 'Failed to summarize'; +} + +export function _renderEmailSummaryError(container, result) { + const message = container.ownerDocument.createElement('span'); + message.style.color = 'var(--red)'; + message.textContent = _emailSummaryErrorMessage(result); + container.replaceChildren(message); +} + function _attrEsc(text) { return String(text ?? '') .replace(/"/g, '"') diff --git a/tests/test_email_summary_error_ui_js.py b/tests/test_email_summary_error_ui_js.py new file mode 100644 index 000000000..1afc3bec9 --- /dev/null +++ b/tests/test_email_summary_error_ui_js.py @@ -0,0 +1,52 @@ +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + + +_REPO = Path(__file__).resolve().parent.parent +_UTILS = (_REPO / "static" / "js" / "emailLibrary" / "utils.js").as_posix() +_HAS_NODE = shutil.which("node") is not None + +pytestmark = pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH") + + +def test_email_summary_renderer_ignores_untrusted_provider_error_text(): + secret = ( + "endpoint=https://private.example.internal/v1 provider=ollama " + "model=private-model response_body=private-response " + "Authorization: Bearer token-secret-value" + ) + script = f""" + import {{ _renderEmailSummaryError }} from '{_UTILS}'; + const host = {{ + ownerDocument: {{ + createElement() {{ return {{ style: {{}}, textContent: '' }}; }}, + }}, + replaceChildren(node) {{ this.child = node; }}, + }}; + _renderEmailSummaryError(host, {{ + error_code: 'email_summary_unavailable', + error: {json.dumps(secret)}, + }}); + console.log(JSON.stringify({{ + text: host.child.textContent, + color: host.child.style.color, + }})); + """ + + proc = subprocess.run( + ["node", "--input-type=module"], + input=script, + capture_output=True, + text=True, + cwd=str(_REPO), + timeout=30, + ) + + assert proc.returncode == 0, proc.stderr + rendered = json.loads(proc.stdout) + assert rendered == {"text": "Failed to summarize", "color": "var(--red)"} + assert secret not in proc.stdout diff --git a/tests/test_email_summary_llm.py b/tests/test_email_summary_llm.py new file mode 100644 index 000000000..b0ab7b3be --- /dev/null +++ b/tests/test_email_summary_llm.py @@ -0,0 +1,406 @@ +import asyncio +import json +import logging +import os +import sqlite3 +import sys +import tempfile +from pathlib import Path + +import pytest + + +_TMP_DATA = Path(tempfile.mkdtemp(prefix="odysseus-email-summary-")) +os.environ.setdefault("DATA_DIR", str(_TMP_DATA)) +os.environ.setdefault("DATABASE_URL", f"sqlite:///{_TMP_DATA / 'app.db'}") + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +def _route_endpoint(router, path: str, method: str): + method = method.upper() + for route in router.routes: + if route.path == path and method in getattr(route, "methods", set()): + return route.endpoint + raise AssertionError(f"route not found: {method} {path}") + + +@pytest.mark.asyncio +async def test_generate_email_summary_uses_shared_llm_adapter(monkeypatch): + import routes.email_helpers as email_helpers + import src.llm_core as llm_core + + calls = {} + + async def fake_llm_call_async(url, model, messages, **kwargs): + calls["url"] = url + calls["model"] = model + calls["messages"] = messages + calls["kwargs"] = kwargs + return "thinking before marker\n<<<SUMMARY>>>\n- Pay the invoice by Friday.\n<<<END>>>" + + monkeypatch.setattr(llm_core, "llm_call_async", fake_llm_call_async) + + summary = await email_helpers._generate_email_summary( + url="https://chatgpt.com/backend-api/codex/responses", + model="gpt-5.5", + sender="Billing <billing@example.com>", + subject="Invoice due", + body_for_llm="Please pay invoice 123 by Friday.", + headers={"Authorization": "Bearer test"}, + max_tokens=1234, + timeout=45, + ) + + assert summary == "- Pay the invoice by Friday." + assert calls["url"] == "https://chatgpt.com/backend-api/codex/responses" + assert calls["model"] == "gpt-5.5" + assert calls["kwargs"]["headers"] == {"Authorization": "Bearer test"} + assert calls["kwargs"]["temperature"] == 0.3 + assert calls["kwargs"]["max_tokens"] == 1234 + assert calls["kwargs"]["timeout"] == 45 + assert calls["kwargs"]["workload"] == "foreground" + assert calls["messages"][0]["role"] == "system" + assert calls["messages"][1]["role"] == "user" + + +@pytest.mark.asyncio +async def test_scheduled_email_summary_uses_background_fallback_chain(monkeypatch): + import routes.email_helpers as email_helpers + import src.llm_core as llm_core + import src.task_endpoint as task_endpoint + + candidates = [ + ("http://primary.invalid/v1", "primary-model", {"X-Candidate": "primary"}), + ("http://fallback.invalid/v1", "fallback-model", {"X-Candidate": "fallback"}), + ] + resolve_calls = [] + wait_calls = [] + llm_calls = [] + + def fake_resolve_task_candidates(**kwargs): + resolve_calls.append(kwargs) + return candidates + + async def fake_wait_for_interactive_quiet(label): + wait_calls.append(label) + return False + + async def fake_llm_call_async(url, model, messages, **kwargs): + llm_calls.append((url, model, messages, kwargs)) + if model == "primary-model": + raise RuntimeError("primary unavailable") + return "<<<SUMMARY>>>\n- Used the fallback model.\n<<<END>>>" + + monkeypatch.setattr(task_endpoint, "resolve_task_candidates", fake_resolve_task_candidates) + monkeypatch.setattr(task_endpoint, "wait_for_interactive_quiet", fake_wait_for_interactive_quiet) + monkeypatch.setattr(llm_core, "llm_call_async", fake_llm_call_async) + + summary = await email_helpers._generate_scheduled_email_summary( + url="http://caller-fallback.invalid/v1", + model="caller-fallback-model", + sender="Sender <sender@example.com>", + subject="Scheduled subject", + body_for_llm="Please summarize this scheduled email.", + headers={"Authorization": "Bearer test"}, + owner="alice", + max_tokens=321, + timeout=54, + ) + + assert summary == "- Used the fallback model." + assert resolve_calls == [{ + "fallback_url": "http://caller-fallback.invalid/v1", + "fallback_model": "caller-fallback-model", + "fallback_headers": {"Authorization": "Bearer test"}, + "owner": "alice", + }] + assert wait_calls == ["background task LLM"] + assert [call[1] for call in llm_calls] == ["primary-model", "fallback-model"] + assert all(call[3]["workload"] == "background" for call in llm_calls) + assert all(call[3]["max_tokens"] == 321 for call in llm_calls) + assert all(call[3]["timeout"] == 54 for call in llm_calls) + + +@pytest.mark.asyncio +async def test_scheduled_local_summary_is_preempted_by_foreground_call(monkeypatch): + import routes.email_helpers as email_helpers + import src.llm_core as llm_core + import src.task_endpoint as task_endpoint + + local_url = "http://127.0.0.1:11434/v1/chat/completions" + background_started = asyncio.Event() + never_release = asyncio.Event() + observed_workloads = [] + + monkeypatch.setenv("ODYSSEUS_LOCAL_MODEL_GATE", "true") + monkeypatch.setenv("BACKGROUND_TASK_FOREGROUND_GATE", "false") + monkeypatch.setattr(llm_core, "_LOCAL_MODEL_LOCK", asyncio.Lock()) + monkeypatch.setattr(llm_core, "_LOCAL_MODEL_CURRENT", {}) + monkeypatch.setattr(llm_core, "_LOCAL_MODEL_WAITING_FOREGROUND", 0) + monkeypatch.setattr( + task_endpoint, + "resolve_task_candidates", + lambda **_kwargs: [(local_url, "scheduled-model", {})], + ) + + async def fake_wait_for_interactive_quiet(_label): + return False + + async def gated_llm_call(url, model, messages, **kwargs): + assert messages + workload = kwargs.get("workload") + observed_workloads.append(workload) + async with llm_core._local_model_slot(url, model, workload=workload): + background_started.set() + await never_release.wait() + return "unreachable" + + monkeypatch.setattr(task_endpoint, "wait_for_interactive_quiet", fake_wait_for_interactive_quiet) + monkeypatch.setattr(llm_core, "llm_call_async", gated_llm_call) + + background_task = asyncio.create_task(email_helpers._generate_scheduled_email_summary( + url=local_url, + model="scheduled-model", + sender="Sender", + subject="Scheduled", + body_for_llm="Scheduled body", + owner="alice", + )) + foreground_task = None + try: + await asyncio.wait_for(background_started.wait(), timeout=1) + + async def run_foreground(): + async with llm_core._local_model_slot( + local_url, + "interactive-model", + workload="foreground", + ): + return True + + foreground_task = asyncio.create_task(run_foreground()) + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(background_task, timeout=1) + assert await asyncio.wait_for(foreground_task, timeout=1) is True + assert observed_workloads == ["background"] + finally: + for task in (background_task, foreground_task): + if task is not None and not task.done(): + task.cancel() + + +@pytest.mark.asyncio +async def test_manual_email_summary_uses_shared_helper_and_caches(tmp_path, monkeypatch): + import routes.email_helpers as email_helpers + import routes.email_routes as email_routes + import src.endpoint_resolver as endpoint_resolver + + db_path = tmp_path / "scheduled_emails.db" + monkeypatch.setattr(email_helpers, "SCHEDULED_DB", db_path) + monkeypatch.setattr(email_routes, "SCHEDULED_DB", db_path) + email_helpers._init_scheduled_db() + + resolve_calls = [] + + def fake_resolve_endpoint(kind, owner=None): + resolve_calls.append((kind, owner)) + assert kind == "utility" + assert owner == "alice" + return ( + "https://chatgpt.com/backend-api/codex/responses", + "gpt-5.5", + {"Authorization": "Bearer test"}, + ) + + helper_calls = {} + + async def fake_generate_email_summary(**kwargs): + helper_calls.update(kwargs) + return "- Manual summary" + + monkeypatch.setattr(endpoint_resolver, "resolve_endpoint", fake_resolve_endpoint) + monkeypatch.setattr(email_routes, "_generate_email_summary", fake_generate_email_summary) + + router = email_routes.setup_email_routes() + summarize = _route_endpoint(router, "/api/email/summarize", "POST") + + result = await summarize( + { + "body": "This is a long enough email body for manual summary.", + "subject": "Manual subject", + "from": "Sender <sender@example.com>", + "message_id": "<manual@example.com>", + "folder": "INBOX", + }, + owner="alice", + ) + + assert result == { + "success": True, + "summary": "- Manual summary", + "model_used": "gpt-5.5", + } + assert resolve_calls == [("utility", "alice")] + assert helper_calls["url"] == "https://chatgpt.com/backend-api/codex/responses" + assert helper_calls["model"] == "gpt-5.5" + assert helper_calls["headers"]["Authorization"] == "Bearer test" + assert helper_calls["headers"]["Content-Type"] == "application/json" + + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT owner, summary, model_used FROM email_summaries WHERE message_id=?", + ("<manual@example.com>",), + ).fetchone() + finally: + conn.close() + assert row == ("alice", "- Manual summary", "gpt-5.5") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("exception_kind", ["http", "runtime"]) +async def test_manual_email_summary_never_exposes_provider_exception( + monkeypatch, + caplog, + exception_kind, +): + from fastapi import HTTPException + import routes.email_routes as email_routes + import src.endpoint_resolver as endpoint_resolver + + secret_detail = ( + "endpoint=https://private.example.internal/v1 provider=ollama " + "model=private-model response_body=private-response " + "Authorization: Bearer token-secret-value" + ) + + def fake_resolve_endpoint(kind, owner=None): + assert kind == "utility" + assert owner == "alice" + return ( + "https://private.example.internal/v1", + "private-model", + {"Authorization": "Bearer token-secret-value"}, + ) + + async def fail_summary(**_kwargs): + if exception_kind == "http": + raise HTTPException(status_code=502, detail=secret_detail) + raise RuntimeError(secret_detail) + + monkeypatch.setattr(endpoint_resolver, "resolve_endpoint", fake_resolve_endpoint) + monkeypatch.setattr(email_routes, "_generate_email_summary", fail_summary) + caplog.set_level(logging.WARNING, logger=email_routes.__name__) + + router = email_routes.setup_email_routes() + summarize = _route_endpoint(router, "/api/email/summarize", "POST") + result = await summarize( + { + "body": "This email body is long enough to summarize.", + "subject": "Sensitive provider failure", + "from": "Sender <sender@example.com>", + }, + owner="alice", + ) + + assert result == { + "success": False, + "error": "Failed to summarize", + "error_code": "email_summary_unavailable", + } + exposed = json.dumps(result) + caplog.text + for marker in ( + "private.example.internal", + "ollama", + "private-model", + "private-response", + "token-secret-value", + ): + assert marker not in exposed + assert f"type={'HTTPException' if exception_kind == 'http' else 'RuntimeError'}" in caplog.text + + +@pytest.mark.asyncio +async def test_scheduled_email_summary_uses_shared_helper_and_caches(tmp_path, monkeypatch): + import routes.email_helpers as email_helpers + import routes.email_pollers as email_pollers + + db_path = tmp_path / "scheduled_emails.db" + monkeypatch.setattr(email_helpers, "SCHEDULED_DB", db_path) + monkeypatch.setattr(email_pollers, "SCHEDULED_DB", db_path) + email_helpers._init_scheduled_db() + + raw_email = ( + b"From: Sender <sender@example.com>\r\n" + b"To: Alice <alice@example.com>\r\n" + b"Subject: Scheduled subject\r\n" + b"Message-ID: <scheduled@example.com>\r\n" + b"Date: Tue, 01 Jan 2026 12:00:00 +0000\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"\r\n" + + (b"Please review this scheduled summary email. " * 8) + ) + + class FakeImap: + def __init__(self): + self.logout_calls = 0 + + def select(self, _folder, readonly=True): + return "OK", [] + + def uid(self, command, *args): + if command == "SEARCH": + return "OK", [b"1"] + if command == "FETCH": + return "OK", [(b"1 (RFC822)", raw_email)] + raise AssertionError(f"unexpected uid command: {command!r} {args!r}") + + def logout(self): + self.logout_calls += 1 + + fake_conn = FakeImap() + + def fake_resolve_task_candidates(owner=None): + assert owner == "alice" + return [( + "https://chatgpt.com/backend-api/codex/responses", + "gpt-5.5", + {"Authorization": "Bearer test"}, + )] + + helper_calls = {} + + async def fake_generate_email_summary(**kwargs): + helper_calls.update(kwargs) + return "- Scheduled summary" + + monkeypatch.setattr(email_pollers, "_load_settings", lambda: {"email_auto_summarize": True}) + monkeypatch.setattr(email_pollers, "_owner_for_email_account", lambda _account_id: "alice") + monkeypatch.setattr(email_pollers, "_imap_connect", lambda account_id=None, owner="": fake_conn) + monkeypatch.setattr(email_pollers, "_get_email_config", lambda account_id=None, owner="": {"from_address": "alice@example.com"}) + monkeypatch.setattr(email_pollers, "resolve_task_candidates", fake_resolve_task_candidates) + monkeypatch.setattr(email_pollers, "_generate_scheduled_email_summary", fake_generate_email_summary) + + result = await email_pollers._auto_summarize_pass_single(account_id="acct-alice") + + assert "summarized 1" in result + assert "summary failed" not in result + assert helper_calls["url"] == "https://chatgpt.com/backend-api/codex/responses" + assert helper_calls["model"] == "gpt-5.5" + assert helper_calls["headers"]["Authorization"] == "Bearer test" + assert helper_calls["headers"]["Content-Type"] == "application/json" + assert helper_calls["owner"] == "alice" + assert fake_conn.logout_calls == 1 + + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT owner, summary, model_used FROM email_summaries WHERE message_id=?", + ("<scheduled@example.com>",), + ).fetchone() + finally: + conn.close() + assert row == ("alice", "- Scheduled summary", "gpt-5.5")